CORD-333 Minimized OVSDB provider dependency
With this patch, cordvtn doesn't need to care for OVSDB connection state
anymore. It will make a connection to OVSDB server like befor but just
for node init and disconnect the OVSDB right after init is done.
- Changed OvsdbNode to CordVtnNode
- Removed OVSDB connect/disconnect and added initNode instead
- Changed ovsdb* commands to cordvtn-node* command, and removed
connect/disconnect command and added init instead
- Fixed to remove OVSDB device from the system after node init or before
making a connection to work around OVSDB device re-connect issue
Change-Id: If69369a06526947122494b2f7e816e37aa931f2c
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java
index e15bc76..c3bf77c 100644
--- a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java
@@ -15,7 +15,6 @@
*/
package org.onosproject.cordvtn;
-import com.google.common.collect.Collections2;
import com.google.common.collect.Sets;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
@@ -32,6 +31,7 @@
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Host;
+import org.onosproject.net.Port;
import org.onosproject.net.behaviour.BridgeConfig;
import org.onosproject.net.behaviour.BridgeName;
import org.onosproject.net.behaviour.ControllerInfo;
@@ -39,6 +39,7 @@
import org.onosproject.net.behaviour.TunnelConfig;
import org.onosproject.net.behaviour.TunnelDescription;
import org.onosproject.net.behaviour.TunnelName;
+import org.onosproject.net.device.DeviceAdminService;
import org.onosproject.net.device.DeviceEvent;
import org.onosproject.net.device.DeviceListener;
import org.onosproject.net.device.DeviceService;
@@ -54,7 +55,6 @@
import org.onosproject.store.service.ConsistentMap;
import org.onosproject.store.service.Serializer;
import org.onosproject.store.service.StorageService;
-import org.onosproject.store.service.Versioned;
import org.slf4j.Logger;
import java.util.ArrayList;
@@ -84,7 +84,8 @@
private static final int NUM_THREADS = 1;
private static final KryoNamespace.Builder NODE_SERIALIZER = KryoNamespace.newBuilder()
.register(KryoNamespaces.API)
- .register(DefaultOvsdbNode.class);
+ .register(CordVtnNode.class)
+ .register(NodeState.class);
private static final String DEFAULT_BRIDGE_NAME = "br-int";
private static final String DEFAULT_TUNNEL = "vxlan";
private static final Map<String, String> DEFAULT_TUNNEL_OPTIONS = new HashMap<String, String>() {
@@ -112,6 +113,9 @@
protected DriverService driverService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DeviceAdminService adminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected OvsdbController controller;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
@@ -127,12 +131,55 @@
private final BridgeHandler bridgeHandler = new BridgeHandler();
private final VmHandler vmHandler = new VmHandler();
- private ConsistentMap<DeviceId, OvsdbNode> nodeStore;
+ private ConsistentMap<CordVtnNode, NodeState> nodeStore;
+
+ private enum NodeState {
+
+ INIT {
+ @Override
+ public void process(CordVtn cordVtn, CordVtnNode node) {
+ cordVtn.connect(node);
+ }
+ },
+ OVSDB_CONNECTED {
+ @Override
+ public void process(CordVtn cordVtn, CordVtnNode node) {
+ if (!cordVtn.getOvsdbConnectionState(node)) {
+ cordVtn.connect(node);
+ } else {
+ cordVtn.createIntegrationBridge(node);
+ }
+ }
+ },
+ BRIDGE_CREATED {
+ @Override
+ public void process(CordVtn cordVtn, CordVtnNode node) {
+ if (!cordVtn.getOvsdbConnectionState(node)) {
+ cordVtn.connect(node);
+ } else {
+ cordVtn.createTunnelInterface(node);
+ }
+ }
+ },
+ COMPLETE {
+ @Override
+ public void process(CordVtn cordVtn, CordVtnNode node) {
+ cordVtn.postInit(node);
+ }
+ },
+ INCOMPLETE {
+ @Override
+ public void process(CordVtn cordVtn, CordVtnNode node) {
+ }
+ };
+
+ public abstract void process(CordVtn cordVtn, CordVtnNode node);
+ }
@Activate
protected void activate() {
ApplicationId appId = coreService.registerApplication("org.onosproject.cordvtn");
- nodeStore = storageService.<DeviceId, OvsdbNode>consistentMapBuilder()
+ nodeStore = storageService.<CordVtnNode, NodeState>consistentMapBuilder()
.withSerializer(Serializer.using(NODE_SERIALIZER.build()))
.withName("cordvtn-nodestore")
.withApplicationId(appId)
@@ -156,79 +203,22 @@
}
@Override
- public void addNode(OvsdbNode ovsdb) {
- checkNotNull(ovsdb);
+ public void addNode(CordVtnNode node) {
+ checkNotNull(node);
- nodeStore.putIfAbsent(ovsdb.deviceId(), ovsdb);
-
- if (isNodeConnected(ovsdb)) {
- init(ovsdb);
- } else {
- connect(ovsdb);
- }
+ nodeStore.putIfAbsent(node, checkNodeState(node));
+ initNode(node);
}
@Override
- public void deleteNode(OvsdbNode ovsdb) {
- checkNotNull(ovsdb);
+ public void deleteNode(CordVtnNode node) {
+ checkNotNull(node);
- if (deviceService.getDevice(ovsdb.deviceId()) != null) {
- if (deviceService.isAvailable(ovsdb.deviceId())) {
- log.warn("Cannot delete connected node {}", ovsdb.host());
- return;
- }
- }
- nodeStore.remove(ovsdb.deviceId());
- }
-
- @Override
- public void connect(OvsdbNode ovsdb) {
- checkNotNull(ovsdb);
-
- if (!nodeStore.containsKey(ovsdb.deviceId())) {
- log.warn("Node {} does not exist", ovsdb.host());
- return;
+ if (getOvsdbConnectionState(node)) {
+ disconnect(node);
}
- if (!isNodeConnected(ovsdb)) {
- controller.connect(ovsdb.ip(), ovsdb.port());
- }
- }
-
- @Override
- public void disconnect(OvsdbNode ovsdb) {
- checkNotNull(ovsdb);
-
- if (!nodeStore.containsKey(ovsdb.deviceId())) {
- log.warn("Node {} does not exist", ovsdb.host());
- return;
- }
-
- if (isNodeConnected(ovsdb)) {
- OvsdbClientService ovsdbClient = getOvsdbClient(ovsdb);
- ovsdbClient.disconnect();
- }
- }
-
- private void init(OvsdbNode ovsdb) {
- checkNotNull(ovsdb);
-
- if (!nodeStore.containsKey(ovsdb.deviceId())) {
- log.warn("Node {} does not exist", ovsdb.host());
- return;
- }
-
- if (!isNodeConnected(ovsdb)) {
- log.warn("Node {} is not connected", ovsdb.host());
- return;
- }
-
- if (deviceService.getDevice(ovsdb.intBrId()) == null ||
- !deviceService.isAvailable(ovsdb.intBrId())) {
- createIntegrationBridge(ovsdb);
- } else if (!checkVxlanInterface(ovsdb)) {
- createVxlanInterface(ovsdb);
- }
+ nodeStore.remove(node);
}
@Override
@@ -237,64 +227,248 @@
}
@Override
- public OvsdbNode getNode(DeviceId deviceId) {
- Versioned<OvsdbNode> ovsdb = nodeStore.get(deviceId);
- if (ovsdb != null) {
- return ovsdb.value();
- } else {
+ public List<CordVtnNode> getNodes() {
+ List<CordVtnNode> nodes = new ArrayList<>();
+ nodes.addAll(nodeStore.keySet());
+ return nodes;
+ }
+
+ @Override
+ public void initNode(CordVtnNode node) {
+ checkNotNull(node);
+
+ if (!nodeStore.containsKey(node)) {
+ log.warn("Node {} does not exist, add node first", node.hostname());
+ return;
+ }
+
+ NodeState state = getNodeState(node);
+ if (state == null) {
+ return;
+ } else if (state.equals(NodeState.INCOMPLETE)) {
+ state = checkNodeState(node);
+ }
+
+ state.process(this, node);
+ }
+
+ @Override
+ public boolean getNodeInitState(CordVtnNode node) {
+ checkNotNull(node);
+
+ NodeState state = getNodeState(node);
+ return state != null && state.equals(NodeState.COMPLETE);
+ }
+
+ /**
+ * Returns state of a given cordvtn node.
+ *
+ * @param node cordvtn node
+ * @return node state, or null if no such node exists
+ */
+ private NodeState getNodeState(CordVtnNode node) {
+ checkNotNull(node);
+
+ try {
+ return nodeStore.get(node).value();
+ } catch (NullPointerException e) {
+ log.error("Failed to get state of {}", node.hostname());
return null;
}
}
- @Override
- public List<OvsdbNode> getNodes() {
- List<OvsdbNode> ovsdbs = new ArrayList<>();
- ovsdbs.addAll(Collections2.transform(nodeStore.values(), Versioned::value));
- return ovsdbs;
+ /**
+ * Sets a new state for a given cordvtn node.
+ *
+ * @param node cordvtn node
+ * @param newState new node state
+ */
+ private void setNodeState(CordVtnNode node, NodeState newState) {
+ checkNotNull(node);
+
+ log.info("Changed {} state: {}", node.hostname(), newState.toString());
+
+ nodeStore.put(node, newState);
+ newState.process(this, node);
}
- @Override
- public boolean isNodeConnected(OvsdbNode ovsdb) {
- checkNotNull(ovsdb);
+ /**
+ * Checks current state of a given cordvtn node and returns it.
+ *
+ * @param node cordvtn node
+ * @return node state
+ */
+ private NodeState checkNodeState(CordVtnNode node) {
+ checkNotNull(node);
- OvsdbClientService ovsdbClient = getOvsdbClient(ovsdb);
- if (ovsdbClient == null) {
- return false;
+ if (checkIntegrationBridge(node) && checkTunnelInterface(node)) {
+ return NodeState.COMPLETE;
+ } else if (checkIntegrationBridge(node)) {
+ return NodeState.BRIDGE_CREATED;
+ } else if (getOvsdbConnectionState(node)) {
+ return NodeState.OVSDB_CONNECTED;
} else {
- return ovsdbClient.isConnected();
+ return NodeState.INIT;
}
}
- private OvsdbClientService getOvsdbClient(OvsdbNode ovsdb) {
- checkNotNull(ovsdb);
+ /**
+ * Performs tasks after node initialization.
+ *
+ * @param node cordvtn node
+ */
+ private void postInit(CordVtnNode node) {
+ disconnect(node);
+ }
+
+ /**
+ * Returns connection state of OVSDB server for a given node.
+ *
+ * @param node cordvtn node
+ * @return true if it is connected, false otherwise
+ */
+ private boolean getOvsdbConnectionState(CordVtnNode node) {
+ checkNotNull(node);
+
+ OvsdbClientService ovsdbClient = getOvsdbClient(node);
+ return deviceService.isAvailable(node.ovsdbId()) &&
+ ovsdbClient != null && ovsdbClient.isConnected();
+ }
+
+ /**
+ * Connects to OVSDB server for a given node.
+ *
+ * @param node cordvtn node
+ */
+ private void connect(CordVtnNode node) {
+ checkNotNull(node);
+
+ if (!nodeStore.containsKey(node)) {
+ log.warn("Node {} does not exist", node.hostname());
+ return;
+ }
+
+ if (!getOvsdbConnectionState(node)) {
+ // FIXME remove existing OVSDB device to work around OVSDB device re-connect issue
+ if (deviceService.getDevice(node.ovsdbId()) != null) {
+ adminService.removeDevice(node.ovsdbId());
+ }
+ controller.connect(node.ovsdbIp(), node.ovsdbPort());
+ }
+ }
+
+ /**
+ * Disconnects OVSDB server for a given node.
+ *
+ * @param node cordvtn node
+ */
+ private void disconnect(CordVtnNode node) {
+ checkNotNull(node);
+
+ if (!nodeStore.containsKey(node)) {
+ log.warn("Node {} does not exist", node.hostname());
+ return;
+ }
+
+ if (getOvsdbConnectionState(node)) {
+ OvsdbClientService ovsdbClient = getOvsdbClient(node);
+ ovsdbClient.disconnect();
+ }
+
+ // FIXME remove existing OVSDB device to work around OVSDB device re-connect issue
+ if (deviceService.getDevice(node.ovsdbId()) != null) {
+ adminService.removeDevice(node.ovsdbId());
+ }
+ }
+
+ /**
+ * Returns cordvtn node associated with a given OVSDB device.
+ *
+ * @param ovsdbId OVSDB device id
+ * @return cordvtn node, null if it fails to find the node
+ */
+ private CordVtnNode getNodeByOvsdbId(DeviceId ovsdbId) {
+ try {
+ return getNodes().stream()
+ .filter(node -> node.ovsdbId().equals(ovsdbId))
+ .findFirst().get();
+ } catch (NoSuchElementException e) {
+ log.debug("Couldn't find node information for {}", ovsdbId);
+ return null;
+ }
+ }
+
+ /**
+ * Returns cordvtn node associated with a given integration bridge.
+ *
+ * @param bridgeId device id of integration bridge
+ * @return cordvtn node, null if it fails to find the node
+ */
+ private CordVtnNode getNodeByBridgeId(DeviceId bridgeId) {
+ try {
+ return getNodes().stream()
+ .filter(node -> node.intBrId().equals(bridgeId))
+ .findFirst().get();
+ } catch (NoSuchElementException e) {
+ log.debug("Couldn't find node information for {}", bridgeId);
+ return null;
+ }
+ }
+
+ /**
+ * Returns OVSDB client for a given node.
+ *
+ * @param node cordvtn node
+ * @return OVSDB client, or null if it fails to get OVSDB client
+ */
+ private OvsdbClientService getOvsdbClient(CordVtnNode node) {
+ checkNotNull(node);
OvsdbClientService ovsdbClient = controller.getOvsdbClient(
- new OvsdbNodeId(ovsdb.ip(), ovsdb.port().toInt()));
+ new OvsdbNodeId(node.ovsdbIp(), node.ovsdbPort().toInt()));
if (ovsdbClient == null) {
- log.debug("Couldn't find ovsdb client for {}", ovsdb.host());
+ log.debug("Couldn't find OVSDB client for {}", node.hostname());
}
return ovsdbClient;
}
- private void createIntegrationBridge(OvsdbNode ovsdb) {
+ /**
+ * Creates an integration bridge for a given node.
+ *
+ * @param node cordvtn node
+ */
+ private void createIntegrationBridge(CordVtnNode node) {
+ if (checkIntegrationBridge(node)) {
+ return;
+ }
+
List<ControllerInfo> controllers = new ArrayList<>();
Sets.newHashSet(clusterService.getNodes())
.forEach(controller -> {
ControllerInfo ctrlInfo = new ControllerInfo(controller.ip(), OFPORT, "tcp");
controllers.add(ctrlInfo);
});
- String dpid = ovsdb.intBrId().toString().substring(DPID_BEGIN);
+ String dpid = node.intBrId().toString().substring(DPID_BEGIN);
try {
- DriverHandler handler = driverService.createHandler(ovsdb.deviceId());
+ DriverHandler handler = driverService.createHandler(node.ovsdbId());
BridgeConfig bridgeConfig = handler.behaviour(BridgeConfig.class);
bridgeConfig.addBridge(BridgeName.bridgeName(DEFAULT_BRIDGE_NAME), dpid, controllers);
} catch (ItemNotFoundException e) {
- log.warn("Failed to create integration bridge on {}", ovsdb.deviceId());
+ log.warn("Failed to create integration bridge on {}", node.ovsdbId());
}
}
- private void createVxlanInterface(OvsdbNode ovsdb) {
+ /**
+ * Creates tunnel interface to the integration bridge for a given node.
+ *
+ * @param node cordvtn node
+ */
+ private void createTunnelInterface(CordVtnNode node) {
+ if (checkTunnelInterface(node)) {
+ return;
+ }
+
DefaultAnnotations.Builder optionBuilder = DefaultAnnotations.builder();
for (String key : DEFAULT_TUNNEL_OPTIONS.keySet()) {
optionBuilder.set(key, DEFAULT_TUNNEL_OPTIONS.get(key));
@@ -304,38 +478,63 @@
TunnelName.tunnelName(DEFAULT_TUNNEL),
optionBuilder.build());
try {
- DriverHandler handler = driverService.createHandler(ovsdb.deviceId());
+ DriverHandler handler = driverService.createHandler(node.ovsdbId());
TunnelConfig tunnelConfig = handler.behaviour(TunnelConfig.class);
tunnelConfig.createTunnelInterface(BridgeName.bridgeName(DEFAULT_BRIDGE_NAME), description);
} catch (ItemNotFoundException e) {
- log.warn("Failed to create VXLAN interface on {}", ovsdb.deviceId());
+ log.warn("Failed to create tunnel interface on {}", node.ovsdbId());
}
}
- private boolean checkVxlanInterface(OvsdbNode ovsdb) {
+ /**
+ * Checks if integration bridge exists and available.
+ *
+ * @param node cordvtn node
+ * @return true if the bridge is available, false otherwise
+ */
+ private boolean checkIntegrationBridge(CordVtnNode node) {
+ return (deviceService.getDevice(node.intBrId()) != null
+ && deviceService.isAvailable(node.intBrId()));
+ }
+
+ /**
+ * Checks if tunnel interface exists.
+ *
+ * @param node cordvtn node
+ * @return true if the interface exists, false otherwise
+ */
+ private boolean checkTunnelInterface(CordVtnNode node) {
try {
- DriverHandler handler = driverService.createHandler(ovsdb.deviceId());
- BridgeConfig bridgeConfig = handler.behaviour(BridgeConfig.class);
- bridgeConfig.getPorts().stream()
- .filter(p -> p.annotations().value("portName").equals(DEFAULT_TUNNEL))
+ deviceService.getPorts(node.intBrId())
+ .stream()
+ .filter(p -> p.annotations().value("portName").contains(DEFAULT_TUNNEL)
+ && p.isEnabled())
.findAny().get();
- } catch (ItemNotFoundException | NoSuchElementException e) {
+ return true;
+ } catch (NoSuchElementException e) {
return false;
}
- return true;
}
private class InternalDeviceListener implements DeviceListener {
@Override
public void event(DeviceEvent event) {
+
Device device = event.subject();
- ConnectionHandler handler = (device.type() == SWITCH ? bridgeHandler : ovsdbHandler);
+ ConnectionHandler<Device> handler =
+ (device.type().equals(SWITCH) ? bridgeHandler : ovsdbHandler);
switch (event.type()) {
- case DEVICE_ADDED:
- eventExecutor.submit(() -> handler.connected(device));
+ case PORT_ADDED:
+ eventExecutor.submit(() -> bridgeHandler.portAdded(event.port()));
break;
+ case PORT_UPDATED:
+ if (!event.port().isEnabled()) {
+ eventExecutor.submit(() -> bridgeHandler.portRemoved(event.port()));
+ }
+ break;
+ case DEVICE_ADDED:
case DEVICE_AVAILABILITY_CHANGED:
if (deviceService.isAvailable(device.id())) {
eventExecutor.submit(() -> handler.connected(device));
@@ -372,17 +571,15 @@
@Override
public void connected(Device device) {
- log.info("Ovsdb {} is connected", device.id());
-
- OvsdbNode ovsdb = getNode(device.id());
- if (ovsdb != null) {
- init(ovsdb);
+ CordVtnNode node = getNodeByOvsdbId(device.id());
+ if (node != null) {
+ setNodeState(node, checkNodeState(node));
}
}
@Override
public void disconnected(Device device) {
- log.warn("Ovsdb {} is disconnected", device.id());
+ log.info("OVSDB {} is disconnected", device.id());
}
}
@@ -390,26 +587,56 @@
@Override
public void connected(Device device) {
- log.info("Integration Bridge {} is detected", device.id());
-
- OvsdbNode ovsdb;
- try {
- ovsdb = getNodes().stream()
- .filter(node -> node.intBrId().equals(device.id()))
- .findFirst().get();
- } catch (NoSuchElementException e) {
- log.warn("Couldn't find OVSDB associated with {}", device.id());
- return;
- }
-
- if (!checkVxlanInterface(ovsdb)) {
- createVxlanInterface(ovsdb);
+ CordVtnNode node = getNodeByBridgeId(device.id());
+ if (node != null) {
+ setNodeState(node, checkNodeState(node));
}
}
@Override
public void disconnected(Device device) {
- log.info("Integration Bridge {} is vanished", device.id());
+ CordVtnNode node = getNodeByBridgeId(device.id());
+ if (node != null) {
+ log.info("Integration Bridge is disconnected from {}", node.hostname());
+ setNodeState(node, NodeState.INCOMPLETE);
+ }
+ }
+
+ /**
+ * Handles port added situation.
+ * If the added port is tunnel port, proceed remaining node initialization.
+ * Otherwise, do nothing.
+ *
+ * @param port port
+ */
+ public void portAdded(Port port) {
+ if (!port.annotations().value("portName").contains(DEFAULT_TUNNEL)) {
+ return;
+ }
+
+ CordVtnNode node = getNodeByBridgeId((DeviceId) port.element().id());
+ if (node != null) {
+ setNodeState(node, checkNodeState(node));
+ }
+ }
+
+ /**
+ * Handles port removed situation.
+ * If the removed port is tunnel port, proceed remaining node initialization.
+ * Others, do nothing.
+ *
+ * @param port port
+ */
+ public void portRemoved(Port port) {
+ if (!port.annotations().value("portName").contains(DEFAULT_TUNNEL)) {
+ return;
+ }
+
+ CordVtnNode node = getNodeByBridgeId((DeviceId) port.element().id());
+ if (node != null) {
+ log.info("Tunnel interface is removed from {}", node.hostname());
+ setNodeState(node, NodeState.INCOMPLETE);
+ }
}
}