CORD-280 Enhanced initial node setup process considering various scenarios
- Handled situations where ovsdb, integration bridge, or vxlan port are
already connected or exists when the application activated
- Don't make use of mastership for ovsdb device, it does not work well in
device disconnected or re-connected situations
Change-Id: I002948f4a06126430f6019c79a0d84df16c9399c
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java
index ba70780..2813205 100644
--- a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtn.java
@@ -27,7 +27,6 @@
import org.onosproject.cluster.ClusterService;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
-import org.onosproject.mastership.MastershipService;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Host;
@@ -52,6 +51,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -75,7 +75,8 @@
.register(KryoNamespaces.API)
.register(DefaultOvsdbNode.class);
private static final String DEFAULT_BRIDGE_NAME = "br-int";
- private static final Map<String, String> VXLAN_OPTIONS = new HashMap<String, String>() {
+ private static final String DEFAULT_TUNNEL = "vxlan";
+ private static final Map<String, String> DEFAULT_TUNNEL_OPTIONS = new HashMap<String, String>() {
{
put("key", "flow");
put("local_ip", "flow");
@@ -98,9 +99,6 @@
protected HostService hostService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected MastershipService mastershipService;
-
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected OvsdbController controller;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
@@ -117,11 +115,10 @@
private final VmHandler vmHandler = new VmHandler();
private ConsistentMap<DeviceId, OvsdbNode> nodeStore;
- private ApplicationId appId;
@Activate
protected void activate() {
- appId = coreService.registerApplication("org.onosproject.cordvtn");
+ ApplicationId appId = coreService.registerApplication("org.onosproject.cordvtn");
nodeStore = storageService.<DeviceId, OvsdbNode>consistentMapBuilder()
.withSerializer(Serializer.using(NODE_SERIALIZER.build()))
.withName("cordvtn-nodestore")
@@ -148,7 +145,16 @@
@Override
public void addNode(OvsdbNode ovsdb) {
checkNotNull(ovsdb);
- nodeStore.put(ovsdb.deviceId(), ovsdb);
+
+ if (!nodeStore.containsKey(ovsdb.deviceId())) {
+ nodeStore.put(ovsdb.deviceId(), ovsdb);
+ }
+
+ if (isNodeConnected(ovsdb)) {
+ init(ovsdb);
+ } else {
+ connect(ovsdb);
+ }
}
@Override
@@ -159,12 +165,13 @@
return;
}
- // check ovsdb and integration bridge connection state first
- if (isNodeConnected(ovsdb)) {
- log.warn("Cannot delete connected node {}", ovsdb.host());
- } else {
- nodeStore.remove(ovsdb.deviceId());
+ if (deviceService.getDevice(ovsdb.deviceId()) != null) {
+ if (deviceService.isAvailable(ovsdb.deviceId())) {
+ log.warn("Cannot delete connected node {}", ovsdb.host());
+ return;
+ }
}
+ nodeStore.remove(ovsdb.deviceId());
}
@Override
@@ -175,7 +182,10 @@
log.warn("Node {} does not exist", ovsdb.host());
return;
}
- controller.connect(ovsdb.ip(), ovsdb.port());
+
+ if (!isNodeConnected(ovsdb)) {
+ controller.connect(ovsdb.ip(), ovsdb.port());
+ }
}
@Override
@@ -187,14 +197,33 @@
return;
}
- OvsdbClientService ovsdbClient = getOvsdbClient(ovsdb);
- checkNotNull(ovsdbClient);
-
- if (ovsdbClient.isConnected()) {
+ if (isNodeConnected(ovsdb)) {
+ OvsdbClientService ovsdbClient = getOvsdbClient(ovsdb);
ovsdbClient.disconnect();
}
}
+ private void init(OvsdbNode ovsdb) {
+ checkNotNull(ovsdb);
+
+ if (!nodeStore.containsKey(ovsdb.deviceId())) {
+ log.warn("Node {} does not exist", ovsdb.host());
+ return;
+ }
+
+ if (!isNodeConnected(ovsdb)) {
+ log.warn("Node {} is not connected", ovsdb.host());
+ return;
+ }
+
+ if (deviceService.getDevice(ovsdb.intBrId()) == null ||
+ !deviceService.isAvailable(ovsdb.intBrId())) {
+ createIntegrationBridge(ovsdb);
+ } else if (!checkVxlanPort(ovsdb)) {
+ createVxlanPort(ovsdb);
+ }
+ }
+
@Override
public int getNodeCount() {
return nodeStore.size();
@@ -235,11 +264,45 @@
OvsdbClientService ovsdbClient = controller.getOvsdbClient(
new OvsdbNodeId(ovsdb.ip(), ovsdb.port().toInt()));
if (ovsdbClient == null) {
- log.warn("Couldn't find ovsdb client of node {}", ovsdb.host());
+ log.debug("Couldn't find ovsdb client for {}", ovsdb.host());
}
return ovsdbClient;
}
+ private void createIntegrationBridge(OvsdbNode ovsdb) {
+ List<ControllerInfo> controllers = new ArrayList<>();
+ Sets.newHashSet(clusterService.getNodes())
+ .forEach(controller -> {
+ ControllerInfo ctrlInfo = new ControllerInfo(controller.ip(), OFPORT, "tcp");
+ controllers.add(ctrlInfo);
+ });
+ String dpid = ovsdb.intBrId().toString().substring(DPID_BEGIN);
+
+ // TODO change to use bridge config
+ OvsdbClientService ovsdbClient = getOvsdbClient(ovsdb);
+ ovsdbClient.createBridge(DEFAULT_BRIDGE_NAME, dpid, controllers);
+ }
+
+ private void createVxlanPort(OvsdbNode ovsdb) {
+ // TODO change to use tunnel config and tunnel description
+ OvsdbClientService ovsdbClient = getOvsdbClient(ovsdb);
+ ovsdbClient.createTunnel(DEFAULT_BRIDGE_NAME, DEFAULT_TUNNEL,
+ DEFAULT_TUNNEL, DEFAULT_TUNNEL_OPTIONS);
+ }
+
+ private boolean checkVxlanPort(OvsdbNode ovsdb) {
+ // TODO change to use tunnel config
+ OvsdbClientService ovsdbClient = getOvsdbClient(ovsdb);
+ try {
+ ovsdbClient.getPorts().stream()
+ .filter(p -> p.portName().value().equals(DEFAULT_TUNNEL))
+ .findFirst().get();
+ } catch (NoSuchElementException e) {
+ return false;
+ }
+ return true;
+ }
+
private class InternalDeviceListener implements DeviceListener {
@Override
@@ -252,8 +315,11 @@
eventExecutor.submit(() -> handler.connected(device));
break;
case DEVICE_AVAILABILITY_CHANGED:
- eventExecutor.submit(() -> handler.disconnected(device));
- // TODO handle the case that the device is recovered
+ if (deviceService.isAvailable(device.id())) {
+ eventExecutor.submit(() -> handler.connected(device));
+ } else {
+ eventExecutor.submit(() -> handler.disconnected(device));
+ }
break;
default:
break;
@@ -286,20 +352,10 @@
public void connected(Device device) {
log.info("Ovsdb {} is connected", device.id());
- if (!mastershipService.isLocalMaster(device.id())) {
- return;
- }
-
- // TODO change to use bridge config
OvsdbNode ovsdb = getNode(device.id());
- OvsdbClientService ovsdbClient = getOvsdbClient(ovsdb);
-
- List<ControllerInfo> controllers = new ArrayList<>();
- Sets.newHashSet(clusterService.getNodes()).forEach(controller ->
- controllers.add(new ControllerInfo(controller.ip(), OFPORT, "tcp")));
- String dpid = ovsdb.intBrId().toString().substring(DPID_BEGIN);
-
- ovsdbClient.createBridge(DEFAULT_BRIDGE_NAME, dpid, controllers);
+ if (ovsdb != null) {
+ init(ovsdb);
+ }
}
@Override
@@ -314,22 +370,19 @@
public void connected(Device device) {
log.info("Integration Bridge {} is detected", device.id());
- OvsdbNode ovsdb = getNodes().stream()
- .filter(node -> node.intBrId().equals(device.id()))
- .findFirst().get();
-
- if (ovsdb == null) {
+ OvsdbNode ovsdb;
+ try {
+ ovsdb = getNodes().stream()
+ .filter(node -> node.intBrId().equals(device.id()))
+ .findFirst().get();
+ } catch (NoSuchElementException e) {
log.warn("Couldn't find OVSDB associated with {}", device.id());
return;
}
- if (!mastershipService.isLocalMaster(ovsdb.deviceId())) {
- return;
+ if (!checkVxlanPort(ovsdb)) {
+ createVxlanPort(ovsdb);
}
-
- // TODO change to use tunnel config and tunnel description
- OvsdbClientService ovsdbClient = getOvsdbClient(ovsdb);
- ovsdbClient.createTunnel(DEFAULT_BRIDGE_NAME, "vxlan", "vxlan", VXLAN_OPTIONS);
}
@Override
diff --git a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnConfigManager.java b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnConfigManager.java
index f276c7c..287f2a3 100644
--- a/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnConfigManager.java
+++ b/apps/cordvtn/src/main/java/org/onosproject/cordvtn/CordVtnConfigManager.java
@@ -94,7 +94,6 @@
DefaultOvsdbNode ovsdb = new DefaultOvsdbNode(
node.host(), node.ip(), node.port(), node.bridgeId());
cordVtnService.addNode(ovsdb);
- cordVtnService.connect(ovsdb);
});
}