Support to inject external bridge into k8s node for external routing
1. Add group bucket related rules on receiving endpoint events
rather than POD events.
Change-Id: I1152343cf8ff6bbccaed3dc34908a3affbc70980
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java
index e07ad6c..6676847 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java
@@ -42,6 +42,7 @@
private static final String MANAGEMENT_IP = "managementIp";
private static final String DATA_IP = "dataIp";
private static final String INTEGRATION_BRIDGE = "integrationBridge";
+ private static final String EXTERNAL_BRIDGE = "externalBridge";
private static final String STATE = "state";
private static final String MISSING_MESSAGE = " is required in K8sNode";
@@ -60,6 +61,10 @@
result.put(INTEGRATION_BRIDGE, node.intgBridge().toString());
}
+ if (node.extBridge() != null) {
+ result.put(EXTERNAL_BRIDGE, node.extBridge().toString());
+ }
+
if (node.dataIp() != null) {
result.put(DATA_IP, node.dataIp().toString());
}
@@ -95,6 +100,11 @@
nodeBuilder.intgBridge(DeviceId.deviceId(intBridgeJson.asText()));
}
+ JsonNode extBridgeJson = json.get(EXTERNAL_BRIDGE);
+ if (extBridgeJson != null) {
+ nodeBuilder.extBridge(DeviceId.deviceId(extBridgeJson.asText()));
+ }
+
log.trace("node is {}", nodeBuilder.build().toString());
return nodeBuilder.build();
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
index 6fb20b7..83e537c 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
@@ -36,8 +36,10 @@
import org.onosproject.net.behaviour.BridgeDescription;
import org.onosproject.net.behaviour.ControllerInfo;
import org.onosproject.net.behaviour.DefaultBridgeDescription;
+import org.onosproject.net.behaviour.DefaultPatchDescription;
import org.onosproject.net.behaviour.DefaultTunnelDescription;
import org.onosproject.net.behaviour.InterfaceConfig;
+import org.onosproject.net.behaviour.PatchDescription;
import org.onosproject.net.behaviour.TunnelDescription;
import org.onosproject.net.behaviour.TunnelEndPoints;
import org.onosproject.net.behaviour.TunnelKeys;
@@ -65,11 +67,14 @@
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.packet.TpPort.tpPort;
import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snode.api.Constants.EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.GENEVE;
import static org.onosproject.k8snode.api.Constants.GENEVE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.GRE;
import static org.onosproject.k8snode.api.Constants.GRE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_EXTERNAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.PHYSICAL_EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.VXLAN;
import static org.onosproject.k8snode.api.Constants.VXLAN_TUNNEL;
import static org.onosproject.k8snode.api.K8sNodeService.APP_ID;
@@ -188,6 +193,9 @@
if (!deviceService.isAvailable(k8sNode.intgBridge())) {
createBridge(k8sNode, INTEGRATION_BRIDGE, k8sNode.intgBridge());
}
+ if (!deviceService.isAvailable(k8sNode.extBridge())) {
+ createBridge(k8sNode, EXTERNAL_BRIDGE, k8sNode.extBridge());
+ }
}
@Override
@@ -198,6 +206,9 @@
return;
}
+ // create patch ports between integration and external bridges
+ createPatchInterfaces(k8sNode);
+
if (k8sNode.dataIp() != null &&
!isIntfEnabled(k8sNode, VXLAN_TUNNEL)) {
createVxlanTunnelInterface(k8sNode);
@@ -310,6 +321,32 @@
createTunnelInterface(k8sNode, GENEVE, GENEVE_TUNNEL);
}
+ private void createPatchInterfaces(K8sNode k8sNode) {
+ Device device = deviceService.getDevice(k8sNode.ovsdb());
+ if (device == null || !device.is(InterfaceConfig.class)) {
+ log.error("Failed to create patch interface on {}", k8sNode.ovsdb());
+ return;
+ }
+
+ PatchDescription brIntPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(INTEGRATION_BRIDGE)
+ .ifaceName(INTEGRATION_TO_EXTERNAL_BRIDGE)
+ .peer(PHYSICAL_EXTERNAL_BRIDGE)
+ .build();
+
+ PatchDescription brExtPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(EXTERNAL_BRIDGE)
+ .ifaceName(PHYSICAL_EXTERNAL_BRIDGE)
+ .peer(INTEGRATION_TO_EXTERNAL_BRIDGE)
+ .build();
+
+ InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
+ ifaceConfig.addPatchMode(INTEGRATION_TO_EXTERNAL_BRIDGE, brIntPatchDesc);
+ ifaceConfig.addPatchMode(PHYSICAL_EXTERNAL_BRIDGE, brExtPatchDesc);
+ }
+
/**
* Creates a tunnel interface in a given kubernetes node.
*
@@ -397,7 +434,9 @@
return false;
}
- return deviceService.isAvailable(k8sNode.intgBridge());
+ return k8sNode.intgBridge() != null && k8sNode.extBridge() != null &&
+ deviceService.isAvailable(k8sNode.intgBridge()) &&
+ deviceService.isAvailable(k8sNode.extBridge());
case DEVICE_CREATED:
if (k8sNode.dataIp() != null &&
!isIntfEnabled(k8sNode, VXLAN_TUNNEL)) {
@@ -463,6 +502,9 @@
// delete integration bridge from the node
client.dropBridge(INTEGRATION_BRIDGE);
+ // delete external bridge from the node
+ client.dropBridge(EXTERNAL_BRIDGE);
+
// disconnect ovsdb
client.disconnect();
}
@@ -554,8 +596,10 @@
return;
}
+ // TODO: also need to check the external bridge's availability
if (deviceService.isAvailable(device.id())) {
- log.debug("Integration bridge created on {}", k8sNode.hostname());
+ log.debug("Integration bridge created on {}",
+ k8sNode.hostname());
bootstrapNode(k8sNode);
} else if (k8sNode.state() == COMPLETE) {
log.info("Device {} disconnected", device.id());
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
index 0289422..122a27b 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
@@ -156,47 +156,73 @@
public void createNode(K8sNode node) {
checkNotNull(node, ERR_NULL_NODE);
- K8sNode updatedNode;
+ K8sNode intNode;
+ K8sNode extNode;
if (node.intgBridge() == null) {
String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
- updatedNode = node.updateIntgBridge(DeviceId.deviceId(deviceIdStr));
- checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
- NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ intNode = node.updateIntgBridge(DeviceId.deviceId(deviceIdStr));
+ checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
+ NOT_DUPLICATED_MSG, intNode.intgBridge());
} else {
- updatedNode = node;
- checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
- NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ intNode = node;
+ checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
+ NOT_DUPLICATED_MSG, intNode.intgBridge());
}
- nodeStore.createNode(updatedNode);
- log.info(String.format(MSG_NODE, updatedNode.hostname(), MSG_CREATED));
+ if (intNode.extBridge() == null) {
+ String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
+ checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
+ extNode = intNode.updateExtBridge(DeviceId.deviceId(deviceIdStr));
+ checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
+ NOT_DUPLICATED_MSG, extNode.extBridge());
+ } else {
+ extNode = intNode;
+ checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
+ NOT_DUPLICATED_MSG, extNode.extBridge());
+ }
+
+ nodeStore.createNode(extNode);
+ log.info(String.format(MSG_NODE, extNode.hostname(), MSG_CREATED));
}
@Override
public void updateNode(K8sNode node) {
checkNotNull(node, ERR_NULL_NODE);
- K8sNode updatedNode;
+ K8sNode intNode;
+ K8sNode extNode;
K8sNode existingNode = nodeStore.node(node.hostname());
checkNotNull(existingNode, ERR_NULL_NODE);
- DeviceId existDeviceId = nodeStore.node(node.hostname()).intgBridge();
+ DeviceId existIntgBridge = nodeStore.node(node.hostname()).intgBridge();
if (node.intgBridge() == null) {
- updatedNode = node.updateIntgBridge(existDeviceId);
- checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
- NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ intNode = node.updateIntgBridge(existIntgBridge);
+ checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
+ NOT_DUPLICATED_MSG, intNode.intgBridge());
} else {
- updatedNode = node;
- checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
- NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ intNode = node;
+ checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
+ NOT_DUPLICATED_MSG, intNode.intgBridge());
}
- nodeStore.updateNode(updatedNode);
- log.info(String.format(MSG_NODE, updatedNode.hostname(), MSG_UPDATED));
+ DeviceId existExtBridge = nodeStore.node(node.hostname()).extBridge();
+
+ if (intNode.extBridge() == null) {
+ extNode = intNode.updateExtBridge(existExtBridge);
+ checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
+ NOT_DUPLICATED_MSG, extNode.extBridge());
+ } else {
+ extNode = intNode;
+ checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
+ NOT_DUPLICATED_MSG, extNode.extBridge());
+ }
+
+ nodeStore.updateNode(extNode);
+ log.info(String.format(MSG_NODE, extNode.hostname(), MSG_UPDATED));
}
@Override
@@ -242,6 +268,7 @@
return nodeStore.node(hostname);
}
+ // TODO: need to differentiate integration bridge and external bridge
@Override
public K8sNode node(DeviceId deviceId) {
return nodeStore.nodes().stream()
@@ -253,7 +280,16 @@
private boolean hasIntgBridge(DeviceId deviceId, String hostname) {
Optional<K8sNode> existNode = nodeStore.nodes().stream()
.filter(n -> !n.hostname().equals(hostname))
- .filter(n -> n.intgBridge().equals(deviceId))
+ .filter(n -> deviceId.equals(n.intgBridge()))
+ .findFirst();
+
+ return existNode.isPresent();
+ }
+
+ private boolean hasExtBridge(DeviceId deviceId, String hostname) {
+ Optional<K8sNode> existNode = nodeStore.nodes().stream()
+ .filter(n -> !n.hostname().equals(hostname))
+ .filter(n -> deviceId.equals(n.extBridge()))
.findFirst();
return existNode.isPresent();
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
index a07d60e..fbccdd0 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
@@ -76,10 +76,15 @@
private static final Device MINION_2_INTG_DEVICE = createDevice(2);
private static final Device MINION_3_INTG_DEVICE = createDevice(3);
+ private static final Device MINION_1_EXT_DEVICE = createDevice(4);
+ private static final Device MINION_2_EXT_DEVICE = createDevice(5);
+ private static final Device MINION_3_EXT_DEVICE = createDevice(6);
+
private static final K8sNode MINION_1 = createNode(
MINION_1_HOSTNAME,
MINION,
MINION_1_INTG_DEVICE,
+ MINION_1_EXT_DEVICE,
IpAddress.valueOf("10.100.0.1"),
INIT
);
@@ -87,6 +92,7 @@
MINION_2_HOSTNAME,
MINION,
MINION_2_INTG_DEVICE,
+ MINION_2_EXT_DEVICE,
IpAddress.valueOf("10.100.0.2"),
INIT
);
@@ -94,6 +100,7 @@
MINION_3_HOSTNAME,
MINION,
MINION_3_INTG_DEVICE,
+ MINION_3_EXT_DEVICE,
IpAddress.valueOf("10.100.0.3"),
COMPLETE
);
@@ -320,12 +327,13 @@
}
private static K8sNode createNode(String hostname, K8sNode.Type type,
- Device intgBridge, IpAddress ipAddr,
- K8sNodeState state) {
+ Device intgBridge, Device extBridge,
+ IpAddress ipAddr, K8sNodeState state) {
return DefaultK8sNode.builder()
.hostname(hostname)
.type(type)
.intgBridge(intgBridge.id())
+ .extBridge(extBridge.id())
.managementIp(ipAddr)
.dataIp(ipAddr)
.state(state)
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java
index 90f7d0d..207bd0e 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java
@@ -94,6 +94,7 @@
.dataIp(IpAddress.valueOf("10.134.34.222"))
.managementIp(IpAddress.valueOf("10.134.231.30"))
.intgBridge(DeviceId.deviceId("of:00000000000000a1"))
+ .extBridge(DeviceId.deviceId("of:00000000000000b1"))
.state(K8sNodeState.INIT)
.build();
diff --git a/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/codec/K8sMinionNode.json b/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/codec/K8sMinionNode.json
index a6760f5..9ed649e 100644
--- a/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/codec/K8sMinionNode.json
+++ b/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/codec/K8sMinionNode.json
@@ -3,5 +3,6 @@
"type": "MINION",
"managementIp": "172.16.130.4",
"dataIp": "172.16.130.4",
- "integrationBridge": "of:00000000000000a1"
+ "integrationBridge": "of:00000000000000a1",
+ "externalBridge": "of:00000000000000b1"
}
\ No newline at end of file
diff --git a/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/web/k8s-node-minion-config.json b/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/web/k8s-node-minion-config.json
index 725766e..8c65bb5 100644
--- a/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/web/k8s-node-minion-config.json
+++ b/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/web/k8s-node-minion-config.json
@@ -5,7 +5,8 @@
"type" : "MINION",
"managementIp" : "10.134.231.32",
"dataIp" : "10.134.34.224",
- "integrationBridge" : "of:00000000000000a2"
+ "integrationBridge" : "of:00000000000000a2",
+ "externalBridge" : "of:00000000000000b2"
}
]
}
\ No newline at end of file