Support to inject external bridge into k8s node for external routing
1. Add group bucket related rules on receiving endpoint events
rather than POD events.
Change-Id: I1152343cf8ff6bbccaed3dc34908a3affbc70980
diff --git a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/K8sGroupRuleService.java b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/K8sGroupRuleService.java
index d91b0be..adc040a 100644
--- a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/K8sGroupRuleService.java
+++ b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/K8sGroupRuleService.java
@@ -51,6 +51,9 @@
/**
* Configures buckets to the existing group.
+ * With install flag true, this method will add buckets to existing buckets,
+ * while with install flag false, this method will remove buckets from
+ * existing buckets.
*
* @param appId application ID
* @param deviceId device ID
@@ -60,4 +63,15 @@
*/
void setBuckets(ApplicationId appId, DeviceId deviceId, int groupId,
List<GroupBucket> buckets, boolean install);
+
+ /**
+ * Configures buckets.
+ *
+ * @param appId application ID
+ * @param deviceId device ID
+ * @param groupId group ID
+ * @param buckets a lit of group buckets
+ */
+ void setBuckets(ApplicationId appId, DeviceId deviceId, int groupId,
+ List<GroupBucket> buckets);
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sGroupRuleManager.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sGroupRuleManager.java
index 51dc9c8..cb2b864 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sGroupRuleManager.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sGroupRuleManager.java
@@ -97,4 +97,11 @@
new GroupBuckets(buckets), getGroupKey(groupId), appId);
}
}
+
+ @Override
+ public void setBuckets(ApplicationId appId, DeviceId deviceId, int groupId,
+ List<GroupBucket> buckets) {
+ groupService.setBucketsForGroup(deviceId, getGroupKey(groupId),
+ new GroupBuckets(buckets), getGroupKey(groupId), appId);
+ }
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
index 76785df..799ecb8 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
@@ -21,7 +21,6 @@
import io.fabric8.kubernetes.api.model.EndpointPort;
import io.fabric8.kubernetes.api.model.EndpointSubset;
import io.fabric8.kubernetes.api.model.Endpoints;
-import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.ServicePort;
import org.onlab.packet.Ethernet;
@@ -47,9 +46,6 @@
import org.onosproject.k8snetworking.api.K8sNetworkEvent;
import org.onosproject.k8snetworking.api.K8sNetworkListener;
import org.onosproject.k8snetworking.api.K8sNetworkService;
-import org.onosproject.k8snetworking.api.K8sPodEvent;
-import org.onosproject.k8snetworking.api.K8sPodListener;
-import org.onosproject.k8snetworking.api.K8sPodService;
import org.onosproject.k8snetworking.api.K8sServiceEvent;
import org.onosproject.k8snetworking.api.K8sServiceListener;
import org.onosproject.k8snetworking.api.K8sServiceService;
@@ -70,7 +66,6 @@
import org.onosproject.net.flow.criteria.ExtensionSelector;
import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.onosproject.net.group.GroupBucket;
-import org.onosproject.store.service.AtomicCounter;
import org.onosproject.store.service.StorageService;
import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
@@ -142,17 +137,11 @@
private static final int HOST_CIDR_NUM = 32;
- private static final String NONE = "None";
private static final String CLUSTER_IP = "ClusterIP";
private static final String TCP = "TCP";
private static final String UDP = "UDP";
private static final String SERVICE_IP_NAT_MODE = "serviceIpNatMode";
- private static final String GROUP_ID_COUNTER_NAME = "group-id-counter";
-
- private static final String IP_ADDRESS = "ipAddress";
- private static final String KUBERNETES = "kubernetes";
-
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected CoreService coreService;
@@ -192,8 +181,6 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected K8sServiceService k8sServiceService;
- @Reference(cardinality = ReferenceCardinality.MANDATORY)
- protected K8sPodService k8sPodService;
/** Service IP address translation mode. */
private String serviceIpNatMode = SERVICE_IP_NAT_MODE_DEFAULT;
@@ -207,15 +194,11 @@
new InternalNodeEventListener();
private final InternalK8sServiceListener internalK8sServiceListener =
new InternalK8sServiceListener();
- private final InternalK8sPodListener internalK8sPodListener =
- new InternalK8sPodListener();
private final InternalK8sEndpointsListener internalK8sEndpointsListener =
new InternalK8sEndpointsListener();
private final InternalK8sNetworkListener internalK8sNetworkListener =
new InternalK8sNetworkListener();
- private AtomicCounter groupIdCounter;
-
private ApplicationId appId;
private NodeId localNodeId;
@@ -227,19 +210,15 @@
leadershipService.runForLeadership(appId.name());
k8sNodeService.addListener(internalNodeEventListener);
k8sServiceService.addListener(internalK8sServiceListener);
- k8sPodService.addListener(internalK8sPodListener);
k8sEndpointsService.addListener(internalK8sEndpointsListener);
k8sNetworkService.addListener(internalK8sNetworkListener);
- groupIdCounter = storageService.getAtomicCounter(GROUP_ID_COUNTER_NAME);
-
log.info("Started");
}
@Deactivate
protected void deactivate() {
leadershipService.withdraw(appId.name());
- k8sPodService.removeListener(internalK8sPodListener);
k8sNodeService.removeListener(internalNodeEventListener);
k8sServiceService.removeListener(internalK8sServiceListener);
k8sEndpointsService.removeListener(internalK8sEndpointsListener);
@@ -376,64 +355,55 @@
return map;
}
- private void setGroupBucketsFromPod(DeviceId deviceId, Service service,
- Pod pod, boolean install) {
- if (pod.getMetadata().getAnnotations() == null) {
- return;
- }
-
- String podIpStr = pod.getMetadata().getAnnotations().get(IP_ADDRESS);
-
- setGroupBuckets(deviceId, service, podIpStr, install);
- }
-
- private void setGroupBuckets(DeviceId deviceId, Service service,
- String podIp, boolean install) {
+ private void setGroupBuckets(Service service, boolean install) {
Map<ServicePort, Set<String>> spEpasMap = getSportEpAddressMap(service);
Map<ServicePort, List<GroupBucket>> spGrpBkts = Maps.newConcurrentMap();
Map<String, String> nodeIpGatewayIpMap =
nodeIpGatewayIpMap(k8sNodeService, k8sNetworkService);
- spEpasMap.forEach((sp, epas) -> {
- List<GroupBucket> bkts = Lists.newArrayList();
+ for (K8sNode node : k8sNodeService.completeNodes()) {
+ spEpasMap.forEach((sp, epas) -> {
+ List<GroupBucket> bkts = Lists.newArrayList();
- if (install) {
- if (epas.contains(podIp)) {
- bkts = buildBuckets(deviceId,
- nodeIpGatewayIpMap.getOrDefault(podIp, podIp), sp);
- }
- } else {
- bkts = buildBuckets(deviceId,
- nodeIpGatewayIpMap.getOrDefault(podIp, podIp), sp);
- }
-
- spGrpBkts.put(sp, bkts);
- });
-
- String serviceIp = service.getSpec().getClusterIP();
- spGrpBkts.forEach((sp, bkts) -> {
- String svcStr = servicePortStr(serviceIp, sp.getPort(), sp.getProtocol());
- int groupId = svcStr.hashCode();
-
- k8sGroupRuleService.setBuckets(appId, deviceId, groupId, bkts, install);
- });
-
- spEpasMap.forEach((sp, epas) ->
- // add flow rules for unshifting IP domain
- epas.forEach(epa -> {
- setUnshiftDomainRules(deviceId, POD_TABLE,
- PRIORITY_NAT_RULE, serviceIp, sp.getPort(),
- sp.getProtocol(), nodeIpGatewayIpMap.getOrDefault(epa, epa),
- sp.getTargetPort().getIntVal(), install);
+ for (String ip : epas) {
+ if (install) {
+ bkts.add(buildBuckets(node.intgBridge(),
+ nodeIpGatewayIpMap.getOrDefault(ip, ip), sp));
+ } else {
+ bkts.add(buildBuckets(node.intgBridge(),
+ nodeIpGatewayIpMap.getOrDefault(ip, ip), sp));
}
- ));
+ }
+
+ spGrpBkts.put(sp, bkts);
+ });
+
+ String serviceIp = service.getSpec().getClusterIP();
+ spGrpBkts.forEach((sp, bkts) -> {
+ String svcStr = servicePortStr(serviceIp, sp.getPort(), sp.getProtocol());
+ int groupId = svcStr.hashCode();
+
+ if (bkts.size() > 0) {
+ k8sGroupRuleService.setBuckets(appId, node.intgBridge(), groupId, bkts);
+ }
+ });
+
+ spEpasMap.forEach((sp, epas) ->
+ // add flow rules for unshifting IP domain
+ epas.forEach(epa -> {
+ setUnshiftDomainRules(node.intgBridge(), POD_TABLE,
+ PRIORITY_NAT_RULE, serviceIp, sp.getPort(),
+ sp.getProtocol(), nodeIpGatewayIpMap.getOrDefault(epa, epa),
+ sp.getTargetPort().getIntVal(), install);
+ }
+ )
+ );
+ }
}
- private List<GroupBucket> buildBuckets(DeviceId deviceId,
+ private GroupBucket buildBuckets(DeviceId deviceId,
String podIpStr,
ServicePort sp) {
- List<GroupBucket> bkts = Lists.newArrayList();
-
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
.setIpDst(IpAddress.valueOf(podIpStr));
@@ -447,9 +417,7 @@
deviceService.getDevice(deviceId), ROUTING_TABLE);
tBuilder.extension(resubmitTreatment, deviceId);
- bkts.add(buildGroupBucket(tBuilder.build(), SELECT, (short) -1));
-
- return bkts;
+ return buildGroupBucket(tBuilder.build(), SELECT, (short) -1);
}
private synchronized void setStatelessGroupFlowRules(DeviceId deviceId,
@@ -622,8 +590,6 @@
private void setStatefulGroupFlowRules(DeviceId deviceId, long ctState,
long ctMask, Service service,
boolean install) {
- int groupId = (int) groupIdCounter.incrementAndGet();
-
List<GroupBucket> buckets = Lists.newArrayList();
String serviceName = service.getMetadata().getName();
@@ -631,6 +597,10 @@
// TODO: multi-ports case should be addressed
Integer servicePort = service.getSpec().getPorts().get(0).getPort();
+ String serviceProtocol = service.getSpec().getPorts().get(0).getProtocol();
+
+ String svcStr = servicePortStr(serviceIp, servicePort, serviceProtocol);
+ int groupId = svcStr.hashCode();
List<Endpoints> endpointses = k8sEndpointsService.endpointses()
.stream()
@@ -763,29 +733,17 @@
install);
}
- private void setK8sApiRules(K8sNode k8sNode, Endpoints endpoints, boolean install) {
- if (KUBERNETES.equals(endpoints.getMetadata().getName())) {
- Service service = k8sServiceService.services().stream().filter(s ->
- KUBERNETES.equals(s.getMetadata().getName()))
- .findFirst().orElse(null);
- if (service == null) {
- return;
- }
+ private void setEndpointsRules(Endpoints endpoints, boolean install) {
+ String appName = endpoints.getMetadata().getName();
+ Service service = k8sServiceService.services().stream().filter(s ->
+ appName.equals(s.getMetadata().getName()))
+ .findFirst().orElse(null);
- if (k8sNode != null) {
- endpoints.getSubsets().forEach(s -> {
- s.getAddresses().forEach(a -> setGroupBuckets(
- k8sNode.intgBridge(), service, a.getIp(), install));
- });
- }
-
- k8sNodeService.completeNodes().forEach(n -> {
- endpoints.getSubsets().forEach(s -> {
- s.getAddresses().forEach(a -> setGroupBuckets(
- n.intgBridge(), service, a.getIp(), install));
- });
- });
+ if (service == null) {
+ return;
}
+
+ setGroupBuckets(service, install);
}
private String servicePortStr(String ip, int port, String protocol) {
@@ -820,20 +778,6 @@
}
}
- private void setServiceRuleFromPod(Pod pod, boolean install) {
- k8sServiceService.services().forEach(s -> {
- pod.getMetadata().getLabels().forEach((pk, pv) -> {
- Map<String, String> selectors = s.getSpec().getSelector();
- if (selectors != null && selectors.containsKey(pk)) {
- if (pv.equals(selectors.get(pk))) {
- k8sNodeService.completeNodes().forEach(n ->
- setGroupBucketsFromPod(n.intgBridge(), s, pod, install));
- }
- }
- });
- });
- }
-
private class InternalK8sServiceListener implements K8sServiceListener {
private boolean isRelevantHelper() {
@@ -893,44 +837,6 @@
}
}
- private class InternalK8sPodListener implements K8sPodListener {
-
- private boolean isRelevantHelper() {
- return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
- }
-
- @Override
- public void event(K8sPodEvent event) {
- switch (event.type()) {
- case K8S_POD_CREATED:
- case K8S_POD_UPDATED:
- eventExecutor.execute(() -> processPodDetection(event.subject()));
- break;
- case K8S_POD_REMOVED:
- eventExecutor.execute(() -> processPodRemoval(event.subject()));
- break;
- default:
- break;
- }
- }
-
- private void processPodDetection(Pod pod) {
- if (!isRelevantHelper()) {
- return;
- }
-
- setServiceRuleFromPod(pod, true);
- }
-
- private void processPodRemoval(Pod pod) {
- if (!isRelevantHelper()) {
- return;
- }
-
- setServiceRuleFromPod(pod, false);
- }
- }
-
private class InternalK8sEndpointsListener implements K8sEndpointsListener {
private boolean isRelevantHelper() {
@@ -943,6 +849,7 @@
switch (event.type()) {
case K8S_ENDPOINTS_CREATED:
+ case K8S_ENDPOINTS_UPDATED:
eventExecutor.execute(() -> processEndpointsCreation(endpoints));
break;
case K8S_ENDPOINTS_REMOVED:
@@ -958,7 +865,7 @@
return;
}
- setK8sApiRules(null, endpoints, true);
+ setEndpointsRules(endpoints, true);
}
private void processEndpointsRemoval(Endpoints endpoints) {
@@ -966,7 +873,7 @@
return;
}
- setK8sApiRules(null, endpoints, false);
+ setEndpointsRules(endpoints, false);
}
}
@@ -996,7 +903,7 @@
}
setServiceNatRules(node.intgBridge(), true);
- k8sEndpointsService.endpointses().forEach(e -> setK8sApiRules(node, e, true));
+ k8sEndpointsService.endpointses().forEach(e -> setEndpointsRules(e, true));
k8sNetworkService.networks().forEach(n -> setupServiceDefaultRule(n, true));
}
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java
index bcc154b..a04664f 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java
@@ -24,6 +24,9 @@
}
public static final String INTEGRATION_BRIDGE = "kbr-int";
+ public static final String EXTERNAL_BRIDGE = "kbr-ex";
+ public static final String INTEGRATION_TO_EXTERNAL_BRIDGE = "kbr-int-ex";
+ public static final String PHYSICAL_EXTERNAL_BRIDGE = "phy-kbr-ex";
public static final String VXLAN_TUNNEL = "vxlan";
public static final String GRE_TUNNEL = "gre";
public static final String GENEVE_TUNNEL = "geneve";
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
index c7c08d3..e38d0a4 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
@@ -49,6 +49,7 @@
private final String hostname;
private final Type type;
private final DeviceId intgBridge;
+ private final DeviceId extBridge;
private final IpAddress managementIp;
private final IpAddress dataIp;
private final K8sNodeState state;
@@ -63,16 +64,18 @@
* @param hostname hostname
* @param type node type
* @param intgBridge integration bridge
+ * @param extBridge external bridge
* @param managementIp management IP address
* @param dataIp data IP address
* @param state node state
*/
protected DefaultK8sNode(String hostname, Type type, DeviceId intgBridge,
- IpAddress managementIp, IpAddress dataIp,
- K8sNodeState state) {
+ DeviceId extBridge, IpAddress managementIp,
+ IpAddress dataIp, K8sNodeState state) {
this.hostname = hostname;
this.type = type;
this.intgBridge = intgBridge;
+ this.extBridge = extBridge;
this.managementIp = managementIp;
this.dataIp = dataIp;
this.state = state;
@@ -99,11 +102,30 @@
}
@Override
+ public DeviceId extBridge() {
+ return extBridge;
+ }
+
+ @Override
public K8sNode updateIntgBridge(DeviceId deviceId) {
return new Builder()
.hostname(hostname)
.type(type)
.intgBridge(deviceId)
+ .extBridge(extBridge)
+ .managementIp(managementIp)
+ .dataIp(dataIp)
+ .state(state)
+ .build();
+ }
+
+ @Override
+ public K8sNode updateExtBridge(DeviceId deviceId) {
+ return new Builder()
+ .hostname(hostname)
+ .type(type)
+ .intgBridge(intgBridge)
+ .extBridge(deviceId)
.managementIp(managementIp)
.dataIp(dataIp)
.state(state)
@@ -189,6 +211,7 @@
return hostname.equals(that.hostname) &&
type == that.type &&
intgBridge.equals(that.intgBridge) &&
+ extBridge.equals(that.extBridge) &&
managementIp.equals(that.managementIp) &&
dataIp.equals(that.dataIp) &&
state == that.state;
@@ -199,7 +222,8 @@
@Override
public int hashCode() {
- return Objects.hash(hostname, type, intgBridge, managementIp, dataIp, state);
+ return Objects.hash(hostname, type, intgBridge, extBridge,
+ managementIp, dataIp, state);
}
@Override
@@ -208,6 +232,7 @@
.add("hostname", hostname)
.add("type", type)
.add("intgBridge", intgBridge)
+ .add("extBridge", extBridge)
.add("managementIp", managementIp)
.add("dataIp", dataIp)
.add("state", state)
@@ -246,6 +271,7 @@
.hostname(node.hostname())
.type(node.type())
.intgBridge(node.intgBridge())
+ .extBridge(node.extBridge())
.managementIp(node.managementIp())
.dataIp(node.dataIp())
.state(node.state());
@@ -256,6 +282,7 @@
private String hostname;
private Type type;
private DeviceId intgBridge;
+ private DeviceId extBridge;
private IpAddress managementIp;
private IpAddress dataIp;
private K8sNodeState state;
@@ -275,6 +302,7 @@
return new DefaultK8sNode(hostname,
type,
intgBridge,
+ extBridge,
managementIp,
dataIp,
state);
@@ -299,6 +327,12 @@
}
@Override
+ public Builder extBridge(DeviceId deviceId) {
+ this.extBridge = deviceId;
+ return this;
+ }
+
+ @Override
public Builder managementIp(IpAddress managementIp) {
this.managementIp = managementIp;
return this;
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
index 2c539f3..6dfbead 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
@@ -69,6 +69,13 @@
DeviceId intgBridge();
/**
+ * Returns the device ID of the external bridge at the node.
+ *
+ * @return device id
+ */
+ DeviceId extBridge();
+
+ /**
* Returns new kubernetes node instance with given integration bridge.
*
* @param deviceId integration bridge device ID
@@ -77,6 +84,14 @@
K8sNode updateIntgBridge(DeviceId deviceId);
/**
+ * Returns new kubernetes node instance with given external bridge.
+ *
+ * @param deviceId external bridge device ID
+ * @return updated kubernetes node
+ */
+ K8sNode updateExtBridge(DeviceId deviceId);
+
+ /**
* Returns the management network IP address of the node.
*
* @return ip address
@@ -169,7 +184,7 @@
Builder type(Type type);
/**
- * Returns kubernetes node builder with supplied bridge name.
+ * Returns kubernetes node builder with supplied integration bridge name.
*
* @param deviceId integration bridge device ID
* @return kubernetes node builder
@@ -177,6 +192,14 @@
Builder intgBridge(DeviceId deviceId);
/**
+ * Returns kubernetes node builder with supplied external bridge name.
+ *
+ * @param deviceId external bridge deviceID
+ * @return kubernetes node builder
+ */
+ Builder extBridge(DeviceId deviceId);
+
+ /**
* Returns kubernetes node builder with supplied management IP address.
*
* @param managementIp management IP address
diff --git a/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sNodeTest.java b/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sNodeTest.java
index d30c798..d93c8dc 100644
--- a/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sNodeTest.java
+++ b/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sNodeTest.java
@@ -55,18 +55,21 @@
HOSTNAME_1,
MINION,
DEVICE_1,
+ DEVICE_1,
TEST_IP,
INIT);
private static final K8sNode K8S_NODE_2 = createNode(
HOSTNAME_1,
MINION,
DEVICE_1,
+ DEVICE_1,
TEST_IP,
INIT);
private static final K8sNode K8S_NODE_3 = createNode(
HOSTNAME_2,
MINION,
DEVICE_2,
+ DEVICE_2,
TEST_IP,
INIT);
@@ -81,6 +84,7 @@
.managementIp(MANAGEMENT_IP)
.dataIp(DATA_IP)
.intgBridge(DEVICE_1.id())
+ .extBridge(DEVICE_1.id())
.state(COMPLETE)
.build();
}
@@ -103,6 +107,7 @@
checkCommonProperties(refNode);
assertSame(refNode.state(), COMPLETE);
assertEquals(refNode.intgBridge(), DEVICE_1.id());
+ assertEquals(refNode.extBridge(), DEVICE_1.id());
}
/**
@@ -134,6 +139,7 @@
DefaultK8sNode.builder()
.type(MINION)
.intgBridge(DEVICE_1.id())
+ .extBridge(DEVICE_1.id())
.managementIp(TEST_IP)
.dataIp(TEST_IP)
.state(INIT)
@@ -148,6 +154,7 @@
DefaultK8sNode.builder()
.hostname(HOSTNAME_1)
.intgBridge(DEVICE_1.id())
+ .extBridge(DEVICE_1.id())
.managementIp(TEST_IP)
.dataIp(TEST_IP)
.state(INIT)
@@ -164,6 +171,7 @@
.hostname(HOSTNAME_1)
.type(MINION)
.intgBridge(DEVICE_1.id())
+ .extBridge(DEVICE_1.id())
.dataIp(TEST_IP)
.state(INIT)
.build();
@@ -188,12 +196,13 @@
}
private static K8sNode createNode(String hostname, Type type,
- Device intgBridge, IpAddress ipAddr,
- K8sNodeState state) {
+ Device intgBridge, Device extBridge,
+ IpAddress ipAddr, K8sNodeState state) {
return DefaultK8sNode.builder()
.hostname(hostname)
.type(type)
.intgBridge(intgBridge.id())
+ .extBridge(extBridge.id())
.managementIp(ipAddr)
.dataIp(ipAddr)
.state(state)
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java
index e07ad6c..6676847 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java
@@ -42,6 +42,7 @@
private static final String MANAGEMENT_IP = "managementIp";
private static final String DATA_IP = "dataIp";
private static final String INTEGRATION_BRIDGE = "integrationBridge";
+ private static final String EXTERNAL_BRIDGE = "externalBridge";
private static final String STATE = "state";
private static final String MISSING_MESSAGE = " is required in K8sNode";
@@ -60,6 +61,10 @@
result.put(INTEGRATION_BRIDGE, node.intgBridge().toString());
}
+ if (node.extBridge() != null) {
+ result.put(EXTERNAL_BRIDGE, node.extBridge().toString());
+ }
+
if (node.dataIp() != null) {
result.put(DATA_IP, node.dataIp().toString());
}
@@ -95,6 +100,11 @@
nodeBuilder.intgBridge(DeviceId.deviceId(intBridgeJson.asText()));
}
+ JsonNode extBridgeJson = json.get(EXTERNAL_BRIDGE);
+ if (extBridgeJson != null) {
+ nodeBuilder.extBridge(DeviceId.deviceId(extBridgeJson.asText()));
+ }
+
log.trace("node is {}", nodeBuilder.build().toString());
return nodeBuilder.build();
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
index 6fb20b7..83e537c 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
@@ -36,8 +36,10 @@
import org.onosproject.net.behaviour.BridgeDescription;
import org.onosproject.net.behaviour.ControllerInfo;
import org.onosproject.net.behaviour.DefaultBridgeDescription;
+import org.onosproject.net.behaviour.DefaultPatchDescription;
import org.onosproject.net.behaviour.DefaultTunnelDescription;
import org.onosproject.net.behaviour.InterfaceConfig;
+import org.onosproject.net.behaviour.PatchDescription;
import org.onosproject.net.behaviour.TunnelDescription;
import org.onosproject.net.behaviour.TunnelEndPoints;
import org.onosproject.net.behaviour.TunnelKeys;
@@ -65,11 +67,14 @@
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.packet.TpPort.tpPort;
import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snode.api.Constants.EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.GENEVE;
import static org.onosproject.k8snode.api.Constants.GENEVE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.GRE;
import static org.onosproject.k8snode.api.Constants.GRE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_EXTERNAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.PHYSICAL_EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.VXLAN;
import static org.onosproject.k8snode.api.Constants.VXLAN_TUNNEL;
import static org.onosproject.k8snode.api.K8sNodeService.APP_ID;
@@ -188,6 +193,9 @@
if (!deviceService.isAvailable(k8sNode.intgBridge())) {
createBridge(k8sNode, INTEGRATION_BRIDGE, k8sNode.intgBridge());
}
+ if (!deviceService.isAvailable(k8sNode.extBridge())) {
+ createBridge(k8sNode, EXTERNAL_BRIDGE, k8sNode.extBridge());
+ }
}
@Override
@@ -198,6 +206,9 @@
return;
}
+ // create patch ports between integration and external bridges
+ createPatchInterfaces(k8sNode);
+
if (k8sNode.dataIp() != null &&
!isIntfEnabled(k8sNode, VXLAN_TUNNEL)) {
createVxlanTunnelInterface(k8sNode);
@@ -310,6 +321,32 @@
createTunnelInterface(k8sNode, GENEVE, GENEVE_TUNNEL);
}
+ private void createPatchInterfaces(K8sNode k8sNode) {
+ Device device = deviceService.getDevice(k8sNode.ovsdb());
+ if (device == null || !device.is(InterfaceConfig.class)) {
+ log.error("Failed to create patch interface on {}", k8sNode.ovsdb());
+ return;
+ }
+
+ PatchDescription brIntPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(INTEGRATION_BRIDGE)
+ .ifaceName(INTEGRATION_TO_EXTERNAL_BRIDGE)
+ .peer(PHYSICAL_EXTERNAL_BRIDGE)
+ .build();
+
+ PatchDescription brExtPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(EXTERNAL_BRIDGE)
+ .ifaceName(PHYSICAL_EXTERNAL_BRIDGE)
+ .peer(INTEGRATION_TO_EXTERNAL_BRIDGE)
+ .build();
+
+ InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
+ ifaceConfig.addPatchMode(INTEGRATION_TO_EXTERNAL_BRIDGE, brIntPatchDesc);
+ ifaceConfig.addPatchMode(PHYSICAL_EXTERNAL_BRIDGE, brExtPatchDesc);
+ }
+
/**
* Creates a tunnel interface in a given kubernetes node.
*
@@ -397,7 +434,9 @@
return false;
}
- return deviceService.isAvailable(k8sNode.intgBridge());
+ return k8sNode.intgBridge() != null && k8sNode.extBridge() != null &&
+ deviceService.isAvailable(k8sNode.intgBridge()) &&
+ deviceService.isAvailable(k8sNode.extBridge());
case DEVICE_CREATED:
if (k8sNode.dataIp() != null &&
!isIntfEnabled(k8sNode, VXLAN_TUNNEL)) {
@@ -463,6 +502,9 @@
// delete integration bridge from the node
client.dropBridge(INTEGRATION_BRIDGE);
+ // delete external bridge from the node
+ client.dropBridge(EXTERNAL_BRIDGE);
+
// disconnect ovsdb
client.disconnect();
}
@@ -554,8 +596,10 @@
return;
}
+ // TODO: also need to check the external bridge's availability
if (deviceService.isAvailable(device.id())) {
- log.debug("Integration bridge created on {}", k8sNode.hostname());
+ log.debug("Integration bridge created on {}",
+ k8sNode.hostname());
bootstrapNode(k8sNode);
} else if (k8sNode.state() == COMPLETE) {
log.info("Device {} disconnected", device.id());
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
index 0289422..122a27b 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
@@ -156,47 +156,73 @@
public void createNode(K8sNode node) {
checkNotNull(node, ERR_NULL_NODE);
- K8sNode updatedNode;
+ K8sNode intNode;
+ K8sNode extNode;
if (node.intgBridge() == null) {
String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
- updatedNode = node.updateIntgBridge(DeviceId.deviceId(deviceIdStr));
- checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
- NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ intNode = node.updateIntgBridge(DeviceId.deviceId(deviceIdStr));
+ checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
+ NOT_DUPLICATED_MSG, intNode.intgBridge());
} else {
- updatedNode = node;
- checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
- NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ intNode = node;
+ checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
+ NOT_DUPLICATED_MSG, intNode.intgBridge());
}
- nodeStore.createNode(updatedNode);
- log.info(String.format(MSG_NODE, updatedNode.hostname(), MSG_CREATED));
+ if (intNode.extBridge() == null) {
+ String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
+ checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
+ extNode = intNode.updateExtBridge(DeviceId.deviceId(deviceIdStr));
+ checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
+ NOT_DUPLICATED_MSG, extNode.extBridge());
+ } else {
+ extNode = intNode;
+ checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
+ NOT_DUPLICATED_MSG, extNode.extBridge());
+ }
+
+ nodeStore.createNode(extNode);
+ log.info(String.format(MSG_NODE, extNode.hostname(), MSG_CREATED));
}
@Override
public void updateNode(K8sNode node) {
checkNotNull(node, ERR_NULL_NODE);
- K8sNode updatedNode;
+ K8sNode intNode;
+ K8sNode extNode;
K8sNode existingNode = nodeStore.node(node.hostname());
checkNotNull(existingNode, ERR_NULL_NODE);
- DeviceId existDeviceId = nodeStore.node(node.hostname()).intgBridge();
+ DeviceId existIntgBridge = nodeStore.node(node.hostname()).intgBridge();
if (node.intgBridge() == null) {
- updatedNode = node.updateIntgBridge(existDeviceId);
- checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
- NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ intNode = node.updateIntgBridge(existIntgBridge);
+ checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
+ NOT_DUPLICATED_MSG, intNode.intgBridge());
} else {
- updatedNode = node;
- checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
- NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ intNode = node;
+ checkArgument(!hasIntgBridge(intNode.intgBridge(), intNode.hostname()),
+ NOT_DUPLICATED_MSG, intNode.intgBridge());
}
- nodeStore.updateNode(updatedNode);
- log.info(String.format(MSG_NODE, updatedNode.hostname(), MSG_UPDATED));
+ DeviceId existExtBridge = nodeStore.node(node.hostname()).extBridge();
+
+ if (intNode.extBridge() == null) {
+ extNode = intNode.updateExtBridge(existExtBridge);
+ checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
+ NOT_DUPLICATED_MSG, extNode.extBridge());
+ } else {
+ extNode = intNode;
+ checkArgument(!hasExtBridge(extNode.extBridge(), extNode.hostname()),
+ NOT_DUPLICATED_MSG, extNode.extBridge());
+ }
+
+ nodeStore.updateNode(extNode);
+ log.info(String.format(MSG_NODE, extNode.hostname(), MSG_UPDATED));
}
@Override
@@ -242,6 +268,7 @@
return nodeStore.node(hostname);
}
+ // TODO: need to differentiate integration bridge and external bridge
@Override
public K8sNode node(DeviceId deviceId) {
return nodeStore.nodes().stream()
@@ -253,7 +280,16 @@
private boolean hasIntgBridge(DeviceId deviceId, String hostname) {
Optional<K8sNode> existNode = nodeStore.nodes().stream()
.filter(n -> !n.hostname().equals(hostname))
- .filter(n -> n.intgBridge().equals(deviceId))
+ .filter(n -> deviceId.equals(n.intgBridge()))
+ .findFirst();
+
+ return existNode.isPresent();
+ }
+
+ private boolean hasExtBridge(DeviceId deviceId, String hostname) {
+ Optional<K8sNode> existNode = nodeStore.nodes().stream()
+ .filter(n -> !n.hostname().equals(hostname))
+ .filter(n -> deviceId.equals(n.extBridge()))
.findFirst();
return existNode.isPresent();
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
index a07d60e..fbccdd0 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
@@ -76,10 +76,15 @@
private static final Device MINION_2_INTG_DEVICE = createDevice(2);
private static final Device MINION_3_INTG_DEVICE = createDevice(3);
+ private static final Device MINION_1_EXT_DEVICE = createDevice(4);
+ private static final Device MINION_2_EXT_DEVICE = createDevice(5);
+ private static final Device MINION_3_EXT_DEVICE = createDevice(6);
+
private static final K8sNode MINION_1 = createNode(
MINION_1_HOSTNAME,
MINION,
MINION_1_INTG_DEVICE,
+ MINION_1_EXT_DEVICE,
IpAddress.valueOf("10.100.0.1"),
INIT
);
@@ -87,6 +92,7 @@
MINION_2_HOSTNAME,
MINION,
MINION_2_INTG_DEVICE,
+ MINION_2_EXT_DEVICE,
IpAddress.valueOf("10.100.0.2"),
INIT
);
@@ -94,6 +100,7 @@
MINION_3_HOSTNAME,
MINION,
MINION_3_INTG_DEVICE,
+ MINION_3_EXT_DEVICE,
IpAddress.valueOf("10.100.0.3"),
COMPLETE
);
@@ -320,12 +327,13 @@
}
private static K8sNode createNode(String hostname, K8sNode.Type type,
- Device intgBridge, IpAddress ipAddr,
- K8sNodeState state) {
+ Device intgBridge, Device extBridge,
+ IpAddress ipAddr, K8sNodeState state) {
return DefaultK8sNode.builder()
.hostname(hostname)
.type(type)
.intgBridge(intgBridge.id())
+ .extBridge(extBridge.id())
.managementIp(ipAddr)
.dataIp(ipAddr)
.state(state)
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java
index 90f7d0d..207bd0e 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java
@@ -94,6 +94,7 @@
.dataIp(IpAddress.valueOf("10.134.34.222"))
.managementIp(IpAddress.valueOf("10.134.231.30"))
.intgBridge(DeviceId.deviceId("of:00000000000000a1"))
+ .extBridge(DeviceId.deviceId("of:00000000000000b1"))
.state(K8sNodeState.INIT)
.build();
diff --git a/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/codec/K8sMinionNode.json b/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/codec/K8sMinionNode.json
index a6760f5..9ed649e 100644
--- a/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/codec/K8sMinionNode.json
+++ b/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/codec/K8sMinionNode.json
@@ -3,5 +3,6 @@
"type": "MINION",
"managementIp": "172.16.130.4",
"dataIp": "172.16.130.4",
- "integrationBridge": "of:00000000000000a1"
+ "integrationBridge": "of:00000000000000a1",
+ "externalBridge": "of:00000000000000b1"
}
\ No newline at end of file
diff --git a/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/web/k8s-node-minion-config.json b/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/web/k8s-node-minion-config.json
index 725766e..8c65bb5 100644
--- a/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/web/k8s-node-minion-config.json
+++ b/apps/k8s-node/app/src/test/resources/org/onosproject/k8snode/web/k8s-node-minion-config.json
@@ -5,7 +5,8 @@
"type" : "MINION",
"managementIp" : "10.134.231.32",
"dataIp" : "10.134.34.224",
- "integrationBridge" : "of:00000000000000a2"
+ "integrationBridge" : "of:00000000000000a2",
+ "externalBridge" : "of:00000000000000b2"
}
]
}
\ No newline at end of file