Fix: allow to invoke service IP from local k8s node
Change-Id: I8c39703f35319bb51bbd175e7ff717d5e6ec5250
diff --git a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
index 2630c30..bca34f2 100644
--- a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
+++ b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
@@ -45,6 +45,7 @@
public static final String SHIFTED_IP_CIDR = "172.10.0.0/16";
public static final String SHIFTED_IP_PREFIX = "172.10";
+ public static final String SHIFTED_LOCAL_IP_PREFIX = "172.11";
public static final String NODE_IP_PREFIX = "182";
public static final String SRC = "src";
@@ -64,6 +65,8 @@
public static final int DEFAULT_SEGMENT_ID = 100;
+ public static final int HOST_PREFIX = 32;
+
// flow priority
public static final int PRIORITY_SNAT_RULE = 26000;
public static final int PRIORITY_TUNNEL_TAG_RULE = 30000;
@@ -73,15 +76,13 @@
public static final int PRIORITY_CT_RULE = 32000;
public static final int PRIORITY_CT_DROP_RULE = 32500;
public static final int PRIORITY_NAT_RULE = 30000;
- public static final int PRIORITY_SERVICE_CIDR_RULE = 32000;
- public static final int PRIORITY_GATEWAY_RULE = 30000;
+ public static final int PRIORITY_GATEWAY_RULE = 31000;
+ public static final int PRIORITY_LOCAL_BRIDGE_RULE = 32000;
public static final int PRIORITY_SWITCHING_RULE = 30000;
public static final int PRIORITY_CIDR_RULE = 30000;
public static final int PRIORITY_NAMESPACE_RULE = 31000;
public static final int PRIORITY_STATEFUL_SNAT_RULE = 41000;
public static final int PRIORITY_EXTERNAL_ROUTING_RULE = 25000;
- public static final int PRIORITY_ARP_GATEWAY_RULE = 41000;
- public static final int PRIORITY_ARP_SUBNET_RULE = 40000;
public static final int PRIORITY_ARP_CONTROL_RULE = 40000;
public static final int PRIORITY_ARP_REPLY_RULE = 40000;
public static final int PRIORITY_ARP_POD_RULE = 39000;
@@ -118,4 +119,6 @@
public static final int EXT_ENTRY_TABLE = 0;
public static final int POD_RESOLUTION_TABLE = 11;
+
+ public static final int LOCAL_ENTRY_TABLE = 0;
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
index 25d8a93..73ce2ef 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
@@ -61,6 +61,7 @@
import static org.onosproject.k8snetworking.api.Constants.DEFAULT_GATEWAY_MAC;
import static org.onosproject.k8snetworking.api.Constants.FORWARDING_TABLE;
import static org.onosproject.k8snetworking.api.Constants.GROUPING_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.HOST_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.JUMP_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.NAMESPACE_TABLE;
@@ -343,7 +344,7 @@
private void setupHostRoutingRule(K8sNetwork k8sNetwork) {
setAnyRoutingRule(IpPrefix.valueOf(
- k8sNetwork.gatewayIp(), 32), null, k8sNetwork);
+ k8sNetwork.gatewayIp(), HOST_PREFIX), null, k8sNetwork);
}
private class InternalK8sNodeListener implements K8sNodeListener {
@@ -391,8 +392,8 @@
deviceEventExecutor.execute(() -> processNetworkCreation(event.subject()));
break;
case K8S_NETWORK_REMOVED:
- break;
default:
+ // do nothing
break;
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 5ddc872..d7d7638 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -15,7 +15,9 @@
*/
package org.onosproject.k8snetworking.impl;
+import org.onlab.packet.ARP;
import org.onlab.packet.Ethernet;
+import org.onlab.packet.Ip4Address;
import org.onlab.packet.IpPrefix;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
@@ -31,6 +33,7 @@
import org.onosproject.k8snode.api.K8sNodeEvent;
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
+import org.onosproject.net.Device;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.driver.DriverService;
@@ -38,6 +41,7 @@
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.onosproject.net.packet.PacketService;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
@@ -51,11 +55,25 @@
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.B_CLASS;
+import static org.onosproject.k8snetworking.api.Constants.DST;
+import static org.onosproject.k8snetworking.api.Constants.HOST_PREFIX;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.api.Constants.LOCAL_ENTRY_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_ARP_REPLY_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_GATEWAY_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_LOCAL_BRIDGE_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.SHIFTED_IP_PREFIX;
+import static org.onosproject.k8snetworking.api.Constants.SHIFTED_LOCAL_IP_PREFIX;
+import static org.onosproject.k8snetworking.api.Constants.SRC;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.shiftIpDomain;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.tunnelPortNumByNetId;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildLoadExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpSpaToTpaExtension;
+import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -67,7 +85,8 @@
private final Logger log = getLogger(getClass());
- private static final int GW_IP_PREFIX = 32;
+ private static final String REQUEST = "req";
+ private static final String REPLY = "rep";
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected CoreService coreService;
@@ -128,11 +147,12 @@
}
private void setGatewayRule(K8sNetwork k8sNetwork, boolean install) {
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(), GW_IP_PREFIX));
-
for (K8sNode node : k8sNodeService.completeNodes()) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
if (node.hostname().equals(k8sNetwork.name())) {
@@ -159,9 +179,120 @@
PRIORITY_GATEWAY_RULE,
ROUTING_TABLE,
install);
+
+ if (node.hostname().equals(k8sNetwork.name())) {
+ sBuilder = DefaultTrafficSelector.builder()
+ .matchInPort(PortNumber.LOCAL)
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+
+ tBuilder = DefaultTrafficTreatment.builder()
+ .setOutput(node.intgToLocalPatchPortNum());
+
+ k8sFlowRuleService.setRule(
+ appId,
+ node.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_LOCAL_BRIDGE_RULE,
+ ROUTING_TABLE,
+ install);
+ }
}
}
+ private void setLocalBridgeRules(K8sNetwork k8sNetwork, boolean install) {
+ for (K8sNode node : k8sNodeService.completeNodes()) {
+ if (node.hostname().equals(k8sNetwork.name())) {
+ setLocalBridgeRule(k8sNetwork, node, REQUEST, install);
+ setLocalBridgeRule(k8sNetwork, node, REPLY, install);
+ }
+ }
+ }
+
+ private void setLocalBridgeRule(K8sNetwork k8sNetwork, K8sNode k8sNode,
+ String type, boolean install) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4);
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ ExtensionTreatment loadTreatment = null;
+
+ if (REQUEST.equals(type)) {
+ loadTreatment = buildLoadExtension(deviceService.getDevice(
+ k8sNode.localBridge()), B_CLASS, SRC, SHIFTED_LOCAL_IP_PREFIX);
+ }
+
+ if (REPLY.equals(type)) {
+ loadTreatment = buildLoadExtension(deviceService.getDevice(
+ k8sNode.localBridge()), B_CLASS, DST, SHIFTED_IP_PREFIX);
+ }
+
+ tBuilder.extension(loadTreatment, k8sNode.localBridge());
+
+ if (REQUEST.equals(type)) {
+ sBuilder.matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+ tBuilder.setOutput(PortNumber.LOCAL);
+ }
+
+ if (REPLY.equals(type)) {
+ sBuilder.matchIPSrc(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+ tBuilder.setOutput(k8sNode.localToIntgPatchPortNumber());
+ }
+
+ k8sFlowRuleService.setRule(
+ appId,
+ k8sNode.localBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_LOCAL_BRIDGE_RULE,
+ LOCAL_ENTRY_TABLE,
+ install);
+ }
+
+ private void setLocalBridgeArpRules(K8sNetwork k8sNetwork, boolean install) {
+ for (K8sNode node : k8sNodeService.completeNodes()) {
+ if (node.hostname().equals(k8sNetwork.name())) {
+ setLocalBridgeArpRule(k8sNetwork, node, install);
+ }
+ }
+ }
+
+ private void setLocalBridgeArpRule(K8sNetwork k8sNetwork, K8sNode k8sNode, boolean install) {
+ Device device = deviceService.getDevice(k8sNode.localBridge());
+
+ String shiftedLocalIp = shiftIpDomain(
+ k8sNetwork.gatewayIp().toString(), SHIFTED_LOCAL_IP_PREFIX);
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_ARP)
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(Ip4Address.valueOf(shiftedLocalIp))
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setArpOp(ARP.OP_REPLY)
+ .extension(buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(buildMoveArpShaToThaExtension(device), device.id())
+ .extension(buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpSpa(Ip4Address.valueOf(shiftedLocalIp))
+ .setArpSha(k8sNode.intgBridgeMac())
+ .setOutput(PortNumber.IN_PORT)
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ device.id(),
+ selector,
+ treatment,
+ PRIORITY_ARP_REPLY_RULE,
+ LOCAL_ENTRY_TABLE,
+ install);
+ }
+
private class InternalK8sNetworkListener implements K8sNetworkListener {
private boolean isRelevantHelper() {
@@ -189,6 +320,8 @@
}
setGatewayRule(event.subject(), true);
+ setLocalBridgeRules(event.subject(), true);
+ setLocalBridgeArpRules(event.subject(), true);
}
private void processNetworkRemoval(K8sNetworkEvent event) {
@@ -197,6 +330,8 @@
}
setGatewayRule(event.subject(), false);
+ setLocalBridgeRules(event.subject(), false);
+ setLocalBridgeArpRules(event.subject(), false);
}
}
@@ -225,6 +360,8 @@
}
k8sNetworkService.networks().forEach(n -> setGatewayRule(n, true));
+ k8sNetworkService.networks().forEach(n -> setLocalBridgeRules(n, true));
+ k8sNetworkService.networks().forEach(n -> setLocalBridgeArpRules(n, true));
}
}
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java
index a04664f..513d21a 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/Constants.java
@@ -25,8 +25,11 @@
public static final String INTEGRATION_BRIDGE = "kbr-int";
public static final String EXTERNAL_BRIDGE = "kbr-ex";
+ public static final String LOCAL_BRIDGE = "kbr-local";
public static final String INTEGRATION_TO_EXTERNAL_BRIDGE = "kbr-int-ex";
public static final String PHYSICAL_EXTERNAL_BRIDGE = "phy-kbr-ex";
+ public static final String INTEGRATION_TO_LOCAL_BRIDGE = "kbr-int-local";
+ public static final String LOCAL_TO_INTEGRATION_BRIDGE = "kbr-local-int";
public static final String VXLAN_TUNNEL = "vxlan";
public static final String GRE_TUNNEL = "gre";
public static final String GENEVE_TUNNEL = "geneve";
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
index 7910500..6a18194 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
@@ -33,6 +33,8 @@
import static org.onosproject.k8snode.api.Constants.GRE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_BRIDGE;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_EXTERNAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_LOCAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.LOCAL_TO_INTEGRATION_BRIDGE;
import static org.onosproject.k8snode.api.Constants.PHYSICAL_EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.VXLAN_TUNNEL;
import static org.onosproject.net.AnnotationKeys.PORT_NAME;
@@ -48,6 +50,7 @@
private final Type type;
private final DeviceId intgBridge;
private final DeviceId extBridge;
+ private final DeviceId localBridge;
private final IpAddress managementIp;
private final IpAddress dataIp;
private final K8sNodeState state;
@@ -68,6 +71,7 @@
* @param type node type
* @param intgBridge integration bridge
* @param extBridge external bridge
+ * @param localBridge local bridge
* @param extIntf external interface
* @param managementIp management IP address
* @param dataIp data IP address
@@ -78,15 +82,16 @@
* @param podCidr POD CIDR
*/
protected DefaultK8sNode(String hostname, Type type, DeviceId intgBridge,
- DeviceId extBridge, String extIntf,
- IpAddress managementIp, IpAddress dataIp,
- K8sNodeState state, IpAddress extBridgeIp,
- IpAddress extGatewayIp, MacAddress extGatewayMac,
- String podCidr) {
+ DeviceId extBridge, DeviceId localBridge,
+ String extIntf, IpAddress managementIp,
+ IpAddress dataIp, K8sNodeState state,
+ IpAddress extBridgeIp, IpAddress extGatewayIp,
+ MacAddress extGatewayMac, String podCidr) {
this.hostname = hostname;
this.type = type;
this.intgBridge = intgBridge;
this.extBridge = extBridge;
+ this.localBridge = localBridge;
this.extIntf = extIntf;
this.managementIp = managementIp;
this.dataIp = dataIp;
@@ -123,6 +128,11 @@
}
@Override
+ public DeviceId localBridge() {
+ return localBridge;
+ }
+
+ @Override
public String extIntf() {
return extIntf;
}
@@ -134,6 +144,7 @@
.type(type)
.intgBridge(deviceId)
.extBridge(extBridge)
+ .localBridge(localBridge)
.extIntf(extIntf)
.managementIp(managementIp)
.dataIp(dataIp)
@@ -152,6 +163,26 @@
.type(type)
.intgBridge(intgBridge)
.extBridge(deviceId)
+ .localBridge(localBridge)
+ .extIntf(extIntf)
+ .managementIp(managementIp)
+ .dataIp(dataIp)
+ .state(state)
+ .extBridgeIp(extBridgeIp)
+ .extGatewayIp(extGatewayIp)
+ .extGatewayMac(extGatewayMac)
+ .podCidr(podCidr)
+ .build();
+ }
+
+ @Override
+ public K8sNode updateLocalBridge(DeviceId deviceId) {
+ return new Builder()
+ .hostname(hostname)
+ .type(type)
+ .intgBridge(intgBridge)
+ .extBridge(extBridge)
+ .localBridge(deviceId)
.extIntf(extIntf)
.managementIp(managementIp)
.dataIp(dataIp)
@@ -190,6 +221,7 @@
.type(type)
.intgBridge(intgBridge)
.extBridge(extBridge)
+ .localBridge(localBridge)
.extIntf(extIntf)
.managementIp(managementIp)
.dataIp(dataIp)
@@ -208,6 +240,7 @@
.type(type)
.intgBridge(intgBridge)
.extBridge(extBridge)
+ .localBridge(localBridge)
.extIntf(extIntf)
.managementIp(managementIp)
.dataIp(dataIp)
@@ -245,6 +278,16 @@
}
@Override
+ public PortNumber intgToLocalPatchPortNum() {
+ return portNumber(intgBridge, INTEGRATION_TO_LOCAL_BRIDGE);
+ }
+
+ @Override
+ public PortNumber localToIntgPatchPortNumber() {
+ return portNumber(localBridge, LOCAL_TO_INTEGRATION_BRIDGE);
+ }
+
+ @Override
public PortNumber extToIntgPatchPortNum() {
return portNumber(extBridge, PHYSICAL_EXTERNAL_BRIDGE);
}
@@ -296,6 +339,7 @@
type == that.type &&
intgBridge.equals(that.intgBridge) &&
extBridge.equals(that.extBridge) &&
+ localBridge.equals(that.localBridge) &&
extIntf.equals(that.extIntf) &&
managementIp.equals(that.managementIp) &&
dataIp.equals(that.dataIp) &&
@@ -310,8 +354,8 @@
@Override
public int hashCode() {
- return Objects.hash(hostname, type, intgBridge, extBridge, extIntf,
- managementIp, dataIp, state, extBridgeIp,
+ return Objects.hash(hostname, type, intgBridge, extBridge, localBridge,
+ extIntf, managementIp, dataIp, state, extBridgeIp,
extGatewayIp, extGatewayMac, podCidr);
}
@@ -322,6 +366,7 @@
.add("type", type)
.add("intgBridge", intgBridge)
.add("extBridge", extBridge)
+ .add("localBridge", localBridge)
.add("extIntf", extIntf)
.add("managementIp", managementIp)
.add("dataIp", dataIp)
@@ -381,6 +426,7 @@
.type(node.type())
.intgBridge(node.intgBridge())
.extBridge(node.extBridge())
+ .localBridge(node.localBridge())
.extIntf(node.extIntf())
.managementIp(node.managementIp())
.dataIp(node.dataIp())
@@ -397,6 +443,7 @@
private Type type;
private DeviceId intgBridge;
private DeviceId extBridge;
+ private DeviceId localBridge;
private IpAddress managementIp;
private IpAddress dataIp;
private K8sNodeState state;
@@ -422,6 +469,7 @@
type,
intgBridge,
extBridge,
+ localBridge,
extIntf,
managementIp,
dataIp,
@@ -457,6 +505,12 @@
}
@Override
+ public Builder localBridge(DeviceId deviceId) {
+ this.localBridge = deviceId;
+ return this;
+ }
+
+ @Override
public Builder extIntf(String intf) {
this.extIntf = intf;
return this;
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
index 18e943a..055b0df 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
@@ -76,6 +76,13 @@
DeviceId extBridge();
/**
+ * Returns the device ID of the local bridge at the node.
+ *
+ * @return device id
+ */
+ DeviceId localBridge();
+
+ /**
* Returns the external interface name.
*
* @return external interface name
@@ -99,6 +106,14 @@
K8sNode updateExtBridge(DeviceId deviceId);
/**
+ * Returns new kubernetes node instance with given local bridge.
+ *
+ * @param deviceId local bridge device ID
+ * @return updated kubernetes node
+ */
+ K8sNode updateLocalBridge(DeviceId deviceId);
+
+ /**
* Returns the management network IP address of the node.
*
* @return ip address
@@ -178,6 +193,20 @@
PortNumber intgToExtPatchPortNum();
/**
+ * Returns the integration to local patch port number.
+ *
+ * @return patch port number
+ */
+ PortNumber intgToLocalPatchPortNum();
+
+ /**
+ * Returns the local to integration patch port number.
+ *
+ * @return patch port number
+ */
+ PortNumber localToIntgPatchPortNumber();
+
+ /**
* Returns the external to integration patch port number.
*
* @return patch port number
@@ -265,12 +294,20 @@
/**
* Returns kubernetes node builder with supplied external bridge name.
*
- * @param deviceId external bridge deviceID
+ * @param deviceId external bridge device ID
* @return kubernetes node builder
*/
Builder extBridge(DeviceId deviceId);
/**
+ * Returns kubernetes node builder with supplied local bridge name.
+ *
+ * @param deviceId local bridge device ID
+ * @return kubernetes node builder
+ */
+ Builder localBridge(DeviceId deviceId);
+
+ /**
* Returns kubernetes node builder with supplied external interface.
*
* @param intf external interface
diff --git a/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sNodeTest.java b/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sNodeTest.java
index 166b528..463d89d 100644
--- a/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sNodeTest.java
+++ b/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sNodeTest.java
@@ -68,6 +68,7 @@
MINION,
DEVICE_1,
DEVICE_1,
+ DEVICE_1,
BRIDGE_INTF_1,
TEST_IP,
INIT,
@@ -79,6 +80,7 @@
MINION,
DEVICE_1,
DEVICE_1,
+ DEVICE_1,
BRIDGE_INTF_1,
TEST_IP,
INIT,
@@ -90,6 +92,7 @@
MINION,
DEVICE_2,
DEVICE_2,
+ DEVICE_2,
BRIDGE_INTF_2,
TEST_IP,
INIT,
@@ -109,6 +112,7 @@
.dataIp(DATA_IP)
.intgBridge(DEVICE_1.id())
.extBridge(DEVICE_1.id())
+ .localBridge(DEVICE_1.id())
.extIntf(BRIDGE_INTF_1)
.state(COMPLETE)
.extBridgeIp(EXT_BRIDGE_IP_1)
@@ -136,6 +140,7 @@
assertSame(refNode.state(), COMPLETE);
assertEquals(refNode.intgBridge(), DEVICE_1.id());
assertEquals(refNode.extBridge(), DEVICE_1.id());
+ assertEquals(refNode.localBridge(), DEVICE_1.id());
}
/**
@@ -168,6 +173,7 @@
.type(MINION)
.intgBridge(DEVICE_1.id())
.extBridge(DEVICE_1.id())
+ .localBridge(DEVICE_1.id())
.extIntf(BRIDGE_INTF_1)
.managementIp(TEST_IP)
.dataIp(TEST_IP)
@@ -187,6 +193,7 @@
.hostname(HOSTNAME_1)
.intgBridge(DEVICE_1.id())
.extBridge(DEVICE_1.id())
+ .localBridge(DEVICE_1.id())
.extIntf(BRIDGE_INTF_1)
.managementIp(TEST_IP)
.dataIp(TEST_IP)
@@ -208,6 +215,7 @@
.type(MINION)
.intgBridge(DEVICE_1.id())
.extBridge(DEVICE_1.id())
+ .localBridge(DEVICE_1.id())
.extIntf(BRIDGE_INTF_1)
.dataIp(TEST_IP)
.state(INIT)
@@ -237,14 +245,16 @@
private static K8sNode createNode(String hostname, Type type,
Device intgBridge, Device extBridge,
- String bridgeIntf, IpAddress ipAddr,
- K8sNodeState state, IpAddress extBridgeIp,
- IpAddress extGatewayIp, String podCidr) {
+ Device localBridge, String bridgeIntf,
+ IpAddress ipAddr, K8sNodeState state,
+ IpAddress extBridgeIp, IpAddress extGatewayIp,
+ String podCidr) {
return DefaultK8sNode.builder()
.hostname(hostname)
.type(type)
.intgBridge(intgBridge.id())
.extBridge(extBridge.id())
+ .localBridge(localBridge.id())
.extIntf(bridgeIntf)
.managementIp(ipAddr)
.dataIp(ipAddr)
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sNodeCheckCommand.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sNodeCheckCommand.java
index 69c64b9..f421de0 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sNodeCheckCommand.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sNodeCheckCommand.java
@@ -32,6 +32,9 @@
import static org.onosproject.k8snode.api.Constants.GRE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_BRIDGE;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_EXTERNAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_LOCAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.LOCAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.LOCAL_TO_INTEGRATION_BRIDGE;
import static org.onosproject.k8snode.api.Constants.PHYSICAL_EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.VXLAN_TUNNEL;
import static org.onosproject.net.AnnotationKeys.PORT_NAME;
@@ -74,6 +77,7 @@
intgBridge.annotations());
printPortState(deviceService, node.intgBridge(), INTEGRATION_BRIDGE);
printPortState(deviceService, node.intgBridge(), INTEGRATION_TO_EXTERNAL_BRIDGE);
+ printPortState(deviceService, node.intgBridge(), INTEGRATION_TO_LOCAL_BRIDGE);
if (node.dataIp() != null) {
printPortState(deviceService, node.intgBridge(), VXLAN_TUNNEL);
printPortState(deviceService, node.intgBridge(), GRE_TUNNEL);
@@ -103,6 +107,19 @@
EXTERNAL_BRIDGE,
node.extBridge());
}
+
+ print("[Local Bridge Status]");
+ Device localBridge = deviceService.getDevice(node.localBridge());
+ if (localBridge != null) {
+ print("%s %s=%s available=%s %s",
+ deviceService.isAvailable(localBridge.id()) ? MSG_OK : MSG_ERROR,
+ LOCAL_BRIDGE,
+ localBridge.id(),
+ deviceService.isAvailable(localBridge.id()),
+ localBridge.annotations());
+ printPortState(deviceService, node.localBridge(), LOCAL_BRIDGE);
+ printPortState(deviceService, node.localBridge(), LOCAL_TO_INTEGRATION_BRIDGE);
+ }
}
private void printPortState(DeviceService deviceService,
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java
index 7c7b32b..41bad16 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sNodeCodec.java
@@ -43,6 +43,7 @@
private static final String DATA_IP = "dataIp";
private static final String INTEGRATION_BRIDGE = "integrationBridge";
private static final String EXTERNAL_BRIDGE = "externalBridge";
+ private static final String LOCAL_BRIDGE = "localBridge";
private static final String STATE = "state";
private static final String EXTERNAL_INTF = "externalInterface";
private static final String EXTERNAL_BRIDGE_IP = "externalBridgeIp";
@@ -68,6 +69,10 @@
result.put(EXTERNAL_BRIDGE, node.extBridge().toString());
}
+ if (node.localBridge() != null) {
+ result.put(LOCAL_BRIDGE, node.localBridge().toString());
+ }
+
if (node.dataIp() != null) {
result.put(DATA_IP, node.dataIp().toString());
}
@@ -120,6 +125,11 @@
nodeBuilder.extBridge(DeviceId.deviceId(extBridgeJson.asText()));
}
+ JsonNode localBridgeJson = json.get(LOCAL_BRIDGE);
+ if (localBridgeJson != null) {
+ nodeBuilder.localBridge(DeviceId.deviceId(localBridgeJson.asText()));
+ }
+
JsonNode extIntfJson = json.get(EXTERNAL_INTF);
if (extIntfJson != null) {
nodeBuilder.extIntf(extIntfJson.asText());
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
index 83e537c..f31301f 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
@@ -74,6 +74,9 @@
import static org.onosproject.k8snode.api.Constants.GRE_TUNNEL;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_BRIDGE;
import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_EXTERNAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.INTEGRATION_TO_LOCAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.LOCAL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.LOCAL_TO_INTEGRATION_BRIDGE;
import static org.onosproject.k8snode.api.Constants.PHYSICAL_EXTERNAL_BRIDGE;
import static org.onosproject.k8snode.api.Constants.VXLAN;
import static org.onosproject.k8snode.api.Constants.VXLAN_TUNNEL;
@@ -196,6 +199,9 @@
if (!deviceService.isAvailable(k8sNode.extBridge())) {
createBridge(k8sNode, EXTERNAL_BRIDGE, k8sNode.extBridge());
}
+ if (!deviceService.isAvailable(k8sNode.localBridge())) {
+ createBridge(k8sNode, LOCAL_BRIDGE, k8sNode.localBridge());
+ }
}
@Override
@@ -328,23 +334,43 @@
return;
}
- PatchDescription brIntPatchDesc =
+ // integration bridge -> external bridge
+ PatchDescription brIntExtPatchDesc =
DefaultPatchDescription.builder()
.deviceId(INTEGRATION_BRIDGE)
.ifaceName(INTEGRATION_TO_EXTERNAL_BRIDGE)
.peer(PHYSICAL_EXTERNAL_BRIDGE)
.build();
- PatchDescription brExtPatchDesc =
+ // external bridge -> integration bridge
+ PatchDescription brExtIntPatchDesc =
DefaultPatchDescription.builder()
.deviceId(EXTERNAL_BRIDGE)
.ifaceName(PHYSICAL_EXTERNAL_BRIDGE)
.peer(INTEGRATION_TO_EXTERNAL_BRIDGE)
.build();
+ // integration bridge -> local bridge
+ PatchDescription brIntLocalPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(INTEGRATION_BRIDGE)
+ .ifaceName(INTEGRATION_TO_LOCAL_BRIDGE)
+ .peer(LOCAL_TO_INTEGRATION_BRIDGE)
+ .build();
+
+ // local bridge -> integration bridge
+ PatchDescription brLocalIntPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(LOCAL_BRIDGE)
+ .ifaceName(LOCAL_TO_INTEGRATION_BRIDGE)
+ .peer(INTEGRATION_TO_LOCAL_BRIDGE)
+ .build();
+
InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
- ifaceConfig.addPatchMode(INTEGRATION_TO_EXTERNAL_BRIDGE, brIntPatchDesc);
- ifaceConfig.addPatchMode(PHYSICAL_EXTERNAL_BRIDGE, brExtPatchDesc);
+ ifaceConfig.addPatchMode(INTEGRATION_TO_EXTERNAL_BRIDGE, brIntExtPatchDesc);
+ ifaceConfig.addPatchMode(PHYSICAL_EXTERNAL_BRIDGE, brExtIntPatchDesc);
+ ifaceConfig.addPatchMode(INTEGRATION_TO_LOCAL_BRIDGE, brIntLocalPatchDesc);
+ ifaceConfig.addPatchMode(LOCAL_TO_INTEGRATION_BRIDGE, brLocalIntPatchDesc);
}
/**
@@ -436,7 +462,8 @@
return k8sNode.intgBridge() != null && k8sNode.extBridge() != null &&
deviceService.isAvailable(k8sNode.intgBridge()) &&
- deviceService.isAvailable(k8sNode.extBridge());
+ deviceService.isAvailable(k8sNode.extBridge()) &&
+ deviceService.isAvailable(k8sNode.localBridge());
case DEVICE_CREATED:
if (k8sNode.dataIp() != null &&
!isIntfEnabled(k8sNode, VXLAN_TUNNEL)) {
@@ -505,6 +532,9 @@
// delete external bridge from the node
client.dropBridge(EXTERNAL_BRIDGE);
+ // delete local bridge from the node
+ client.dropBridge(LOCAL_BRIDGE);
+
// disconnect ovsdb
client.disconnect();
}
@@ -597,6 +627,7 @@
}
// TODO: also need to check the external bridge's availability
+ // TODO: also need to check the local bridge's availability
if (deviceService.isAvailable(device.id())) {
log.debug("Integration bridge created on {}",
k8sNode.hostname());
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
index 122a27b..5b3f353 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
@@ -158,6 +158,7 @@
K8sNode intNode;
K8sNode extNode;
+ K8sNode localNode;
if (node.intgBridge() == null) {
String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
@@ -183,7 +184,19 @@
NOT_DUPLICATED_MSG, extNode.extBridge());
}
- nodeStore.createNode(extNode);
+ if (node.localBridge() == null) {
+ String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
+ checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
+ localNode = extNode.updateLocalBridge(DeviceId.deviceId(deviceIdStr));
+ checkArgument(!hasLocalBridge(localNode.localBridge(), localNode.hostname()),
+ NOT_DUPLICATED_MSG, localNode.localBridge());
+ } else {
+ localNode = extNode;
+ checkArgument(!hasLocalBridge(localNode.localBridge(), localNode.hostname()),
+ NOT_DUPLICATED_MSG, localNode.localBridge());
+ }
+
+ nodeStore.createNode(localNode);
log.info(String.format(MSG_NODE, extNode.hostname(), MSG_CREATED));
}
@@ -193,6 +206,7 @@
K8sNode intNode;
K8sNode extNode;
+ K8sNode localNode;
K8sNode existingNode = nodeStore.node(node.hostname());
checkNotNull(existingNode, ERR_NULL_NODE);
@@ -221,7 +235,19 @@
NOT_DUPLICATED_MSG, extNode.extBridge());
}
- nodeStore.updateNode(extNode);
+ DeviceId existLocalBridge = nodeStore.node(node.hostname()).localBridge();
+
+ if (extNode.localBridge() == null) {
+ localNode = extNode.updateLocalBridge(existLocalBridge);
+ checkArgument(!hasLocalBridge(localNode.localBridge(), localNode.hostname()),
+ NOT_DUPLICATED_MSG, localNode.localBridge());
+ } else {
+ localNode = extNode;
+ checkArgument(!hasLocalBridge(localNode.localBridge(), localNode.hostname()),
+ NOT_DUPLICATED_MSG, localNode.localBridge());
+ }
+
+ nodeStore.updateNode(localNode);
log.info(String.format(MSG_NODE, extNode.hostname(), MSG_UPDATED));
}
@@ -295,6 +321,15 @@
return existNode.isPresent();
}
+ private boolean hasLocalBridge(DeviceId deviceId, String hostname) {
+ Optional<K8sNode> existNode = nodeStore.nodes().stream()
+ .filter(n -> !n.hostname().equals(hostname))
+ .filter(n -> deviceId.equals(n.localBridge()))
+ .findFirst();
+
+ return existNode.isPresent();
+ }
+
private class InternalNodeStoreDelegate implements K8sNodeStoreDelegate {
@Override
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
index fbccdd0..50d769f 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
@@ -80,11 +80,17 @@
private static final Device MINION_2_EXT_DEVICE = createDevice(5);
private static final Device MINION_3_EXT_DEVICE = createDevice(6);
+ private static final Device MINION_1_LOCAL_DEVICE = createDevice(7);
+ private static final Device MINION_2_LOCAL_DEVICE = createDevice(8);
+ private static final Device MINION_3_LOCAL_DEVICE = createDevice(9);
+
+
private static final K8sNode MINION_1 = createNode(
MINION_1_HOSTNAME,
MINION,
MINION_1_INTG_DEVICE,
MINION_1_EXT_DEVICE,
+ MINION_1_LOCAL_DEVICE,
IpAddress.valueOf("10.100.0.1"),
INIT
);
@@ -93,6 +99,7 @@
MINION,
MINION_2_INTG_DEVICE,
MINION_2_EXT_DEVICE,
+ MINION_2_LOCAL_DEVICE,
IpAddress.valueOf("10.100.0.2"),
INIT
);
@@ -101,6 +108,7 @@
MINION,
MINION_3_INTG_DEVICE,
MINION_3_EXT_DEVICE,
+ MINION_3_LOCAL_DEVICE,
IpAddress.valueOf("10.100.0.3"),
COMPLETE
);
@@ -328,12 +336,14 @@
private static K8sNode createNode(String hostname, K8sNode.Type type,
Device intgBridge, Device extBridge,
- IpAddress ipAddr, K8sNodeState state) {
+ Device localBridge, IpAddress ipAddr,
+ K8sNodeState state) {
return DefaultK8sNode.builder()
.hostname(hostname)
.type(type)
.intgBridge(intgBridge.id())
.extBridge(extBridge.id())
+ .localBridge(localBridge.id())
.managementIp(ipAddr)
.dataIp(ipAddr)
.state(state)