Inter-connect k8s and openstack intg bridge, kbr-ex and kbr-router
Change-Id: Id7d3c874e8b267252ca387b1ca6f67b9f9bc5116
(cherry picked from commit 019ce6a7143620fab32b7b0579aa7381aa102af2)
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
index 5fd2758..56fe815 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
@@ -33,10 +33,15 @@
import org.onosproject.k8snetworking.api.K8sNetworkService;
import org.onosproject.k8snetworking.api.K8sPort;
import org.onosproject.k8snetworking.util.RulePopulatorUtil;
+import org.onosproject.k8snode.api.K8sHost;
+import org.onosproject.k8snode.api.K8sHostEvent;
+import org.onosproject.k8snode.api.K8sHostListener;
+import org.onosproject.k8snode.api.K8sHostService;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNodeEvent;
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
+import org.onosproject.k8snode.api.K8sRouterBridge;
import org.onosproject.mastership.MastershipService;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
@@ -64,14 +69,17 @@
import static org.onosproject.k8snetworking.api.Constants.EXT_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.POD_RESOLUTION_TABLE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_DEFAULT_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_EXTERNAL_ROUTING_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
+import static org.onosproject.k8snetworking.api.Constants.ROUTER_EXTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.CT_NAT_SRC_FLAG;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveArpSpaToTpaExtension;
import static org.onosproject.k8snetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
import static org.onosproject.k8snode.api.Constants.DEFAULT_EXTERNAL_GATEWAY_MAC;
+import static org.onosproject.k8snode.api.K8sApiConfig.Mode.PASSTHROUGH;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -115,12 +123,17 @@
protected K8sNodeService k8sNodeService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sHostService k8sHostService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
protected K8sFlowRuleService k8sFlowRuleService;
private final InternalK8sNetworkListener k8sNetworkListener =
new InternalK8sNetworkListener();
private final InternalK8sNodeListener k8sNodeListener =
new InternalK8sNodeListener();
+ private final InternalK8sHostListener k8sHostListener =
+ new InternalK8sHostListener();
private final ExecutorService eventExecutor = newSingleThreadExecutor(
groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
@@ -135,12 +148,14 @@
leadershipService.runForLeadership(appId.name());
k8sNetworkService.addListener(k8sNetworkListener);
k8sNodeService.addListener(k8sNodeListener);
+ k8sHostService.addListener(k8sHostListener);
log.info("Started");
}
@Deactivate
protected void deactivate() {
+ k8sHostService.removeListener(k8sHostListener);
k8sNodeService.removeListener(k8sNodeListener);
k8sNetworkService.removeListener(k8sNetworkListener);
leadershipService.withdraw(appId.name());
@@ -201,8 +216,8 @@
install);
}
- private void setSnatDownstreamRule(K8sNode k8sNode,
- boolean install) {
+ private void setExtSnatDownstreamRule(K8sNode k8sNode,
+ boolean install) {
DeviceId deviceId = k8sNode.extBridge();
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
@@ -231,8 +246,8 @@
install);
}
- private void setSnatUpstreamRule(K8sNode k8sNode,
- boolean install) {
+ private void setExtSnatUpstreamRule(K8sNode k8sNode,
+ boolean install) {
K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());
@@ -263,11 +278,15 @@
.setEthSrc(k8sNode.extBridgeMac())
.setEthDst(k8sNode.extGatewayMac());
- if (MacAddress.valueOf(DEFAULT_EXTERNAL_GATEWAY_MAC).equals(
- k8sNode.extGatewayMac())) {
- tBuilder.setOutput(k8sNode.extIntfPortNum());
+ if (k8sNode.mode() == PASSTHROUGH) {
+ tBuilder.setOutput(k8sNode.extToRouterPortNum());
} else {
- tBuilder.setOutput(k8sNode.extBridgePortNum());
+ if (MacAddress.valueOf(DEFAULT_EXTERNAL_GATEWAY_MAC).equals(
+ k8sNode.extGatewayMac())) {
+ tBuilder.setOutput(k8sNode.extIntfPortNum());
+ } else {
+ tBuilder.setOutput(k8sNode.extBridgePortNum());
+ }
}
}
@@ -312,6 +331,52 @@
});
}
+ private void setRouterSnatUpstreamRule(K8sNode k8sNode,
+ K8sRouterBridge bridge,
+ boolean install) {
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchInPort(k8sNode.routerToExtPortNum())
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(k8sNode.routerPortNum())
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ bridge.deviceId(),
+ selector,
+ treatment,
+ PRIORITY_DEFAULT_RULE,
+ ROUTER_EXTRY_TABLE,
+ install);
+ }
+
+ private void setRouterSnatDownstreamRule(K8sNode k8sNode,
+ K8sRouterBridge bridge,
+ boolean install) {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchInPort(k8sNode.routerPortNum())
+ .matchIPDst(IpPrefix.valueOf(k8sNode.extBridgeIp(), 32))
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(k8sNode.routerToExtPortNum())
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ bridge.deviceId(),
+ selector,
+ treatment,
+ PRIORITY_DEFAULT_RULE,
+ ROUTER_EXTRY_TABLE,
+ install);
+ }
+
private class InternalK8sNodeListener implements K8sNodeListener {
private boolean isRelevantHelper() {
@@ -339,13 +404,54 @@
}
setExtIntfArpRule(k8sNode, true);
- setSnatDownstreamRule(k8sNode, true);
+ setExtSnatDownstreamRule(k8sNode, true);
setContainerToExtRule(k8sNode, true);
}
private void processNodeUpdate(K8sNode k8sNode) {
if (k8sNode.extGatewayMac() != null) {
- setSnatUpstreamRule(k8sNode, true);
+ setExtSnatUpstreamRule(k8sNode, true);
+ }
+ }
+ }
+
+ private class InternalK8sHostListener implements K8sHostListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sHostEvent event) {
+ switch (event.type()) {
+ case K8S_HOST_COMPLETE:
+ eventExecutor.execute(() -> processNodeCompletion(event.subject()));
+ break;
+ case K8S_HOST_INCOMPLETE:
+ default:
+ break;
+ }
+ }
+
+ private void processNodeCompletion(K8sHost k8sHost) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ for (String name : k8sHost.nodeNames()) {
+ K8sNode node = k8sNodeService.node(name);
+ if (node == null) {
+ return;
+ }
+ K8sRouterBridge bridge = k8sHost.routerBridges().stream()
+ .filter(b -> b.segmentId() == node.segmentId())
+ .findAny().orElse(null);
+ if (bridge == null) {
+ return;
+ }
+
+ setRouterSnatUpstreamRule(node, bridge, true);
+ setRouterSnatDownstreamRule(node, bridge, true);
}
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 03a953e..dde6828 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -158,7 +158,7 @@
if (node.hostname().equals(k8sNetwork.name())) {
tBuilder.setEthDst(node.intgBridgeMac())
- .setOutput(PortNumber.LOCAL);
+ .setOutput(node.intgEntryPortNum());
} else {
K8sNode localNode = k8sNodeService.node(k8sNetwork.name());
@@ -198,7 +198,7 @@
if (node.hostname().equals(k8sNetwork.name())) {
sBuilder = DefaultTrafficSelector.builder()
- .matchInPort(PortNumber.LOCAL)
+ .matchInPort(node.intgEntryPortNum())
.matchEthType(Ethernet.TYPE_IPV4)
.matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
HOST_PREFIX));
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
index d62a923..a0ccf5d 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
@@ -349,7 +349,7 @@
private void setLocalTunnelTagFlowRules(K8sNode k8sNode, boolean install) {
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
- .matchInPort(PortNumber.LOCAL)
+ .matchInPort(k8sNode.intgEntryPortNum())
.build();
K8sNetwork net = k8sNetworkService.network(k8sNode.hostname());