Remove bridges and associated flow rules when removing k8s nodes
Change-Id: Iab54503a4bb75874f69e8e1623bb11c66cad9eee
diff --git a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
index 2c60c79..4768043 100644
--- a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
+++ b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
@@ -104,6 +104,7 @@
public static final int PRIORITY_FORCED_ACL_RULE = 50000;
public static final int PRIORITY_ICMP_PROBE_RULE = 50000;
public static final int PRIORITY_NODE_PORT_RULE = 42000;
+ public static final int PRIORITY_ROUTER_RULE = 10000;
public static final int PRIORITY_DEFAULT_RULE = 0;
// flow table index
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
index 0234613..f5394dc 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
@@ -28,10 +28,13 @@
import org.onosproject.k8snetworking.api.K8sNetworkEvent;
import org.onosproject.k8snetworking.api.K8sNetworkListener;
import org.onosproject.k8snetworking.api.K8sNetworkService;
+import org.onosproject.k8snode.api.K8sHost;
+import org.onosproject.k8snode.api.K8sHostService;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNodeEvent;
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
+import org.onosproject.k8snode.api.K8sRouterBridge;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
@@ -66,7 +69,9 @@
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.NAMESPACE_TABLE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_CIDR_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_DEFAULT_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_SNAT_RULE;
+import static org.onosproject.k8snetworking.api.Constants.ROUTER_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
import static org.onosproject.k8snetworking.api.Constants.STAT_EGRESS_TABLE;
import static org.onosproject.k8snetworking.api.Constants.STAT_INGRESS_TABLE;
@@ -111,6 +116,9 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected K8sNodeService k8sNodeService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sHostService k8sHostService;
+
private final ExecutorService deviceEventExecutor =
Executors.newSingleThreadExecutor(groupedThreads(
getClass().getSimpleName(), "device-event"));
@@ -262,6 +270,29 @@
connectTables(deviceId, VTAP_EGRESS_TABLE, FORWARDING_TABLE);
}
+ private void setupRouter(K8sNode k8sNode, K8sRouterBridge bridge) {
+ if (k8sNode.routerPortNum() == null) {
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder().build();
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(PortNumber.NORMAL)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .forDevice(bridge.deviceId())
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(PRIORITY_DEFAULT_RULE)
+ .fromApp(appId)
+ .makePermanent()
+ .forTable(ROUTER_ENTRY_TABLE)
+ .build();
+
+ applyRule(flowRule, true);
+ }
+
private void setupJumpTable(K8sNode k8sNode) {
DeviceId deviceId = k8sNode.intgBridge();
@@ -301,6 +332,52 @@
applyRule(flowRule, true);
}
+ private void purgeAnyRoutingRule(K8sNode localNode) {
+ K8sNetwork k8sNetwork = k8sNetworkService.network(localNode.hostname());
+ IpPrefix srcIpPrefix = IpPrefix.valueOf(k8sNetwork.gatewayIp(), HOST_PREFIX);
+ IpPrefix dstIpPrefix = IpPrefix.valueOf(k8sNetwork.cidr());
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(srcIpPrefix)
+ .matchIPDst(dstIpPrefix);
+
+ for (K8sNode node : k8sNodeService.nodes()) {
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setTunnelId(Long.valueOf(k8sNetwork.segmentId()));
+
+ if (node.hostname().equals(k8sNetwork.name())) {
+ tBuilder.transition(STAT_EGRESS_TABLE);
+ } else {
+ tBuilder.setOutput(node.intgToTunPortNum());
+
+ // install flows into tunnel bridge
+ PortNumber portNum = tunnelPortNumByNetId(k8sNetwork.networkId(),
+ k8sNetworkService, node);
+ TrafficTreatment treatmentToRemote = DefaultTrafficTreatment.builder()
+ .extension(buildExtension(
+ deviceService,
+ node.tunBridge(),
+ localNode.dataIp().getIp4Address()),
+ node.tunBridge())
+ .setTunnelId(Long.valueOf(k8sNetwork.segmentId()))
+ .setOutput(portNum)
+ .build();
+
+ FlowRule remoteFlowRule = DefaultFlowRule.builder()
+ .forDevice(node.tunBridge())
+ .withSelector(sBuilder.build())
+ .withTreatment(treatmentToRemote)
+ .withPriority(PRIORITY_CIDR_RULE)
+ .fromApp(appId)
+ .makePermanent()
+ .forTable(TUN_ENTRY_TABLE)
+ .build();
+ applyRule(remoteFlowRule, false);
+ }
+ }
+ }
+
private void setAnyRoutingRule(IpPrefix srcIpPrefix, MacAddress mac,
K8sNetwork k8sNetwork) {
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
@@ -375,6 +452,9 @@
case K8S_NODE_COMPLETE:
deviceEventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ deviceEventExecutor.execute(() -> processNodeOffboard(event.subject()));
+ break;
case K8S_NODE_CREATED:
default:
// do nothing
@@ -392,6 +472,24 @@
initializePipeline(node);
k8sNetworkService.networks().forEach(K8sFlowRuleManager.this::setupHostRoutingRule);
+
+ for (K8sHost host : k8sHostService.completeHosts()) {
+ if (host.nodeNames().contains(node.hostname())) {
+ host.routerBridges().stream()
+ .filter(b -> b.segmentId() == node.segmentId())
+ .findAny().ifPresent(bridge -> setupRouter(node, bridge));
+ }
+ }
+ }
+
+ private void processNodeOffboard(K8sNode node) {
+ log.info("Offboarded node {} is detected", node.hostname());
+
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ purgeAnyRoutingRule(node);
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
index 387494a..6d201ba 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
@@ -400,6 +400,9 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
+ break;
case K8S_NODE_INCOMPLETE:
default:
break;
@@ -432,5 +435,9 @@
setIntgToExtRules(updatedNode, getServiceCidr(), true);
setTunToIntgRules(updatedNode, true);
}
+
+ private void processNodeOffboard(K8sNode k8sNode) {
+ setTunToIntgRules(k8sNode, false);
+ }
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
index 9e24daa..a631c62 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
@@ -215,8 +215,8 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
- case K8S_NODE_INCOMPLETE:
- eventExecutor.execute(() -> processNodeIncompletion(event.subject()));
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
break;
default:
break;
@@ -232,7 +232,7 @@
setCniPtNodePortRules(k8sNode, true);
}
- private void processNodeIncompletion(K8sNode k8sNode) {
+ private void processNodeOffboard(K8sNode k8sNode) {
if (!isRelevantHelper()) {
return;
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
index 1b5b9fd..1ad3e9e 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
@@ -67,8 +67,8 @@
import static org.onosproject.k8snetworking.api.Constants.EXT_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.POD_RESOLUTION_TABLE;
-import static org.onosproject.k8snetworking.api.Constants.PRIORITY_DEFAULT_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_EXTERNAL_ROUTING_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_ROUTER_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTER_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
@@ -346,7 +346,7 @@
bridge.deviceId(),
ipSelector,
treatment,
- PRIORITY_DEFAULT_RULE,
+ PRIORITY_ROUTER_RULE,
ROUTER_ENTRY_TABLE,
install);
@@ -360,7 +360,7 @@
bridge.deviceId(),
arpSelector,
treatment,
- PRIORITY_DEFAULT_RULE,
+ PRIORITY_ROUTER_RULE,
ROUTER_ENTRY_TABLE,
install);
}
@@ -387,7 +387,7 @@
bridge.deviceId(),
ipSelector,
treatment,
- PRIORITY_DEFAULT_RULE,
+ PRIORITY_ROUTER_RULE,
ROUTER_ENTRY_TABLE,
install);
@@ -402,7 +402,7 @@
bridge.deviceId(),
arpSelector,
treatment,
- PRIORITY_DEFAULT_RULE,
+ PRIORITY_ROUTER_RULE,
ROUTER_ENTRY_TABLE,
install);
}
@@ -436,6 +436,9 @@
case K8S_NODE_UPDATED:
eventExecutor.execute(() -> processNodeUpdate(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
+ break;
case K8S_NODE_INCOMPLETE:
default:
break;
@@ -453,6 +456,17 @@
setRouterSnatRules(k8sNode, true);
}
+ private void processNodeOffboard(K8sNode k8sNode) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setExtIntfArpRule(k8sNode, false);
+ setExtSnatDownstreamRule(k8sNode, false);
+ setContainerToExtRule(k8sNode, false);
+ setRouterSnatRules(k8sNode, false);
+ }
+
private void processNodeUpdate(K8sNode k8sNode) {
if (k8sNode.extGatewayMac() != null) {
setExtSnatUpstreamRule(k8sNode, true);
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
index 45b592e..0dca524 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
@@ -631,7 +631,9 @@
.matchIPSrc(prefix)
.matchIPDst(IpPrefix.valueOf(network.cidr()));
- k8sNodeService.completeNodes().forEach(n -> {
+ Set<K8sNode> nodes = install ? k8sNodeService.completeNodes() : k8sNodeService.nodes();
+
+ nodes.forEach(n -> {
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
.setTunnelId(Long.valueOf(network.segmentId()));
@@ -981,8 +983,10 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(k8sNode));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(k8sNode));
+ break;
case K8S_NODE_INCOMPLETE:
- case K8S_NODE_REMOVED:
default:
break;
}
@@ -997,6 +1001,15 @@
k8sEndpointsService.endpointses().forEach(e -> setEndpointsRules(e, true));
k8sNetworkService.networks().forEach(n -> setupServiceDefaultRule(n, true));
}
+
+ private void processNodeOffboard(K8sNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ K8sNetwork network = k8sNetworkService.network(node.hostname());
+ setupServiceDefaultRule(network, false);
+ }
}
private class InternalK8sNetworkListener implements K8sNetworkListener {
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 2c580cb..953a75e 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -227,14 +227,50 @@
}
}
- private void setInterNodeRoutingRules(K8sNetwork k8sNetwork, boolean install) {
- K8sNode srcNode = k8sNodeService.node(k8sNetwork.name());
+ private void setGatewayTunnelRule(K8sNode node, boolean install) {
+ K8sNetwork k8sNetwork = k8sNetworkService.network(node.hostname());
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ K8sNode localNode = k8sNodeService.node(k8sNetwork.name());
+
+ tBuilder.setOutput(node.intgToTunPortNum());
+
+ // install flows into tunnel bridge
+ PortNumber portNum = tunnelPortNumByNetId(k8sNetwork.networkId(),
+ k8sNetworkService, node);
+ TrafficTreatment treatmentToRemote = DefaultTrafficTreatment.builder()
+ .extension(buildExtension(
+ deviceService,
+ node.tunBridge(),
+ localNode.dataIp().getIp4Address()),
+ node.tunBridge())
+ .setTunnelId(Long.valueOf(k8sNetwork.segmentId()))
+ .setOutput(portNum)
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ node.tunBridge(),
+ sBuilder.build(),
+ treatmentToRemote,
+ PRIORITY_GATEWAY_RULE,
+ TUN_ENTRY_TABLE,
+ install);
+ }
+
+ private void setInterNodeRoutingRules(K8sNode srcNode, boolean install) {
if (srcNode == null) {
return;
}
- for (K8sNode dstNode : k8sNodeService.completeNodes()) {
+ for (K8sNode dstNode : k8sNodeService.nodes()) {
if (StringUtils.equals(srcNode.hostname(), dstNode.hostname())) {
continue;
}
@@ -425,7 +461,9 @@
setGatewayRule(event.subject(), true);
setLocalBridgeRules(event.subject(), true);
setLocalBridgeArpRules(event.subject(), true);
- setInterNodeRoutingRules(event.subject(), true);
+
+ K8sNode k8sNode = k8sNodeService.node(event.subject().networkId());
+ setInterNodeRoutingRules(k8sNode, true);
}
private void processNetworkRemoval(K8sNetworkEvent event) {
@@ -436,7 +474,9 @@
setGatewayRule(event.subject(), false);
setLocalBridgeRules(event.subject(), false);
setLocalBridgeArpRules(event.subject(), false);
- setInterNodeRoutingRules(event.subject(), false);
+
+ K8sNode k8sNode = k8sNodeService.node(event.subject().networkId());
+ setInterNodeRoutingRules(k8sNode, false);
}
}
@@ -451,6 +491,9 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
+ break;
case K8S_NODE_INCOMPLETE:
default:
break;
@@ -467,7 +510,17 @@
k8sNetworkService.networks().forEach(n -> setGatewayRule(n, true));
k8sNetworkService.networks().forEach(n -> setLocalBridgeRules(n, true));
k8sNetworkService.networks().forEach(n -> setLocalBridgeArpRules(n, true));
- k8sNetworkService.networks().forEach(n -> setInterNodeRoutingRules(n, true));
+
+ setInterNodeRoutingRules(node, true);
+ }
+
+ private void processNodeOffboard(K8sNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setGatewayTunnelRule(node, false);
+ setInterNodeRoutingRules(node, false);
}
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
index bec0540..99ccbd9 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
@@ -497,6 +497,8 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
default:
break;
}
@@ -511,5 +513,13 @@
setLocalTunnelTagFlowRules(k8sNode, true);
setRulesForTunnelBridge(k8sNode, true);
}
+
+ private void processNodeOffboard(K8sNode k8sNode) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setRulesForTunnelBridge(k8sNode, false);
+ }
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java
index f636b1a..88543ec 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java
@@ -126,4 +126,17 @@
adminService.removePort(id);
return Response.noContent().build();
}
+
+ /**
+ * Removes the port with the given id.
+ *
+ * @return 204 NO_CONTENT, 400 BAD_REQUEST if the port does not exist
+ */
+ @DELETE
+ public Response removeAllPorts() {
+ adminService.ports().stream()
+ .map(K8sPort::portId)
+ .forEach(adminService::removePort);
+ return Response.noContent().build();
+ }
}