Remove bridges and associated flow rules when removing k8s nodes
Change-Id: Iab54503a4bb75874f69e8e1623bb11c66cad9eee
diff --git a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
index 2c60c79..4768043 100644
--- a/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
+++ b/apps/k8s-networking/api/src/main/java/org/onosproject/k8snetworking/api/Constants.java
@@ -104,6 +104,7 @@
public static final int PRIORITY_FORCED_ACL_RULE = 50000;
public static final int PRIORITY_ICMP_PROBE_RULE = 50000;
public static final int PRIORITY_NODE_PORT_RULE = 42000;
+ public static final int PRIORITY_ROUTER_RULE = 10000;
public static final int PRIORITY_DEFAULT_RULE = 0;
// flow table index
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
index 0234613..f5394dc 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
@@ -28,10 +28,13 @@
import org.onosproject.k8snetworking.api.K8sNetworkEvent;
import org.onosproject.k8snetworking.api.K8sNetworkListener;
import org.onosproject.k8snetworking.api.K8sNetworkService;
+import org.onosproject.k8snode.api.K8sHost;
+import org.onosproject.k8snode.api.K8sHostService;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNodeEvent;
import org.onosproject.k8snode.api.K8sNodeListener;
import org.onosproject.k8snode.api.K8sNodeService;
+import org.onosproject.k8snode.api.K8sRouterBridge;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
@@ -66,7 +69,9 @@
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.NAMESPACE_TABLE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_CIDR_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_DEFAULT_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_SNAT_RULE;
+import static org.onosproject.k8snetworking.api.Constants.ROUTER_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
import static org.onosproject.k8snetworking.api.Constants.STAT_EGRESS_TABLE;
import static org.onosproject.k8snetworking.api.Constants.STAT_INGRESS_TABLE;
@@ -111,6 +116,9 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected K8sNodeService k8sNodeService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sHostService k8sHostService;
+
private final ExecutorService deviceEventExecutor =
Executors.newSingleThreadExecutor(groupedThreads(
getClass().getSimpleName(), "device-event"));
@@ -262,6 +270,29 @@
connectTables(deviceId, VTAP_EGRESS_TABLE, FORWARDING_TABLE);
}
+ private void setupRouter(K8sNode k8sNode, K8sRouterBridge bridge) {
+ if (k8sNode.routerPortNum() == null) {
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder().build();
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(PortNumber.NORMAL)
+ .build();
+
+ FlowRule flowRule = DefaultFlowRule.builder()
+ .forDevice(bridge.deviceId())
+ .withSelector(selector)
+ .withTreatment(treatment)
+ .withPriority(PRIORITY_DEFAULT_RULE)
+ .fromApp(appId)
+ .makePermanent()
+ .forTable(ROUTER_ENTRY_TABLE)
+ .build();
+
+ applyRule(flowRule, true);
+ }
+
private void setupJumpTable(K8sNode k8sNode) {
DeviceId deviceId = k8sNode.intgBridge();
@@ -301,6 +332,52 @@
applyRule(flowRule, true);
}
+ private void purgeAnyRoutingRule(K8sNode localNode) {
+ K8sNetwork k8sNetwork = k8sNetworkService.network(localNode.hostname());
+ IpPrefix srcIpPrefix = IpPrefix.valueOf(k8sNetwork.gatewayIp(), HOST_PREFIX);
+ IpPrefix dstIpPrefix = IpPrefix.valueOf(k8sNetwork.cidr());
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(srcIpPrefix)
+ .matchIPDst(dstIpPrefix);
+
+ for (K8sNode node : k8sNodeService.nodes()) {
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setTunnelId(Long.valueOf(k8sNetwork.segmentId()));
+
+ if (node.hostname().equals(k8sNetwork.name())) {
+ tBuilder.transition(STAT_EGRESS_TABLE);
+ } else {
+ tBuilder.setOutput(node.intgToTunPortNum());
+
+ // install flows into tunnel bridge
+ PortNumber portNum = tunnelPortNumByNetId(k8sNetwork.networkId(),
+ k8sNetworkService, node);
+ TrafficTreatment treatmentToRemote = DefaultTrafficTreatment.builder()
+ .extension(buildExtension(
+ deviceService,
+ node.tunBridge(),
+ localNode.dataIp().getIp4Address()),
+ node.tunBridge())
+ .setTunnelId(Long.valueOf(k8sNetwork.segmentId()))
+ .setOutput(portNum)
+ .build();
+
+ FlowRule remoteFlowRule = DefaultFlowRule.builder()
+ .forDevice(node.tunBridge())
+ .withSelector(sBuilder.build())
+ .withTreatment(treatmentToRemote)
+ .withPriority(PRIORITY_CIDR_RULE)
+ .fromApp(appId)
+ .makePermanent()
+ .forTable(TUN_ENTRY_TABLE)
+ .build();
+ applyRule(remoteFlowRule, false);
+ }
+ }
+ }
+
private void setAnyRoutingRule(IpPrefix srcIpPrefix, MacAddress mac,
K8sNetwork k8sNetwork) {
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
@@ -375,6 +452,9 @@
case K8S_NODE_COMPLETE:
deviceEventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ deviceEventExecutor.execute(() -> processNodeOffboard(event.subject()));
+ break;
case K8S_NODE_CREATED:
default:
// do nothing
@@ -392,6 +472,24 @@
initializePipeline(node);
k8sNetworkService.networks().forEach(K8sFlowRuleManager.this::setupHostRoutingRule);
+
+ for (K8sHost host : k8sHostService.completeHosts()) {
+ if (host.nodeNames().contains(node.hostname())) {
+ host.routerBridges().stream()
+ .filter(b -> b.segmentId() == node.segmentId())
+ .findAny().ifPresent(bridge -> setupRouter(node, bridge));
+ }
+ }
+ }
+
+ private void processNodeOffboard(K8sNode node) {
+ log.info("Offboarded node {} is detected", node.hostname());
+
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ purgeAnyRoutingRule(node);
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
index 387494a..6d201ba 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
@@ -400,6 +400,9 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
+ break;
case K8S_NODE_INCOMPLETE:
default:
break;
@@ -432,5 +435,9 @@
setIntgToExtRules(updatedNode, getServiceCidr(), true);
setTunToIntgRules(updatedNode, true);
}
+
+ private void processNodeOffboard(K8sNode k8sNode) {
+ setTunToIntgRules(k8sNode, false);
+ }
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
index 9e24daa..a631c62 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sOpenstackIntegrationHandler.java
@@ -215,8 +215,8 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
- case K8S_NODE_INCOMPLETE:
- eventExecutor.execute(() -> processNodeIncompletion(event.subject()));
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
break;
default:
break;
@@ -232,7 +232,7 @@
setCniPtNodePortRules(k8sNode, true);
}
- private void processNodeIncompletion(K8sNode k8sNode) {
+ private void processNodeOffboard(K8sNode k8sNode) {
if (!isRelevantHelper()) {
return;
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
index 1b5b9fd..1ad3e9e 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sRoutingSnatHandler.java
@@ -67,8 +67,8 @@
import static org.onosproject.k8snetworking.api.Constants.EXT_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
import static org.onosproject.k8snetworking.api.Constants.POD_RESOLUTION_TABLE;
-import static org.onosproject.k8snetworking.api.Constants.PRIORITY_DEFAULT_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_EXTERNAL_ROUTING_RULE;
+import static org.onosproject.k8snetworking.api.Constants.PRIORITY_ROUTER_RULE;
import static org.onosproject.k8snetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
import static org.onosproject.k8snetworking.api.Constants.ROUTER_ENTRY_TABLE;
import static org.onosproject.k8snetworking.api.Constants.ROUTING_TABLE;
@@ -346,7 +346,7 @@
bridge.deviceId(),
ipSelector,
treatment,
- PRIORITY_DEFAULT_RULE,
+ PRIORITY_ROUTER_RULE,
ROUTER_ENTRY_TABLE,
install);
@@ -360,7 +360,7 @@
bridge.deviceId(),
arpSelector,
treatment,
- PRIORITY_DEFAULT_RULE,
+ PRIORITY_ROUTER_RULE,
ROUTER_ENTRY_TABLE,
install);
}
@@ -387,7 +387,7 @@
bridge.deviceId(),
ipSelector,
treatment,
- PRIORITY_DEFAULT_RULE,
+ PRIORITY_ROUTER_RULE,
ROUTER_ENTRY_TABLE,
install);
@@ -402,7 +402,7 @@
bridge.deviceId(),
arpSelector,
treatment,
- PRIORITY_DEFAULT_RULE,
+ PRIORITY_ROUTER_RULE,
ROUTER_ENTRY_TABLE,
install);
}
@@ -436,6 +436,9 @@
case K8S_NODE_UPDATED:
eventExecutor.execute(() -> processNodeUpdate(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
+ break;
case K8S_NODE_INCOMPLETE:
default:
break;
@@ -453,6 +456,17 @@
setRouterSnatRules(k8sNode, true);
}
+ private void processNodeOffboard(K8sNode k8sNode) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setExtIntfArpRule(k8sNode, false);
+ setExtSnatDownstreamRule(k8sNode, false);
+ setContainerToExtRule(k8sNode, false);
+ setRouterSnatRules(k8sNode, false);
+ }
+
private void processNodeUpdate(K8sNode k8sNode) {
if (k8sNode.extGatewayMac() != null) {
setExtSnatUpstreamRule(k8sNode, true);
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
index 45b592e..0dca524 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
@@ -631,7 +631,9 @@
.matchIPSrc(prefix)
.matchIPDst(IpPrefix.valueOf(network.cidr()));
- k8sNodeService.completeNodes().forEach(n -> {
+ Set<K8sNode> nodes = install ? k8sNodeService.completeNodes() : k8sNodeService.nodes();
+
+ nodes.forEach(n -> {
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
.setTunnelId(Long.valueOf(network.segmentId()));
@@ -981,8 +983,10 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(k8sNode));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(k8sNode));
+ break;
case K8S_NODE_INCOMPLETE:
- case K8S_NODE_REMOVED:
default:
break;
}
@@ -997,6 +1001,15 @@
k8sEndpointsService.endpointses().forEach(e -> setEndpointsRules(e, true));
k8sNetworkService.networks().forEach(n -> setupServiceDefaultRule(n, true));
}
+
+ private void processNodeOffboard(K8sNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ K8sNetwork network = k8sNetworkService.network(node.hostname());
+ setupServiceDefaultRule(network, false);
+ }
}
private class InternalK8sNetworkListener implements K8sNetworkListener {
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 2c580cb..953a75e 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -227,14 +227,50 @@
}
}
- private void setInterNodeRoutingRules(K8sNetwork k8sNetwork, boolean install) {
- K8sNode srcNode = k8sNodeService.node(k8sNetwork.name());
+ private void setGatewayTunnelRule(K8sNode node, boolean install) {
+ K8sNetwork k8sNetwork = k8sNetworkService.network(node.hostname());
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
+ HOST_PREFIX));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ K8sNode localNode = k8sNodeService.node(k8sNetwork.name());
+
+ tBuilder.setOutput(node.intgToTunPortNum());
+
+ // install flows into tunnel bridge
+ PortNumber portNum = tunnelPortNumByNetId(k8sNetwork.networkId(),
+ k8sNetworkService, node);
+ TrafficTreatment treatmentToRemote = DefaultTrafficTreatment.builder()
+ .extension(buildExtension(
+ deviceService,
+ node.tunBridge(),
+ localNode.dataIp().getIp4Address()),
+ node.tunBridge())
+ .setTunnelId(Long.valueOf(k8sNetwork.segmentId()))
+ .setOutput(portNum)
+ .build();
+
+ k8sFlowRuleService.setRule(
+ appId,
+ node.tunBridge(),
+ sBuilder.build(),
+ treatmentToRemote,
+ PRIORITY_GATEWAY_RULE,
+ TUN_ENTRY_TABLE,
+ install);
+ }
+
+ private void setInterNodeRoutingRules(K8sNode srcNode, boolean install) {
if (srcNode == null) {
return;
}
- for (K8sNode dstNode : k8sNodeService.completeNodes()) {
+ for (K8sNode dstNode : k8sNodeService.nodes()) {
if (StringUtils.equals(srcNode.hostname(), dstNode.hostname())) {
continue;
}
@@ -425,7 +461,9 @@
setGatewayRule(event.subject(), true);
setLocalBridgeRules(event.subject(), true);
setLocalBridgeArpRules(event.subject(), true);
- setInterNodeRoutingRules(event.subject(), true);
+
+ K8sNode k8sNode = k8sNodeService.node(event.subject().networkId());
+ setInterNodeRoutingRules(k8sNode, true);
}
private void processNetworkRemoval(K8sNetworkEvent event) {
@@ -436,7 +474,9 @@
setGatewayRule(event.subject(), false);
setLocalBridgeRules(event.subject(), false);
setLocalBridgeArpRules(event.subject(), false);
- setInterNodeRoutingRules(event.subject(), false);
+
+ K8sNode k8sNode = k8sNodeService.node(event.subject().networkId());
+ setInterNodeRoutingRules(k8sNode, false);
}
}
@@ -451,6 +491,9 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
+ break;
case K8S_NODE_INCOMPLETE:
default:
break;
@@ -467,7 +510,17 @@
k8sNetworkService.networks().forEach(n -> setGatewayRule(n, true));
k8sNetworkService.networks().forEach(n -> setLocalBridgeRules(n, true));
k8sNetworkService.networks().forEach(n -> setLocalBridgeArpRules(n, true));
- k8sNetworkService.networks().forEach(n -> setInterNodeRoutingRules(n, true));
+
+ setInterNodeRoutingRules(node, true);
+ }
+
+ private void processNodeOffboard(K8sNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setGatewayTunnelRule(node, false);
+ setInterNodeRoutingRules(node, false);
}
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
index bec0540..99ccbd9 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
@@ -497,6 +497,8 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
+ case K8S_NODE_OFF_BOARDED:
+ eventExecutor.execute(() -> processNodeOffboard(event.subject()));
default:
break;
}
@@ -511,5 +513,13 @@
setLocalTunnelTagFlowRules(k8sNode, true);
setRulesForTunnelBridge(k8sNode, true);
}
+
+ private void processNodeOffboard(K8sNode k8sNode) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setRulesForTunnelBridge(k8sNode, false);
+ }
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java
index f636b1a..88543ec 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java
@@ -126,4 +126,17 @@
adminService.removePort(id);
return Response.noContent().build();
}
+
+ /**
+ * Removes the port with the given id.
+ *
+ * @return 204 NO_CONTENT, 400 BAD_REQUEST if the port does not exist
+ */
+ @DELETE
+ public Response removeAllPorts() {
+ adminService.ports().stream()
+ .map(K8sPort::portId)
+ .forEach(adminService::removePort);
+ return Response.noContent().build();
+ }
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostService.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostService.java
index cf7b77f..57c2598 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostService.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostService.java
@@ -65,4 +65,12 @@
* @return kubernetes host
*/
K8sHost hostByTunBridge(DeviceId deviceId);
+
+ /**
+ * Returns the host with the specified router bridge device ID.
+ *
+ * @param deviceId router bridge's device ID
+ * @return kubernetes host
+ */
+ K8sHost hostByRouterBridge(DeviceId deviceId);
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeEvent.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeEvent.java
index e9209184..efa1d2f 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeEvent.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeEvent.java
@@ -50,7 +50,12 @@
/**
* Signifies that the node state is incomplete.
*/
- K8S_NODE_INCOMPLETE
+ K8S_NODE_INCOMPLETE,
+
+ /**
+ * Signifies that the node state is off-boarded.
+ */
+ K8S_NODE_OFF_BOARDED
}
/**
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeHandler.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeHandler.java
index 0ef7302..3ad369f 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeHandler.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeHandler.java
@@ -75,4 +75,11 @@
* @param k8sNode kubernetes node
*/
void processPostOnBoardState(K8sNode k8sNode);
+
+ /**
+ * Processes the given node for off boarded state.
+ *
+ * @param k8sNode kubernetes node
+ */
+ void processOffBoardedState(K8sNode k8sNode);
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeState.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeState.java
index 3311ac6..9cc68e2 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeState.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeState.java
@@ -117,6 +117,20 @@
public K8sNodeState nextState() {
return INIT;
}
+ },
+ /**
+ * Indicates node is removed.
+ */
+ OFF_BOARDED {
+ @Override
+ public void process(K8sNodeHandler handler, K8sNode node) {
+ handler.processOffBoardedState(node);
+ }
+
+ @Override
+ public K8sNodeState nextState() {
+ return OFF_BOARDED;
+ }
};
/**
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
index 2fc26be..a3a6fcd 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
@@ -448,6 +448,7 @@
log.error("Exception caused during init state checking...");
}
+ // checks whether all tunneling ports exist
for (K8sTunnelBridge bridge: k8sHost.tunBridges()) {
if (!isTunPortEnabled(bridge, bridge.vxlanPortName())) {
return false;
@@ -460,6 +461,26 @@
}
}
+ // checks whether all patch ports attached to tunnel bridge exist
+ for (K8sTunnelBridge bridge : k8sHost.tunBridges()) {
+ for (String node : k8sHost.nodeNames()) {
+ K8sNode k8sNode = k8sNodeAdminService.node(node);
+ if (!isTunPortEnabled(bridge, k8sNode.tunToIntgPatchPortName())) {
+ return false;
+ }
+ }
+ }
+
+ // checks whether all patch ports attached to router bridge exist
+ for (K8sRouterBridge bridge : k8sHost.routerBridges()) {
+ for (String node : k8sHost.nodeNames()) {
+ K8sNode k8sNode = k8sNodeAdminService.node(node);
+ if (!isRouterPortEnabled(bridge, k8sNode.routerToExtPatchPortName())) {
+ return false;
+ }
+ }
+ }
+
return true;
}
@@ -471,6 +492,14 @@
port.isEnabled());
}
+ private boolean isRouterPortEnabled(K8sRouterBridge routerBridge, String intf) {
+ return deviceService.isAvailable(routerBridge.deviceId()) &&
+ deviceService.getPorts(routerBridge.deviceId()).stream()
+ .anyMatch(port -> Objects.equals(
+ port.annotations().value(PORT_NAME), intf) &&
+ port.isEnabled());
+ }
+
/**
* Configures the kubernetes host with new state.
*
@@ -600,29 +629,30 @@
return;
}
- K8sHost k8sHost = k8sHostAdminService.hostByTunBridge(device.id());
+ K8sHost tunnelHost = k8sHostAdminService.hostByTunBridge(device.id());
- if (k8sHost == null) {
+ if (tunnelHost == null) {
return;
}
- Port port = event.port();
- String portName = port.annotations().value(PORT_NAME);
- if (k8sHost.state() == DEVICE_CREATED) {
-
- K8sTunnelBridge tunBridge = k8sHost.tunBridges().stream().filter(
+ if (tunnelHost.state() == DEVICE_CREATED) {
+ // we bootstrap the host whenever any ports added to the tunnel bridge
+ tunnelHost.tunBridges().stream().filter(
br -> br.deviceId().equals(device.id())
- ).findAny().orElse(null);
+ ).findAny().ifPresent(tunBridge -> bootstrapHost(tunnelHost));
+ }
- if (tunBridge != null) {
- if (Objects.equals(portName, tunBridge.vxlanPortName()) ||
- Objects.equals(portName, tunBridge.grePortName()) ||
- Objects.equals(portName, tunBridge.genevePortName())) {
- log.info("Interface {} added or updated to {}",
- portName, device.id());
- bootstrapHost(k8sHost);
- }
- }
+ K8sHost routerHost = k8sHostAdminService.hostByRouterBridge(device.id());
+
+ if (routerHost == null) {
+ return;
+ }
+
+ if (routerHost.state() == DEVICE_CREATED) {
+ // we bootstrap the host whenever any ports added to the router bridge
+ routerHost.routerBridges().stream().filter(
+ br -> br.deviceId().equals(device.id())
+ ).findAny().ifPresent(routerBridge -> bootstrapHost(routerHost));
}
});
break;
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
index 46f56a6..e5c8b00 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
@@ -86,6 +86,7 @@
import static org.onosproject.net.AnnotationKeys.PORT_NAME;
import static org.slf4j.LoggerFactory.getLogger;
+
/**
* Service bootstraps kubernetes node based on its type.
*/
@@ -261,6 +262,11 @@
// do something if needed
}
+ @Override
+ public void processOffBoardedState(K8sNode k8sNode) {
+ // do something if needed
+ }
+
/**
* Extracts properties from the component configuration context.
*
@@ -630,6 +636,18 @@
return;
}
+ if (k8sNode.mode() == NORMAL) {
+ // delete tunnel bridge from the node
+ client.dropBridge(k8sNode.tunBridgeName());
+ } else {
+ // remove the patch ports direct to the integration bridge from tunnel bridge
+ removeTunnelPatchPort(k8sNode);
+ // remove the patch ports direct to the external bridge from the router bridge
+ removeRouterPatchPort(k8sNode);
+ // remove the patch ports directs to the openstack's br-int bridge from the int and ext bridges
+ removeOpenstackPatchPorts(k8sNode);
+ }
+
// delete integration bridge from the node
client.dropBridge(k8sNode.intgBridgeName());
@@ -639,13 +657,42 @@
// delete local bridge from the node
client.dropBridge(k8sNode.localBridgeName());
- if (k8sNode.mode() == NORMAL) {
- // delete tunnel bridge from the node
- client.dropBridge(k8sNode.tunBridgeName());
+ // disconnect ovsdb
+ // client.disconnect();
+ }
+
+ private void removeTunnelPatchPort(K8sNode k8sNode) {
+ OvsdbClientService client = getOvsdbClient(k8sNode, ovsdbPortNum, ovsdbController);
+ if (client == null) {
+ log.info("Failed to get ovsdb client");
+ return;
}
- // disconnect ovsdb
- client.disconnect();
+ client.dropInterface(k8sNode.tunToIntgPatchPortName());
+ }
+
+ private void removeRouterPatchPort(K8sNode k8sNode) {
+ OvsdbClientService client = getOvsdbClient(k8sNode, ovsdbPortNum, ovsdbController);
+ if (client == null) {
+ log.info("Failed to get ovsdb client");
+ return;
+ }
+
+ client.dropInterface(k8sNode.routerToExtPatchPortName());
+ }
+
+ private void removeOpenstackPatchPorts(K8sNode k8sNode) {
+ OvsdbClientService client = getOvsdbClient(k8sNode, ovsdbPortNum, ovsdbController);
+ if (client == null) {
+ log.info("Failed to get ovsdb client");
+ return;
+ }
+
+ // remove patch port attached at br-int peers with the k8s integration bridge
+ client.dropInterface(k8sNode.osToK8sIntgPatchPortName());
+
+ // remove patch port attached at br-int peers with the k8s external bridge
+ client.dropInterface(k8sNode.osToK8sExtPatchPortName());
}
/**
@@ -831,21 +878,17 @@
case K8S_NODE_CREATED:
case K8S_NODE_UPDATED:
eventExecutor.execute(() -> {
-
if (!isRelevantHelper()) {
return;
}
-
bootstrapNode(event.subject());
});
break;
case K8S_NODE_REMOVED:
eventExecutor.execute(() -> {
-
if (!isRelevantHelper()) {
return;
}
-
processK8sNodeRemoved(event.subject());
});
break;
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sNodeStore.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sNodeStore.java
index 90ec2a0..b221ac4 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sNodeStore.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sNodeStore.java
@@ -51,13 +51,8 @@
import static com.google.common.base.Preconditions.checkArgument;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
-import static org.onosproject.k8snode.api.K8sNodeEvent.Type.K8S_NODE_COMPLETE;
-import static org.onosproject.k8snode.api.K8sNodeEvent.Type.K8S_NODE_CREATED;
-import static org.onosproject.k8snode.api.K8sNodeEvent.Type.K8S_NODE_INCOMPLETE;
-import static org.onosproject.k8snode.api.K8sNodeEvent.Type.K8S_NODE_REMOVED;
-import static org.onosproject.k8snode.api.K8sNodeEvent.Type.K8S_NODE_UPDATED;
-import static org.onosproject.k8snode.api.K8sNodeState.COMPLETE;
-import static org.onosproject.k8snode.api.K8sNodeState.INCOMPLETE;
+import static org.onosproject.k8snode.api.K8sNodeEvent.Type.*;
+import static org.onosproject.k8snode.api.K8sNodeState.*;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -190,6 +185,11 @@
K8S_NODE_INCOMPLETE,
event.newValue().value()
));
+ } else if (event.newValue().value().state() == OFF_BOARDED) {
+ notifyDelegate(new K8sNodeEvent(
+ K8S_NODE_OFF_BOARDED,
+ event.newValue().value()
+ ));
}
});
break;
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sHostManager.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sHostManager.java
index 1747ada..b527dd1 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sHostManager.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sHostManager.java
@@ -206,6 +206,18 @@
return null;
}
+ @Override
+ public K8sHost hostByRouterBridge(DeviceId deviceId) {
+ for (K8sHost host : hostStore.hosts()) {
+ long cnt = host.routerBridges().stream().filter(
+ br -> br.dpid().equals(deviceId.toString())).count();
+ if (cnt > 0) {
+ return host;
+ }
+ }
+ return null;
+ }
+
private class InternalHostStoreDelegate implements K8sHostStoreDelegate {
@Override
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/web/K8sNodeWebResource.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/web/K8sNodeWebResource.java
index 1b20fd4..8eefe14 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/web/K8sNodeWebResource.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/web/K8sNodeWebResource.java
@@ -52,6 +52,7 @@
import static javax.ws.rs.core.Response.created;
import static org.onlab.util.Tools.nullIsIllegal;
import static org.onlab.util.Tools.readTreeFromStream;
+import static org.onosproject.k8snode.api.K8sNodeState.OFF_BOARDED;
import static org.onosproject.k8snode.api.K8sNodeState.POST_ON_BOARD;
import static org.onosproject.k8snode.util.K8sNodeUtil.endpoint;
@@ -177,6 +178,33 @@
return Response.noContent().build();
}
+ /**
+ * Off-board a kubernetes node.
+ *
+ * @param hostname host name contained in kubernetes nodes configuration
+ * @return 200 OK with the updated kubernetes node's config, 400 BAD_REQUEST
+ * if the JSON is malformed, and 304 NOT_MODIFIED without the updated config
+ */
+ @PUT
+ @Path("node/offboard/{hostname}")
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response offboardNode(@PathParam("hostname") String hostname) {
+ log.trace(String.format(MESSAGE_NODE, UPDATE));
+
+ K8sNode existing = nodeAdminService.node(
+ nullIsIllegal(hostname, HOST_NAME + ERROR_MESSAGE));
+
+ if (existing == null) {
+ log.warn("There is no node configuration to offboard : {}", hostname);
+ return Response.notModified().build();
+ } else {
+ K8sNode updated = existing.updateState(OFF_BOARDED);
+ nodeAdminService.updateNode(updated);
+ }
+
+ return Response.ok().build();
+ }
/**
* Obtains the state of the kubernetes node.