Purge the tunnel and router bridge on removing kubernetes host
Change-Id: I8743b1064b29c76fe96bc4dbfa267f425430a96b
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
index f5394dc..5f22da3 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
@@ -332,52 +332,6 @@
applyRule(flowRule, true);
}
- private void purgeAnyRoutingRule(K8sNode localNode) {
- K8sNetwork k8sNetwork = k8sNetworkService.network(localNode.hostname());
- IpPrefix srcIpPrefix = IpPrefix.valueOf(k8sNetwork.gatewayIp(), HOST_PREFIX);
- IpPrefix dstIpPrefix = IpPrefix.valueOf(k8sNetwork.cidr());
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPSrc(srcIpPrefix)
- .matchIPDst(dstIpPrefix);
-
- for (K8sNode node : k8sNodeService.nodes()) {
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
- .setTunnelId(Long.valueOf(k8sNetwork.segmentId()));
-
- if (node.hostname().equals(k8sNetwork.name())) {
- tBuilder.transition(STAT_EGRESS_TABLE);
- } else {
- tBuilder.setOutput(node.intgToTunPortNum());
-
- // install flows into tunnel bridge
- PortNumber portNum = tunnelPortNumByNetId(k8sNetwork.networkId(),
- k8sNetworkService, node);
- TrafficTreatment treatmentToRemote = DefaultTrafficTreatment.builder()
- .extension(buildExtension(
- deviceService,
- node.tunBridge(),
- localNode.dataIp().getIp4Address()),
- node.tunBridge())
- .setTunnelId(Long.valueOf(k8sNetwork.segmentId()))
- .setOutput(portNum)
- .build();
-
- FlowRule remoteFlowRule = DefaultFlowRule.builder()
- .forDevice(node.tunBridge())
- .withSelector(sBuilder.build())
- .withTreatment(treatmentToRemote)
- .withPriority(PRIORITY_CIDR_RULE)
- .fromApp(appId)
- .makePermanent()
- .forTable(TUN_ENTRY_TABLE)
- .build();
- applyRule(remoteFlowRule, false);
- }
- }
- }
-
private void setAnyRoutingRule(IpPrefix srcIpPrefix, MacAddress mac,
K8sNetwork k8sNetwork) {
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
@@ -452,9 +406,6 @@
case K8S_NODE_COMPLETE:
deviceEventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
- case K8S_NODE_OFF_BOARDED:
- deviceEventExecutor.execute(() -> processNodeOffboard(event.subject()));
- break;
case K8S_NODE_CREATED:
default:
// do nothing
@@ -481,16 +432,6 @@
}
}
}
-
- private void processNodeOffboard(K8sNode node) {
- log.info("Offboarded node {} is detected", node.hostname());
-
- if (!isRelevantHelper()) {
- return;
- }
-
- purgeAnyRoutingRule(node);
- }
}
private class InternalK8sNetworkListener implements K8sNetworkListener {
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
index 6d201ba..387494a 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sNodePortHandler.java
@@ -400,9 +400,6 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
- case K8S_NODE_OFF_BOARDED:
- eventExecutor.execute(() -> processNodeOffboard(event.subject()));
- break;
case K8S_NODE_INCOMPLETE:
default:
break;
@@ -435,9 +432,5 @@
setIntgToExtRules(updatedNode, getServiceCidr(), true);
setTunToIntgRules(updatedNode, true);
}
-
- private void processNodeOffboard(K8sNode k8sNode) {
- setTunToIntgRules(k8sNode, false);
- }
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
index 0dca524..5286a2f 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceHandler.java
@@ -983,9 +983,6 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(k8sNode));
break;
- case K8S_NODE_OFF_BOARDED:
- eventExecutor.execute(() -> processNodeOffboard(k8sNode));
- break;
case K8S_NODE_INCOMPLETE:
default:
break;
@@ -1001,15 +998,6 @@
k8sEndpointsService.endpointses().forEach(e -> setEndpointsRules(e, true));
k8sNetworkService.networks().forEach(n -> setupServiceDefaultRule(n, true));
}
-
- private void processNodeOffboard(K8sNode node) {
- if (!isRelevantHelper()) {
- return;
- }
-
- K8sNetwork network = k8sNetworkService.network(node.hostname());
- setupServiceDefaultRule(network, false);
- }
}
private class InternalK8sNetworkListener implements K8sNetworkListener {
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
index 953a75e..fd32586 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingGatewayHandler.java
@@ -227,44 +227,6 @@
}
}
- private void setGatewayTunnelRule(K8sNode node, boolean install) {
-
- K8sNetwork k8sNetwork = k8sNetworkService.network(node.hostname());
-
- TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
- .matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(k8sNetwork.gatewayIp(),
- HOST_PREFIX));
-
- TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
- K8sNode localNode = k8sNodeService.node(k8sNetwork.name());
-
- tBuilder.setOutput(node.intgToTunPortNum());
-
- // install flows into tunnel bridge
- PortNumber portNum = tunnelPortNumByNetId(k8sNetwork.networkId(),
- k8sNetworkService, node);
- TrafficTreatment treatmentToRemote = DefaultTrafficTreatment.builder()
- .extension(buildExtension(
- deviceService,
- node.tunBridge(),
- localNode.dataIp().getIp4Address()),
- node.tunBridge())
- .setTunnelId(Long.valueOf(k8sNetwork.segmentId()))
- .setOutput(portNum)
- .build();
-
- k8sFlowRuleService.setRule(
- appId,
- node.tunBridge(),
- sBuilder.build(),
- treatmentToRemote,
- PRIORITY_GATEWAY_RULE,
- TUN_ENTRY_TABLE,
- install);
- }
-
private void setInterNodeRoutingRules(K8sNode srcNode, boolean install) {
if (srcNode == null) {
return;
@@ -491,9 +453,6 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
- case K8S_NODE_OFF_BOARDED:
- eventExecutor.execute(() -> processNodeOffboard(event.subject()));
- break;
case K8S_NODE_INCOMPLETE:
default:
break;
@@ -513,14 +472,5 @@
setInterNodeRoutingRules(node, true);
}
-
- private void processNodeOffboard(K8sNode node) {
- if (!isRelevantHelper()) {
- return;
- }
-
- setGatewayTunnelRule(node, false);
- setInterNodeRoutingRules(node, false);
- }
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
index 99ccbd9..bec0540 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sSwitchingHandler.java
@@ -497,8 +497,6 @@
case K8S_NODE_COMPLETE:
eventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
- case K8S_NODE_OFF_BOARDED:
- eventExecutor.execute(() -> processNodeOffboard(event.subject()));
default:
break;
}
@@ -513,13 +511,5 @@
setLocalTunnelTagFlowRules(k8sNode, true);
setRulesForTunnelBridge(k8sNode, true);
}
-
- private void processNodeOffboard(K8sNode k8sNode) {
- if (!isRelevantHelper()) {
- return;
- }
-
- setRulesForTunnelBridge(k8sNode, false);
- }
}
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sManagementWebResource.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sManagementWebResource.java
index 1b96e70..bd51767 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sManagementWebResource.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sManagementWebResource.java
@@ -16,6 +16,7 @@
package org.onosproject.k8snetworking.web;
import io.fabric8.kubernetes.client.KubernetesClient;
+import org.onlab.packet.IpAddress;
import org.onlab.util.ItemNotFoundException;
import org.onosproject.k8snetworking.api.K8sEndpointsAdminService;
import org.onosproject.k8snetworking.api.K8sIngressAdminService;
@@ -23,10 +24,13 @@
import org.onosproject.k8snetworking.api.K8sNetworkAdminService;
import org.onosproject.k8snetworking.api.K8sNetworkPolicyAdminService;
import org.onosproject.k8snetworking.api.K8sPodAdminService;
+import org.onosproject.k8snetworking.api.K8sPort;
import org.onosproject.k8snetworking.api.K8sServiceAdminService;
import org.onosproject.k8snetworking.util.K8sNetworkingUtil;
import org.onosproject.k8snode.api.K8sApiConfig;
import org.onosproject.k8snode.api.K8sApiConfigService;
+import org.onosproject.k8snode.api.K8sHost;
+import org.onosproject.k8snode.api.K8sHostAdminService;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNodeAdminService;
import org.onosproject.k8snode.api.K8sNodeState;
@@ -34,11 +38,15 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import java.util.Set;
+import java.util.stream.Collectors;
import static java.lang.Thread.sleep;
import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.syncPortFromPod;
@@ -53,9 +61,13 @@
public class K8sManagementWebResource extends AbstractWebResource {
private final Logger log = LoggerFactory.getLogger(getClass());
+ private static final long MID_SLEEP_MS = 3000; // we wait 3s
private static final long SLEEP_MS = 10000; // we wait 10s
private static final long TIMEOUT_MS = 30000; // we wait 30s
+ private static final String MESSAGE_ALL = "Received all %s request";
+ private static final String REMOVE = "REMOVE";
+
private final K8sApiConfigService configService = get(K8sApiConfigService.class);
private final K8sPodAdminService podAdminService = get(K8sPodAdminService.class);
private final K8sNamespaceAdminService namespaceAdminService =
@@ -70,6 +82,8 @@
get(K8sNetworkAdminService.class);
private final K8sNodeAdminService nodeAdminService =
get(K8sNodeAdminService.class);
+ private final K8sHostAdminService hostAdminService =
+ get(K8sHostAdminService.class);
private final K8sNetworkPolicyAdminService policyAdminService =
get(K8sNetworkPolicyAdminService.class);
@@ -162,6 +176,64 @@
return ok(mapper().createObjectNode()).build();
}
+ /**
+ * Removes all nodes and hosts.
+ *
+ * @return 204 NO_CONTENT, 400 BAD_REQUEST if the JSON is malformed, and
+ * 304 NOT_MODIFIED without the updated config
+ */
+ @DELETE
+ @Path("purge/all")
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response purgeAll() {
+ log.trace(String.format(MESSAGE_ALL, REMOVE));
+
+ Set<String> portIds = networkAdminService.ports().stream().map(K8sPort::portId).collect(Collectors.toSet());
+ portIds.forEach(networkAdminService::removePort);
+
+ try {
+ sleep(MID_SLEEP_MS);
+ } catch (InterruptedException e) {
+ log.error("Exception caused during node synchronization...");
+ }
+
+ Set<String> masters = nodeAdminService.nodes(K8sNode.Type.MASTER).stream()
+ .map(K8sNode::hostname).collect(Collectors.toSet());
+ Set<String> workers = nodeAdminService.nodes(K8sNode.Type.MINION).stream()
+ .map(K8sNode::hostname).collect(Collectors.toSet());
+
+ for (String hostname : workers) {
+ nodeAdminService.removeNode(hostname);
+ try {
+ sleep(MID_SLEEP_MS);
+ } catch (InterruptedException e) {
+ log.error("Exception caused during node synchronization...");
+ }
+ }
+
+ for (String hostname : masters) {
+ nodeAdminService.removeNode(hostname);
+ try {
+ sleep(MID_SLEEP_MS);
+ } catch (InterruptedException e) {
+ log.error("Exception caused during node synchronization...");
+ }
+ }
+
+ Set<IpAddress> allHosts = hostAdminService.hosts().stream().map(K8sHost::hostIp).collect(Collectors.toSet());
+ for (IpAddress hostIp : allHosts) {
+ hostAdminService.removeHost(hostIp);
+ try {
+ sleep(MID_SLEEP_MS);
+ } catch (InterruptedException e) {
+ log.error("Exception caused during node synchronization...");
+ }
+ }
+
+ return Response.noContent().build();
+ }
+
private void syncRulesBase() {
nodeAdminService.completeNodes(MASTER).forEach(this::syncRulesBaseForNode);
nodeAdminService.completeNodes(MINION).forEach(this::syncRulesBaseForNode);
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java
index 88543ec..f636b1a 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/web/K8sPortWebResource.java
@@ -126,17 +126,4 @@
adminService.removePort(id);
return Response.noContent().build();
}
-
- /**
- * Removes the port with the given id.
- *
- * @return 204 NO_CONTENT, 400 BAD_REQUEST if the port does not exist
- */
- @DELETE
- public Response removeAllPorts() {
- adminService.ports().stream()
- .map(K8sPort::portId)
- .forEach(adminService::removePort);
- return Response.noContent().build();
- }
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeHandler.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeHandler.java
index 3ad369f..0ef7302 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeHandler.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeHandler.java
@@ -75,11 +75,4 @@
* @param k8sNode kubernetes node
*/
void processPostOnBoardState(K8sNode k8sNode);
-
- /**
- * Processes the given node for off boarded state.
- *
- * @param k8sNode kubernetes node
- */
- void processOffBoardedState(K8sNode k8sNode);
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeState.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeState.java
index 9cc68e2..3311ac6 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeState.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNodeState.java
@@ -117,20 +117,6 @@
public K8sNodeState nextState() {
return INIT;
}
- },
- /**
- * Indicates node is removed.
- */
- OFF_BOARDED {
- @Override
- public void process(K8sNodeHandler handler, K8sNode node) {
- handler.processOffBoardedState(node);
- }
-
- @Override
- public K8sNodeState nextState() {
- return OFF_BOARDED;
- }
};
/**
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
index a3a6fcd..b5fd93c 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
@@ -530,6 +530,20 @@
}
}
+ private void processHostRemoval(K8sHost k8sHost) {
+ OvsdbClientService client = getOvsdbClient(k8sHost, ovsdbPortNum, ovsdbController);
+ if (client == null) {
+ log.info("Failed to get ovsdb client");
+ return;
+ }
+
+ // delete tunnel bridge from the host
+ k8sHost.tunBridges().forEach(br -> client.dropBridge(br.name()));
+
+ // delete router bridge from the host
+ k8sHost.routerBridges().forEach(br -> client.dropBridge(br.name()));
+ }
+
private class InternalOvsdbListener implements DeviceListener {
@Override
@@ -715,6 +729,14 @@
});
break;
case K8S_HOST_REMOVED:
+ eventExecutor.execute(() -> {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ processHostRemoval(event.subject());
+ });
+ break;
case K8S_HOST_INCOMPLETE:
default:
break;
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
index e5c8b00..7449bc3 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sNodeHandler.java
@@ -262,11 +262,6 @@
// do something if needed
}
- @Override
- public void processOffBoardedState(K8sNode k8sNode) {
- // do something if needed
- }
-
/**
* Extracts properties from the component configuration context.
*
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sNodeStore.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sNodeStore.java
index b221ac4..4557590 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sNodeStore.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sNodeStore.java
@@ -185,11 +185,6 @@
K8S_NODE_INCOMPLETE,
event.newValue().value()
));
- } else if (event.newValue().value().state() == OFF_BOARDED) {
- notifyDelegate(new K8sNodeEvent(
- K8S_NODE_OFF_BOARDED,
- event.newValue().value()
- ));
}
});
break;
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/web/K8sNodeWebResource.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/web/K8sNodeWebResource.java
index 8eefe14..971dd68 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/web/K8sNodeWebResource.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/web/K8sNodeWebResource.java
@@ -52,7 +52,6 @@
import static javax.ws.rs.core.Response.created;
import static org.onlab.util.Tools.nullIsIllegal;
import static org.onlab.util.Tools.readTreeFromStream;
-import static org.onosproject.k8snode.api.K8sNodeState.OFF_BOARDED;
import static org.onosproject.k8snode.api.K8sNodeState.POST_ON_BOARD;
import static org.onosproject.k8snode.util.K8sNodeUtil.endpoint;
@@ -179,34 +178,6 @@
}
/**
- * Off-board a kubernetes node.
- *
- * @param hostname host name contained in kubernetes nodes configuration
- * @return 200 OK with the updated kubernetes node's config, 400 BAD_REQUEST
- * if the JSON is malformed, and 304 NOT_MODIFIED without the updated config
- */
- @PUT
- @Path("node/offboard/{hostname}")
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public Response offboardNode(@PathParam("hostname") String hostname) {
- log.trace(String.format(MESSAGE_NODE, UPDATE));
-
- K8sNode existing = nodeAdminService.node(
- nullIsIllegal(hostname, HOST_NAME + ERROR_MESSAGE));
-
- if (existing == null) {
- log.warn("There is no node configuration to offboard : {}", hostname);
- return Response.notModified().build();
- } else {
- K8sNode updated = existing.updateState(OFF_BOARDED);
- nodeAdminService.updateNode(updated);
- }
-
- return Response.ok().build();
- }
-
- /**
* Obtains the state of the kubernetes node.
*
* @param hostname hostname of the kubernetes