Use host handler to bootstrap tunnel bridge and related ports
Change-Id: I19bb28d86620b9c42c33e0b570ff176b467d71ac
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sHost.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sHost.java
index 1d49757..e961910 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sHost.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sHost.java
@@ -34,6 +34,7 @@
private final IpAddress hostIp;
private final Set<String> nodeNames;
private final K8sHostState state;
+ private final Set<K8sTunnelBridge> tunBridges;
private static final String NOT_NULL_MSG = "Host % cannot be null";
@@ -45,12 +46,14 @@
* @param hostIp host IP address
* @param nodeNames node names
* @param state host state
+ * @param tunBridges a set of tunnel bridges
*/
protected DefaultK8sHost(IpAddress hostIp, Set<String> nodeNames,
- K8sHostState state) {
+ K8sHostState state, Set<K8sTunnelBridge> tunBridges) {
this.hostIp = hostIp;
this.nodeNames = nodeNames;
this.state = state;
+ this.tunBridges = tunBridges;
}
@Override
@@ -74,11 +77,17 @@
}
@Override
+ public Set<K8sTunnelBridge> tunBridges() {
+ return ImmutableSet.copyOf(tunBridges);
+ }
+
+ @Override
public K8sHost updateState(K8sHostState newState) {
return new Builder()
.hostIp(hostIp)
.nodeNames(nodeNames)
.state(newState)
+ .tunBridges(tunBridges)
.build();
}
@@ -88,6 +97,7 @@
.hostIp(hostIp)
.nodeNames(nodeNames)
.state(state)
+ .tunBridges(tunBridges)
.build();
}
@@ -102,7 +112,8 @@
DefaultK8sHost that = (DefaultK8sHost) o;
return Objects.equals(hostIp, that.hostIp) &&
Objects.equals(nodeNames, that.nodeNames) &&
- state == that.state;
+ state == that.state &&
+ Objects.equals(tunBridges, that.tunBridges);
}
@Override
@@ -116,6 +127,7 @@
.add("hostIp", hostIp)
.add("nodeNames", nodeNames)
.add("state", state)
+ .add("tunBridges", tunBridges)
.toString();
}
@@ -133,6 +145,7 @@
private IpAddress hostIp;
private Set<String> nodeNames;
private K8sHostState state;
+ private Set<K8sTunnelBridge> tunBridges;
// private constructor not intended to use from external
private Builder() {
@@ -147,7 +160,11 @@
nodeNames = new HashSet<>();
}
- return new DefaultK8sHost(hostIp, nodeNames, state);
+ if (tunBridges == null) {
+ tunBridges = new HashSet<>();
+ }
+
+ return new DefaultK8sHost(hostIp, nodeNames, state, tunBridges);
}
@Override
@@ -167,5 +184,11 @@
this.state = state;
return this;
}
+
+ @Override
+ public Builder tunBridges(Set<K8sTunnelBridge> tunBridges) {
+ this.tunBridges = tunBridges;
+ return this;
+ }
}
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHost.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHost.java
index c93d4d1..2eeb9c1 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHost.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHost.java
@@ -55,6 +55,13 @@
DeviceId ovsdb();
/**
+ * Returns the set of tunnel bridges belong to the host.
+ *
+ * @return a set of tunnel bridges
+ */
+ Set<K8sTunnelBridge> tunBridges();
+
+ /**
* Returns new kubernetes host instance with given state.
*
* @param newState updated state
@@ -63,7 +70,7 @@
K8sHost updateState(K8sHostState newState);
/**
- * Returns new kuberentes host instance with given node names.
+ * Returns new kubernetes host instance with given node names.
*
* @param nodeNames a set of node names
* @return updated kubernetes host
@@ -105,5 +112,13 @@
* @return kubernetes host builder
*/
Builder state(K8sHostState state);
+
+ /**
+ * Returns kubernetes host builder with supplied tunnel bridges set.
+ *
+ * @param tunBridges tunnel bridges
+ * @return kubernetes host builder
+ */
+ Builder tunBridges(Set<K8sTunnelBridge> tunBridges);
}
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostHandler.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostHandler.java
index 335c89c..84d7d83 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostHandler.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostHandler.java
@@ -29,6 +29,14 @@
void processInitState(K8sHost k8sHost);
/**
+ * Processes the given node for device created state.
+ * It creates required ports on the bridges based on the node type.
+ *
+ * @param k8sHost kubernetes host
+ */
+ void processDeviceCreatedState(K8sHost k8sHost);
+
+ /**
* Processes the given host for complete state.
* It performs post-init jobs for the complete host.
*
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostService.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostService.java
index abdbb74..cf7b77f 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostService.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostService.java
@@ -17,6 +17,7 @@
import org.onlab.packet.IpAddress;
import org.onosproject.event.ListenerService;
+import org.onosproject.net.DeviceId;
import java.util.Set;
@@ -48,4 +49,20 @@
* @return kubernetes host
*/
K8sHost host(IpAddress hostIp);
+
+ /**
+ * Returns the host with the specified device ID.
+ *
+ * @param deviceId device identifier
+ * @return kubernetes host
+ */
+ K8sHost host(DeviceId deviceId);
+
+ /**
+ * Returns the host with the specified tunnel bridge device ID.
+ *
+ * @param deviceId tunnel bridge's device ID
+ * @return kubernetes host
+ */
+ K8sHost hostByTunBridge(DeviceId deviceId);
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostState.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostState.java
index f9a07c3..1455297 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostState.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sHostState.java
@@ -31,6 +31,20 @@
@Override
public K8sHostState nextState() {
+ return DEVICE_CREATED;
+ }
+ },
+ /**
+ * Indicates bridge devices are added according to the host state.
+ */
+ DEVICE_CREATED {
+ @Override
+ public void process(K8sHostHandler handler, K8sHost host) {
+ handler.processDeviceCreatedState(host);
+ }
+
+ @Override
+ public K8sHostState nextState() {
return COMPLETE;
}
},
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sTunnelBridge.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sTunnelBridge.java
new file mode 100644
index 0000000..50d155e
--- /dev/null
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sTunnelBridge.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2020-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snode.api;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Strings;
+import org.onosproject.net.DeviceId;
+
+import java.util.Objects;
+
+import static org.onosproject.k8snode.api.Constants.GENEVE_TUNNEL;
+import static org.onosproject.k8snode.api.Constants.GRE_TUNNEL;
+import static org.onosproject.k8snode.api.Constants.TUNNEL_BRIDGE;
+import static org.onosproject.k8snode.api.Constants.VXLAN_TUNNEL;
+
+/**
+ * K8s tunnel bridge.
+ */
+public class K8sTunnelBridge {
+
+ private static final String OF_PREFIX = "of:";
+
+ private final int tunnelId;
+
+ /**
+ * Default constructor.
+ *
+ * @param tunnelId tunnel identifier
+ */
+ public K8sTunnelBridge(int tunnelId) {
+ this.tunnelId = tunnelId;
+ }
+
+ /**
+ * Returns device identifier.
+ *
+ * @return device identifier
+ */
+ public DeviceId deviceId() {
+ return DeviceId.deviceId(dpid());
+ }
+
+ /**
+ * Returns tunnel ID.
+ *
+ * @return tunnel ID
+ */
+ public int tunnelId() {
+ return tunnelId;
+ }
+
+ /**
+ * Return the datapath identifier.
+ *
+ * @return datapath identifier
+ */
+ public String dpid() {
+ return genDpidFromName(name());
+ }
+
+ /**
+ * Returns bridge name.
+ *
+ * @return bridge name
+ */
+ public String name() {
+ return TUNNEL_BRIDGE + "-" + tunnelId;
+ }
+
+ /**
+ * Returns GRE port name.
+ *
+ * @return GRE port name
+ */
+ public String grePortName() {
+ return GRE_TUNNEL + "-" + tunnelId;
+ }
+
+ /**
+ * Returns VXLAN port name.
+ *
+ * @return VXLAN port name
+ */
+ public String vxlanPortName() {
+ return VXLAN_TUNNEL + "-" + tunnelId;
+ }
+
+ /**
+ * Returns GENEVE port name.
+ *
+ * @return GENEVE port name
+ */
+ public String genevePortName() {
+ return GENEVE_TUNNEL + "-" + tunnelId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ K8sTunnelBridge that = (K8sTunnelBridge) o;
+ return tunnelId == that.tunnelId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tunnelId);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("tunnelId", tunnelId)
+ .toString();
+ }
+
+ private String genDpidFromName(String name) {
+ if (name != null) {
+ String hexString = Integer.toHexString(name.hashCode());
+ return OF_PREFIX + Strings.padStart(hexString, 16, '0');
+ }
+
+ return null;
+ }
+}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sHostListCommand.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sHostListCommand.java
new file mode 100644
index 0000000..fa996ba
--- /dev/null
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sHostListCommand.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snode.cli;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.karaf.shell.api.action.Command;
+import org.apache.karaf.shell.api.action.lifecycle.Service;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.k8snode.api.K8sHost;
+import org.onosproject.k8snode.api.K8sHostService;
+import org.onosproject.k8snode.api.K8sTunnelBridge;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.onosproject.k8snode.util.K8sNodeUtil.genFormatString;
+import static org.onosproject.k8snode.util.K8sNodeUtil.prettyJson;
+
+/**
+ * Lists all host registered to the service.
+ */
+@Service
+@Command(scope = "onos", name = "k8s-hosts",
+ description = "Lists all hosts registered in kubernetes host service")
+public class K8sHostListCommand extends AbstractShellCommand {
+
+ private static final int HOST_IP_LENGTH = 15;
+ private static final int NODES_LENGTH = 40;
+ private static final int TUNBRS_LENGTH = 40;
+ private static final int STATUS_LENGTH = 15;
+
+ @Override
+ protected void doExecute() {
+ K8sHostService hostService = get(K8sHostService.class);
+ List<K8sHost> hosts = Lists.newArrayList(hostService.hosts());
+ hosts.sort(Comparator.comparing(K8sHost::hostIp));
+
+ String format = genFormatString(
+ ImmutableList.of(HOST_IP_LENGTH, NODES_LENGTH, TUNBRS_LENGTH, STATUS_LENGTH));
+
+ if (outputJson()) {
+ print("%s", json(hosts));
+ } else {
+ print(format, "Host IP", "Nodes", "Tunnel Bridges", "State");
+ for (K8sHost host : hosts) {
+ print(format,
+ host.hostIp().toString(),
+ host.nodeNames().toString(),
+ host.tunBridges().stream().map(K8sTunnelBridge::name)
+ .collect(Collectors.toSet()).toString(),
+ host.state().toString());
+ }
+ print("Total %s hosts", hosts.size());
+ }
+ }
+
+ private String json(List<K8sHost> hosts) {
+ ObjectMapper mapper = new ObjectMapper();
+ ArrayNode result = mapper.createArrayNode();
+ for (K8sHost host: hosts) {
+ result.add(jsonForEntity(host, K8sHost.class));
+ }
+ return prettyJson(mapper, result.toString());
+ }
+}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
index 33415a8..c188871 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
@@ -15,6 +15,7 @@
*/
package org.onosproject.k8snode.impl;
+import com.google.common.collect.ImmutableSet;
import io.fabric8.kubernetes.api.model.Node;
import io.fabric8.kubernetes.api.model.NodeAddress;
import io.fabric8.kubernetes.client.KubernetesClient;
@@ -24,6 +25,7 @@
import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
+import org.onosproject.k8snode.api.DefaultK8sHost;
import org.onosproject.k8snode.api.DefaultK8sNode;
import org.onosproject.k8snode.api.ExternalNetworkService;
import org.onosproject.k8snode.api.HostNodesInfo;
@@ -31,8 +33,12 @@
import org.onosproject.k8snode.api.K8sApiConfigAdminService;
import org.onosproject.k8snode.api.K8sApiConfigEvent;
import org.onosproject.k8snode.api.K8sApiConfigListener;
+import org.onosproject.k8snode.api.K8sHost;
+import org.onosproject.k8snode.api.K8sHostAdminService;
+import org.onosproject.k8snode.api.K8sHostState;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.k8snode.api.K8sNodeAdminService;
+import org.onosproject.k8snode.api.K8sTunnelBridge;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
@@ -44,6 +50,7 @@
import java.util.Objects;
import java.util.concurrent.ExecutorService;
+import static java.lang.Thread.sleep;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.k8snode.api.Constants.DEFAULT_CLUSTER_NAME;
@@ -73,6 +80,8 @@
private static final String DEFAULT_GATEWAY_IP = "127.0.0.1";
private static final String DEFAULT_BRIDGE_IP = "127.0.0.1";
+ private static final long SLEEP_MS = 3000; // we wait 3s
+
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected CoreService coreService;
@@ -89,6 +98,9 @@
protected K8sNodeAdminService k8sNodeAdminService;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sHostAdminService k8sHostAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
protected ExternalNetworkService extNetworkService;
private final ExecutorService eventExecutor = newSingleThreadExecutor(
@@ -142,6 +154,32 @@
);
}
+ private void bootstrapK8sHosts(K8sApiConfig config) {
+ KubernetesClient k8sClient = k8sClient(config);
+
+ if (k8sClient == null) {
+ log.warn("Failed to connect to kubernetes API server");
+ return;
+ }
+
+ config.infos().forEach(h -> {
+ k8sHostAdminService.createHost(buildK8sHost(h, config));
+ });
+
+ }
+
+ private K8sHost buildK8sHost(HostNodesInfo hostNodesInfo, K8sApiConfig config) {
+ int segmentId = config.segmentId();
+ K8sTunnelBridge bridge = new K8sTunnelBridge(segmentId);
+
+ return DefaultK8sHost.builder()
+ .hostIp(hostNodesInfo.hostIp())
+ .state(K8sHostState.INIT)
+ .tunBridges(ImmutableSet.of(bridge))
+ .nodeNames(hostNodesInfo.nodes())
+ .build();
+ }
+
private K8sNode buildK8sNode(Node node, K8sApiConfig config) {
String hostname = node.getMetadata().getName();
IpAddress managementIp = null;
@@ -253,6 +291,14 @@
k8sApiConfigAdminService.updateApiConfig(newConfig);
bootstrapK8sNodes(config);
+
+ try {
+ sleep(SLEEP_MS);
+ } catch (InterruptedException e) {
+ log.error("Exception caused during init state checking...");
+ }
+
+ bootstrapK8sHosts(config);
}
}
}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
new file mode 100644
index 0000000..5da9624
--- /dev/null
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sHostHandler.java
@@ -0,0 +1,620 @@
+/*
+ * Copyright 2020-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snode.impl;
+
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snode.api.K8sHost;
+import org.onosproject.k8snode.api.K8sHostAdminService;
+import org.onosproject.k8snode.api.K8sHostEvent;
+import org.onosproject.k8snode.api.K8sHostHandler;
+import org.onosproject.k8snode.api.K8sHostListener;
+import org.onosproject.k8snode.api.K8sHostState;
+import org.onosproject.k8snode.api.K8sNode;
+import org.onosproject.k8snode.api.K8sNodeAdminService;
+import org.onosproject.k8snode.api.K8sTunnelBridge;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.behaviour.BridgeConfig;
+import org.onosproject.net.behaviour.BridgeDescription;
+import org.onosproject.net.behaviour.ControllerInfo;
+import org.onosproject.net.behaviour.DefaultBridgeDescription;
+import org.onosproject.net.behaviour.DefaultPatchDescription;
+import org.onosproject.net.behaviour.DefaultTunnelDescription;
+import org.onosproject.net.behaviour.InterfaceConfig;
+import org.onosproject.net.behaviour.PatchDescription;
+import org.onosproject.net.behaviour.TunnelDescription;
+import org.onosproject.net.behaviour.TunnelEndPoints;
+import org.onosproject.net.behaviour.TunnelKey;
+import org.onosproject.net.device.DeviceAdminService;
+import org.onosproject.net.device.DeviceEvent;
+import org.onosproject.net.device.DeviceListener;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.ovsdb.controller.OvsdbClientService;
+import org.onosproject.ovsdb.controller.OvsdbController;
+import org.onosproject.ovsdb.controller.OvsdbNodeId;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Collectors;
+
+import static java.lang.Thread.sleep;
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.packet.TpPort.tpPort;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snode.api.Constants.GENEVE;
+import static org.onosproject.k8snode.api.Constants.GRE;
+import static org.onosproject.k8snode.api.Constants.VXLAN;
+import static org.onosproject.k8snode.api.K8sHostState.COMPLETE;
+import static org.onosproject.k8snode.api.K8sHostState.DEVICE_CREATED;
+import static org.onosproject.k8snode.api.K8sHostState.INCOMPLETE;
+import static org.onosproject.k8snode.api.K8sHostState.INIT;
+import static org.onosproject.k8snode.api.K8sNodeService.APP_ID;
+import static org.onosproject.net.AnnotationKeys.PORT_NAME;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Service bootstraps kubernetes host.
+ */
+@Component(immediate = true)
+public class DefaultK8sHostHandler implements K8sHostHandler {
+
+ private final Logger log = getLogger(getClass());
+
+ private static final String DEFAULT_OF_PROTO = "tcp";
+ private static final int DEFAULT_OFPORT = 6653;
+ private static final int DPID_BEGIN = 3;
+ private static final long SLEEP_MS = 3000; // we wait 3s
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DeviceAdminService deviceAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected OvsdbController ovsdbController;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sHostAdminService k8sHostAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sNodeAdminService k8sNodeAdminService;
+
+
+ private int ovsdbPortNum = 6640;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+
+ private final DeviceListener ovsdbListener = new InternalOvsdbListener();
+ private final DeviceListener bridgeListener = new InternalBridgeListener();
+ private final K8sHostListener k8sHostListener = new InternalK8sHostListener();
+
+ private ApplicationId appId;
+ private NodeId localNode;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.getAppId(APP_ID);
+ localNode = clusterService.getLocalNode().id();
+
+ leadershipService.runForLeadership(appId.name());
+ deviceService.addListener(ovsdbListener);
+ deviceService.addListener(bridgeListener);
+ k8sHostAdminService.addListener(k8sHostListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ k8sHostAdminService.removeListener(k8sHostListener);
+ deviceService.removeListener(bridgeListener);
+ deviceService.removeListener(ovsdbListener);
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ @Override
+ public void processInitState(K8sHost k8sHost) {
+ if (!isOvsdbConnected(k8sHost, ovsdbPortNum, ovsdbController, deviceService)) {
+ ovsdbController.connect(k8sHost.hostIp(), tpPort(ovsdbPortNum));
+ return;
+ }
+
+ for (K8sTunnelBridge tunBridge : k8sHost.tunBridges()) {
+ if (!deviceService.isAvailable(tunBridge.deviceId())) {
+ createBridge(k8sHost.ovsdb(), tunBridge);
+ }
+ }
+ }
+
+ @Override
+ public void processDeviceCreatedState(K8sHost k8sHost) {
+ try {
+ if (!isOvsdbConnected(k8sHost, ovsdbPortNum, ovsdbController, deviceService)) {
+ ovsdbController.connect(k8sHost.hostIp(), tpPort(ovsdbPortNum));
+ return;
+ }
+
+ // create patch ports into tunnel bridge face to integration bridge
+ for (K8sTunnelBridge bridge : k8sHost.tunBridges()) {
+ for (String node : k8sHost.nodeNames()) {
+ K8sNode k8sNode = k8sNodeAdminService.node(node);
+ if (k8sNode.segmentId() == bridge.tunnelId()) {
+ createPatchInterfaces(k8sHost.ovsdb(), bridge, k8sNode);
+ }
+ }
+ }
+
+ // create tunnel ports
+ for (K8sTunnelBridge bridge : k8sHost.tunBridges()) {
+ if (!isTunPortEnabled(bridge, bridge.vxlanPortName())) {
+ createVxlanTunnelInterface(k8sHost.ovsdb(), bridge);
+ }
+
+ if (!isTunPortEnabled(bridge, bridge.grePortName())) {
+ createGreTunnelInterface(k8sHost.ovsdb(), bridge);
+ }
+
+ if (!isTunPortEnabled(bridge, bridge.genevePortName())) {
+ createGeneveTunnelInterface(k8sHost.ovsdb(), bridge);
+ }
+ }
+ } catch (Exception e) {
+ log.error("Exception occurred because of {}", e);
+ }
+ }
+
+ @Override
+ public void processCompleteState(K8sHost k8sHost) {
+ // do something if needed
+ }
+
+ @Override
+ public void processIncompleteState(K8sHost k8sHost) {
+ // do something if needed
+ }
+
+ private void createBridge(DeviceId ovsdb, K8sTunnelBridge bridge) {
+ Device device = deviceService.getDevice(ovsdb);
+
+ List<ControllerInfo> controllers = clusterService.getNodes().stream()
+ .map(n -> new ControllerInfo(n.ip(), DEFAULT_OFPORT, DEFAULT_OF_PROTO))
+ .collect(Collectors.toList());
+
+ String dpid = bridge.dpid().substring(DPID_BEGIN);
+
+ BridgeDescription.Builder builder = DefaultBridgeDescription.builder()
+ .name(bridge.name())
+ .failMode(BridgeDescription.FailMode.SECURE)
+ .datapathId(dpid)
+ .disableInBand()
+ .controllers(controllers);
+
+ BridgeConfig bridgeConfig = device.as(BridgeConfig.class);
+ bridgeConfig.addBridge(builder.build());
+ }
+
+ private void createPatchInterfaces(DeviceId ovsdb, K8sTunnelBridge bridge, K8sNode k8sNode) {
+ Device device = deviceService.getDevice(ovsdb);
+ if (device == null || !device.is(InterfaceConfig.class)) {
+ log.error("Failed to create patch interface on {}", ovsdb);
+ return;
+ }
+
+ InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
+
+ // tunnel bridge -> integration bridge
+ PatchDescription brTunIntPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(bridge.name())
+ .ifaceName(k8sNode.tunToIntgPatchPortName())
+ .peer(k8sNode.intgToTunPatchPortName())
+ .build();
+
+ ifaceConfig.addPatchMode(k8sNode.tunToIntgPatchPortName(), brTunIntPatchDesc);
+ }
+
+ private void createVxlanTunnelInterface(DeviceId ovsdb, K8sTunnelBridge bridge) {
+ createTunnelInterface(ovsdb, bridge, VXLAN, bridge.vxlanPortName());
+ }
+
+ private void createGreTunnelInterface(DeviceId ovsdb, K8sTunnelBridge bridge) {
+ createTunnelInterface(ovsdb, bridge, GRE, bridge.grePortName());
+ }
+
+ private void createGeneveTunnelInterface(DeviceId ovsdb, K8sTunnelBridge bridge) {
+ createTunnelInterface(ovsdb, bridge, GENEVE, bridge.genevePortName());
+ }
+
+ private void createTunnelInterface(DeviceId ovsdb, K8sTunnelBridge bridge,
+ String type, String intfName) {
+ if (isTunPortEnabled(bridge, intfName)) {
+ return;
+ }
+
+ Device device = deviceService.getDevice(ovsdb);
+ if (device == null || !device.is(InterfaceConfig.class)) {
+ log.error("Failed to create tunnel interface on {}", ovsdb);
+ return;
+ }
+
+ TunnelDescription tunnelDesc = buildTunnelDesc(bridge, type, intfName);
+
+ InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
+ ifaceConfig.addTunnelMode(intfName, tunnelDesc);
+ }
+
+ private TunnelDescription buildTunnelDesc(K8sTunnelBridge bridge, String type, String intfName) {
+ TunnelKey<String> key = new TunnelKey<>(String.valueOf(bridge.tunnelId()));
+
+ if (VXLAN.equals(type) || GRE.equals(type) || GENEVE.equals(type)) {
+ TunnelDescription.Builder tdBuilder =
+ DefaultTunnelDescription.builder()
+ .deviceId(bridge.name())
+ .ifaceName(intfName)
+ .remote(TunnelEndPoints.flowTunnelEndpoint())
+ .key(key);
+
+ switch (type) {
+ case VXLAN:
+ tdBuilder.type(TunnelDescription.Type.VXLAN);
+ break;
+ case GRE:
+ tdBuilder.type(TunnelDescription.Type.GRE);
+ break;
+ case GENEVE:
+ tdBuilder.type(TunnelDescription.Type.GENEVE);
+ break;
+ default:
+ return null;
+ }
+
+ return tdBuilder.build();
+ }
+
+ return null;
+ }
+
+ private boolean isOvsdbConnected(K8sHost host, int ovsdbPort,
+ OvsdbController ovsdbController,
+ DeviceService deviceService) {
+ OvsdbClientService client = getOvsdbClient(host, ovsdbPort, ovsdbController);
+ return deviceService.isAvailable(host.ovsdb()) &&
+ client != null &&
+ client.isConnected();
+ }
+
+ private OvsdbClientService getOvsdbClient(K8sHost host, int ovsdbPort,
+ OvsdbController ovsdbController) {
+ OvsdbNodeId ovsdb = new OvsdbNodeId(host.hostIp(), ovsdbPort);
+ return ovsdbController.getOvsdbClient(ovsdb);
+ }
+
+ private boolean isCurrentStateDone(K8sHost k8sHost) {
+ switch (k8sHost.state()) {
+ case INIT:
+ return isInitStateDone(k8sHost);
+ case DEVICE_CREATED:
+ return isDeviceCreatedStateDone(k8sHost);
+ case COMPLETE:
+ case INCOMPLETE:
+ return false;
+ default:
+ return true;
+ }
+ }
+
+ private boolean isInitStateDone(K8sHost k8sHost) {
+ if (!isOvsdbConnected(k8sHost, ovsdbPortNum,
+ ovsdbController, deviceService)) {
+ return false;
+ }
+
+ try {
+ // we need to wait a while, in case interface and bridge
+ // creation requires some time
+ sleep(SLEEP_MS);
+ } catch (InterruptedException e) {
+ log.error("Exception caused during init state checking...");
+ }
+
+ for (K8sTunnelBridge tunBridge : k8sHost.tunBridges()) {
+ if (!deviceService.isAvailable(tunBridge.deviceId())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private boolean isDeviceCreatedStateDone(K8sHost k8sHost) {
+ try {
+ // we need to wait a while, in case interface and bridge
+ // creation requires some time
+ sleep(SLEEP_MS);
+ } catch (InterruptedException e) {
+ log.error("Exception caused during init state checking...");
+ }
+
+ for (K8sTunnelBridge bridge: k8sHost.tunBridges()) {
+ if (!isTunPortEnabled(bridge, bridge.vxlanPortName())) {
+ return false;
+ }
+ if (!isTunPortEnabled(bridge, bridge.grePortName())) {
+ return false;
+ }
+ if (!isTunPortEnabled(bridge, bridge.genevePortName())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private boolean isTunPortEnabled(K8sTunnelBridge tunBridge, String intf) {
+ return deviceService.isAvailable(tunBridge.deviceId()) &&
+ deviceService.getPorts(tunBridge.deviceId()).stream()
+ .anyMatch(port -> Objects.equals(
+ port.annotations().value(PORT_NAME), intf) &&
+ port.isEnabled());
+ }
+
+ /**
+ * Configures the kubernetes host with new state.
+ *
+ * @param k8sHost kubernetes host
+ * @param newState a new state
+ */
+ private void setState(K8sHost k8sHost, K8sHostState newState) {
+ if (k8sHost.state() == newState) {
+ return;
+ }
+ K8sHost updated = k8sHost.updateState(newState);
+ k8sHostAdminService.updateHost(updated);
+ log.info("Changed {} state: {}", k8sHost.hostIp(), newState);
+ }
+
+ /**
+ * Bootstraps a new kubernetes host.
+ *
+ * @param k8sHost kubernetes host
+ */
+ private void bootstrapHost(K8sHost k8sHost) {
+ if (isCurrentStateDone(k8sHost)) {
+ setState(k8sHost, k8sHost.state().nextState());
+ } else {
+ log.trace("Processing {} state for {}", k8sHost.state(),
+ k8sHost.hostIp());
+ k8sHost.state().process(this, k8sHost);
+ }
+ }
+
+ private class InternalOvsdbListener implements DeviceListener {
+
+ @Override
+ public boolean isRelevant(DeviceEvent event) {
+ return event.subject().type() == Device.Type.CONTROLLER;
+ }
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNode, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(DeviceEvent event) {
+ Device device = event.subject();
+
+ switch (event.type()) {
+ case DEVICE_AVAILABILITY_CHANGED:
+ case DEVICE_ADDED:
+ eventExecutor.execute(() -> {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ K8sHost k8sHost = k8sHostAdminService.host(device.id());
+
+ if (k8sHost == null) {
+ return;
+ }
+
+ if (deviceService.isAvailable(device.id())) {
+ log.debug("OVSDB {} detected", device.id());
+ bootstrapHost(k8sHost);
+ }
+ });
+ break;
+ case PORT_ADDED:
+ case PORT_REMOVED:
+ case DEVICE_REMOVED:
+ default:
+ // do nothing
+ break;
+ }
+ }
+ }
+
+ private class InternalBridgeListener implements DeviceListener {
+
+ @Override
+ public boolean isRelevant(DeviceEvent event) {
+ return event.subject().type() == Device.Type.SWITCH;
+ }
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNode, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(DeviceEvent event) {
+ Device device = event.subject();
+
+ switch (event.type()) {
+ case DEVICE_AVAILABILITY_CHANGED:
+ case DEVICE_ADDED:
+ eventExecutor.execute(() -> {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ K8sHost k8sHost = k8sHostAdminService.hostByTunBridge(device.id());
+
+ if (k8sHost == null) {
+ return;
+ }
+
+ if (deviceService.isAvailable(device.id())) {
+ log.debug("Tunnel bridge created on {}",
+ k8sHost.hostIp());
+ log.debug("OVSDB {} detected", device.id());
+ bootstrapHost(k8sHost);
+ } else if (k8sHost.state() == COMPLETE) {
+ log.info("Device {} disconnected", device.id());
+ setState(k8sHost, INCOMPLETE);
+ }
+
+ if (k8sHost.state() == INCOMPLETE ||
+ k8sHost.state() == DEVICE_CREATED) {
+ log.info("Device {} is reconnected", device.id());
+ k8sHostAdminService.updateHost(
+ k8sHost.updateState(INIT));
+ }
+ });
+ break;
+ case PORT_UPDATED:
+ case PORT_ADDED:
+ eventExecutor.execute(() -> {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ K8sHost k8sHost = k8sHostAdminService.hostByTunBridge(device.id());
+
+ if (k8sHost == null) {
+ return;
+ }
+
+ Port port = event.port();
+ String portName = port.annotations().value(PORT_NAME);
+ if (k8sHost.state() == DEVICE_CREATED) {
+
+ K8sTunnelBridge tunBridge = k8sHost.tunBridges().stream().filter(
+ br -> br.deviceId().equals(device.id())
+ ).findAny().orElse(null);
+
+ if (tunBridge != null) {
+ if (Objects.equals(portName, tunBridge.vxlanPortName()) ||
+ Objects.equals(portName, tunBridge.grePortName()) ||
+ Objects.equals(portName, tunBridge.genevePortName())) {
+ log.info("Interface {} added or updated to {}",
+ portName, device.id());
+ bootstrapHost(k8sHost);
+ }
+ }
+ }
+ });
+ break;
+ case PORT_REMOVED:
+ eventExecutor.execute(() -> {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ K8sHost k8sHost = k8sHostAdminService.hostByTunBridge(device.id());
+
+ if (k8sHost == null) {
+ return;
+ }
+
+ Port port = event.port();
+ String portName = port.annotations().value(PORT_NAME);
+ if (k8sHost.state() == COMPLETE) {
+ K8sTunnelBridge tunBridge = k8sHost.tunBridges().stream().filter(
+ br -> br.deviceId().equals(device.id())
+ ).findAny().orElse(null);
+
+ if (tunBridge != null) {
+ if (Objects.equals(portName, tunBridge.vxlanPortName()) ||
+ Objects.equals(portName, tunBridge.grePortName()) ||
+ Objects.equals(portName, tunBridge.genevePortName())) {
+ log.warn("Interface {} removed from {}",
+ portName, event.subject().id());
+ setState(k8sHost, INCOMPLETE);
+ }
+ }
+ }
+ });
+ break;
+ case DEVICE_REMOVED:
+ default:
+ // do nothing
+ break;
+ }
+ }
+ }
+
+ private class InternalK8sHostListener implements K8sHostListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNode, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sHostEvent event) {
+ switch (event.type()) {
+ case K8S_HOST_CREATED:
+ case K8S_HOST_UPDATED:
+ eventExecutor.execute(() -> {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ bootstrapHost(event.subject());
+ });
+ break;
+ case K8S_HOST_REMOVED:
+ case K8S_HOST_INCOMPLETE:
+ default:
+ break;
+ }
+ }
+ }
+}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sHostStore.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sHostStore.java
index 17b9fde..73dd7a4 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sHostStore.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sHostStore.java
@@ -26,6 +26,7 @@
import org.onosproject.k8snode.api.K8sHostState;
import org.onosproject.k8snode.api.K8sHostStore;
import org.onosproject.k8snode.api.K8sHostStoreDelegate;
+import org.onosproject.k8snode.api.K8sTunnelBridge;
import org.onosproject.store.AbstractStore;
import org.onosproject.store.serializers.KryoNamespaces;
import org.onosproject.store.service.ConsistentMap;
@@ -80,6 +81,7 @@
.register(K8sHost.class)
.register(DefaultK8sHost.class)
.register(K8sHostState.class)
+ .register(K8sTunnelBridge.class)
.register(Collection.class)
.build();
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sHostManager.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sHostManager.java
index fd26763..1747ada 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sHostManager.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sHostManager.java
@@ -30,6 +30,7 @@
import org.onosproject.k8snode.api.K8sHostService;
import org.onosproject.k8snode.api.K8sHostStore;
import org.onosproject.k8snode.api.K8sHostStoreDelegate;
+import org.onosproject.net.DeviceId;
import org.onosproject.net.device.DeviceService;
import org.onosproject.store.service.StorageService;
import org.osgi.service.component.ComponentContext;
@@ -186,6 +187,25 @@
.findFirst().orElse(null);
}
+ @Override
+ public K8sHost host(DeviceId deviceId) {
+ return hostStore.hosts().stream()
+ .filter(host -> Objects.equals(host.ovsdb(), deviceId))
+ .findFirst().orElse(null);
+ }
+
+ @Override
+ public K8sHost hostByTunBridge(DeviceId deviceId) {
+ for (K8sHost host : hostStore.hosts()) {
+ long cnt = host.tunBridges().stream().filter(
+ br -> br.dpid().equals(deviceId.toString())).count();
+ if (cnt > 0) {
+ return host;
+ }
+ }
+ return null;
+ }
+
private class InternalHostStoreDelegate implements K8sHostStoreDelegate {
@Override
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/util/K8sNodeUtil.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/util/K8sNodeUtil.java
index bf76feb..fc05eca 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/util/K8sNodeUtil.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/util/K8sNodeUtil.java
@@ -259,4 +259,19 @@
return new DefaultKubernetesClient(configBuilder.build());
}
+
+ /**
+ * Auto generates DPID from the given name.
+ *
+ * @param name name
+ * @return auto generated DPID
+ */
+ public static String genDpidFromName(String name) {
+ if (name != null) {
+ String hexString = Integer.toHexString(name.hashCode());
+ return OF_PREFIX + Strings.padStart(hexString, 16, '0');
+ }
+
+ return null;
+ }
}