Manage k8s API config life cycle, bootstrap k8s node using k8s client
Change-Id: I9926c6d4903da514b66a3bcbe05358c605c9cc1a
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
new file mode 100644
index 0000000..7415395
--- /dev/null
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
@@ -0,0 +1,202 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snode.impl;
+
+import io.fabric8.kubernetes.api.model.Node;
+import io.fabric8.kubernetes.api.model.NodeAddress;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import org.onlab.packet.IpAddress;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snode.api.DefaultK8sNode;
+import org.onosproject.k8snode.api.K8sApiConfig;
+import org.onosproject.k8snode.api.K8sApiConfigAdminService;
+import org.onosproject.k8snode.api.K8sApiConfigEvent;
+import org.onosproject.k8snode.api.K8sApiConfigListener;
+import org.onosproject.k8snode.api.K8sNode;
+import org.onosproject.k8snode.api.K8sNodeAdminService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snode.api.K8sNode.Type.MASTER;
+import static org.onosproject.k8snode.api.K8sNode.Type.MINION;
+import static org.onosproject.k8snode.api.K8sNodeService.APP_ID;
+import static org.onosproject.k8snode.api.K8sNodeState.INIT;
+import static org.onosproject.k8snode.util.K8sNodeUtil.k8sClient;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Handles the state of kubernetes API server configuration.
+ */
+@Component(immediate = true)
+public class DefaultK8sApiConfigHandler {
+
+ private final Logger log = getLogger(getClass());
+
+ private static final String INTERNAL_IP = "InternalIP";
+ private static final String K8S_ROLE = "node-role.kubernetes.io";
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sApiConfigAdminService k8sApiConfigAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sNodeAdminService k8sNodeAdminService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+
+ private final K8sApiConfigListener k8sApiConfigListener = new InternalK8sApiConfigListener();
+
+ private ApplicationId appId;
+ private NodeId localNode;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.getAppId(APP_ID);
+ localNode = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ k8sApiConfigAdminService.addListener(k8sApiConfigListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ k8sApiConfigAdminService.removeListener(k8sApiConfigListener);
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ /**
+ * Checks the validity of the given kubernetes API server configuration.
+ *
+ * @param config kubernetes API server configuration
+ * @return validity result
+ */
+ private boolean checkApiServerConfig(K8sApiConfig config) {
+ KubernetesClient k8sClient = k8sClient(config);
+ return k8sClient != null && k8sClient.getApiVersion() != null;
+ }
+
+ private void bootstrapK8sNodes(K8sApiConfig config) {
+ KubernetesClient k8sClient = k8sClient(config);
+
+ if (k8sClient == null) {
+ log.warn("Failed to connect to kubernetes API server");
+ return;
+ }
+
+ k8sClient.nodes().list().getItems().forEach(n ->
+ k8sNodeAdminService.createNode(buildK8sNode(n))
+ );
+ }
+
+ private K8sNode buildK8sNode(Node node) {
+ String hostname = node.getMetadata().getName();
+ IpAddress managementIp = null;
+ IpAddress dataIp = null;
+
+ for (NodeAddress nodeAddress:node.getStatus().getAddresses()) {
+ // we need to consider assigning managementIp and dataIp differently
+ // FIXME: ExternalIp is not considered currently
+ if (nodeAddress.getType().equals(INTERNAL_IP)) {
+ managementIp = IpAddress.valueOf(nodeAddress.getAddress());
+ dataIp = IpAddress.valueOf(nodeAddress.getAddress());
+ }
+ }
+
+ String roleStr = node.getMetadata().getLabels().keySet().stream()
+ .filter(l -> l.contains(K8S_ROLE))
+ .findFirst().orElse(null);
+
+ K8sNode.Type nodeType = MASTER;
+
+ if (roleStr != null) {
+ String role = roleStr.split("/")[1];
+ if (MASTER.name().equalsIgnoreCase(role)) {
+ nodeType = MASTER;
+ } else {
+ nodeType = MINION;
+ }
+ }
+
+ return DefaultK8sNode.builder()
+ .hostname(hostname)
+ .managementIp(managementIp)
+ .dataIp(dataIp)
+ .type(nodeType) // need to get correct node type
+ .state(INIT)
+ .build();
+ }
+
+ /**
+ * An internal kubernetes API server config listener.
+ * The notification is triggered by K8sApiConfigStore.
+ */
+ private class InternalK8sApiConfigListener implements K8sApiConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNode, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sApiConfigEvent event) {
+
+ switch (event.type()) {
+ case K8S_API_CONFIG_CREATED:
+ eventExecutor.execute(() -> processConfigCreation(event.subject()));
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void processConfigCreation(K8sApiConfig config) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (checkApiServerConfig(config)) {
+ K8sApiConfig newConfig = config.updateState(K8sApiConfig.State.CONNECTED);
+ k8sApiConfigAdminService.updateApiConfig(newConfig);
+ bootstrapK8sNodes(config);
+ }
+ }
+ }
+}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sApiConfigStore.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sApiConfigStore.java
index 321aac0..e5a65ab 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sApiConfigStore.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sApiConfigStore.java
@@ -73,6 +73,7 @@
.register(K8sApiConfig.class)
.register(DefaultK8sApiConfig.class)
.register(K8sApiConfig.Scheme.class)
+ .register(K8sApiConfig.State.class)
.register(Collection.class)
.build();
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
index d80ef21..0289422 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
@@ -33,6 +33,7 @@
import org.onosproject.k8snode.api.K8sNodeStoreDelegate;
import org.onosproject.net.DeviceId;
import org.onosproject.net.device.DeviceService;
+import org.onosproject.store.service.AtomicCounter;
import org.onosproject.store.service.StorageService;
import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
@@ -45,6 +46,7 @@
import java.util.Dictionary;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
@@ -56,6 +58,7 @@
import static org.onosproject.k8snode.api.K8sNodeState.COMPLETE;
import static org.onosproject.k8snode.impl.OsgiPropertyConstants.OVSDB_PORT;
import static org.onosproject.k8snode.impl.OsgiPropertyConstants.OVSDB_PORT_NUM_DEFAULT;
+import static org.onosproject.k8snode.util.K8sNodeUtil.genDpid;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -81,6 +84,10 @@
private static final String ERR_NULL_NODE = "Kubernetes node cannot be null";
private static final String ERR_NULL_HOSTNAME = "Kubernetes node hostname cannot be null";
+ private static final String ERR_NULL_DEVICE_ID = "Kubernetes node device ID cannot be null";
+
+ private static final String DEVICE_ID_COUNTER_NAME = "device-id-counter";
+ private static final String NOT_DUPLICATED_MSG = "% cannot be duplicated";
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected K8sNodeStore nodeStore;
@@ -108,6 +115,8 @@
private final K8sNodeStoreDelegate delegate = new K8sNodeManager.InternalNodeStoreDelegate();
+ private AtomicCounter deviceIdCounter;
+
private ApplicationId appId;
@Activate
@@ -117,6 +126,8 @@
leadershipService.runForLeadership(appId.name());
+ deviceIdCounter = storageService.getAtomicCounter(DEVICE_ID_COUNTER_NAME);
+
log.info("Started");
}
@@ -144,15 +155,48 @@
@Override
public void createNode(K8sNode node) {
checkNotNull(node, ERR_NULL_NODE);
- nodeStore.createNode(node);
- log.info(String.format(MSG_NODE, node.hostname(), MSG_CREATED));
+
+ K8sNode updatedNode;
+
+ if (node.intgBridge() == null) {
+ String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
+ checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
+ updatedNode = node.updateIntgBridge(DeviceId.deviceId(deviceIdStr));
+ checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
+ NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ } else {
+ updatedNode = node;
+ checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
+ NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ }
+
+ nodeStore.createNode(updatedNode);
+ log.info(String.format(MSG_NODE, updatedNode.hostname(), MSG_CREATED));
}
@Override
public void updateNode(K8sNode node) {
checkNotNull(node, ERR_NULL_NODE);
- nodeStore.updateNode(node);
- log.info(String.format(MSG_NODE, node.hostname(), MSG_UPDATED));
+
+ K8sNode updatedNode;
+
+ K8sNode existingNode = nodeStore.node(node.hostname());
+ checkNotNull(existingNode, ERR_NULL_NODE);
+
+ DeviceId existDeviceId = nodeStore.node(node.hostname()).intgBridge();
+
+ if (node.intgBridge() == null) {
+ updatedNode = node.updateIntgBridge(existDeviceId);
+ checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
+ NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ } else {
+ updatedNode = node;
+ checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
+ NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ }
+
+ nodeStore.updateNode(updatedNode);
+ log.info(String.format(MSG_NODE, updatedNode.hostname(), MSG_UPDATED));
}
@Override
@@ -206,6 +250,15 @@
.findFirst().orElse(null);
}
+ private boolean hasIntgBridge(DeviceId deviceId, String hostname) {
+ Optional<K8sNode> existNode = nodeStore.nodes().stream()
+ .filter(n -> !n.hostname().equals(hostname))
+ .filter(n -> n.intgBridge().equals(deviceId))
+ .findFirst();
+
+ return existNode.isPresent();
+ }
+
private class InternalNodeStoreDelegate implements K8sNodeStoreDelegate {
@Override