Manage k8s API config life cycle, bootstrap k8s node using k8s client
Change-Id: I9926c6d4903da514b66a3bcbe05358c605c9cc1a
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
index b0993bf..9bfd8ea 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sFlowRuleManager.java
@@ -327,19 +327,9 @@
@Override
public void event(K8sNodeEvent event) {
- K8sNode k8sNode = event.subject();
-
switch (event.type()) {
case K8S_NODE_COMPLETE:
- deviceEventExecutor.execute(() -> {
- log.info("COMPLETE node {} is detected", k8sNode.hostname());
-
- if (!isRelevantHelper()) {
- return;
- }
-
- initializePipeline(k8sNode);
- });
+ deviceEventExecutor.execute(() -> processNodeCompletion(event.subject()));
break;
case K8S_NODE_CREATED:
default:
@@ -347,6 +337,17 @@
break;
}
}
+
+ private void processNodeCompletion(K8sNode node) {
+ log.info("COMPLETE node {} is detected", node.hostname());
+
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ initializePipeline(node);
+ k8sNetworkService.networks().forEach(K8sFlowRuleManager.this::setupHostGwRule);
+ }
}
private class InternalK8sNetworkListener implements K8sNetworkListener {
diff --git a/apps/k8s-node/BUILD b/apps/k8s-node/BUILD
index b17a604..c28c480 100644
--- a/apps/k8s-node/BUILD
+++ b/apps/k8s-node/BUILD
@@ -1,6 +1,13 @@
-BUNDLES = [
+BUNDLES = JACKSON + [
"//apps/k8s-node/api:onos-apps-k8s-node-api",
"//apps/k8s-node/app:onos-apps-k8s-node-app",
+ "@k8s_client//jar",
+ "@k8s_model//jar",
+ "@okhttp//jar",
+ "@okio//jar",
+ "@logging_interceptor//jar",
+ "@jackson_dataformat_yaml//jar",
+ "@snakeyaml//jar",
]
onos_app(
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sApiConfig.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sApiConfig.java
index 157763f..48f6b13 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sApiConfig.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sApiConfig.java
@@ -33,17 +33,19 @@
private final Scheme scheme;
private final IpAddress ipAddress;
private final int port;
+ private final State state;
private final String token;
private final String caCertData;
private final String clientCertData;
private final String clientKeyData;
private DefaultK8sApiConfig(Scheme scheme, IpAddress ipAddress, int port,
- String token, String caCertData,
+ State state, String token, String caCertData,
String clientCertData, String clientKeyData) {
this.scheme = scheme;
this.ipAddress = ipAddress;
this.port = port;
+ this.state = state;
this.token = token;
this.caCertData = caCertData;
this.clientCertData = clientCertData;
@@ -66,6 +68,25 @@
}
@Override
+ public State state() {
+ return state;
+ }
+
+ @Override
+ public K8sApiConfig updateState(State newState) {
+ return new Builder()
+ .scheme(scheme)
+ .ipAddress(ipAddress)
+ .port(port)
+ .state(newState)
+ .token(token)
+ .caCertData(caCertData)
+ .clientCertData(clientCertData)
+ .clientKeyData(clientKeyData)
+ .build();
+ }
+
+ @Override
public String token() {
return token;
}
@@ -97,6 +118,7 @@
return port == that.port &&
scheme == that.scheme &&
ipAddress.equals(that.ipAddress) &&
+ state == that.state &&
token.equals(that.token) &&
caCertData.equals(that.caCertData) &&
clientCertData.equals(that.clientCertData) &&
@@ -105,7 +127,7 @@
@Override
public int hashCode() {
- return Objects.hash(scheme, ipAddress, port, token, caCertData,
+ return Objects.hash(scheme, ipAddress, port, state, token, caCertData,
clientCertData, clientKeyData);
}
@@ -115,6 +137,7 @@
.add("scheme", scheme)
.add("ipAddress", ipAddress)
.add("port", port)
+ .add("state", state)
.add("token", token)
.add("caCertData", caCertData)
.add("clientCertData", clientCertData)
@@ -136,6 +159,7 @@
private Scheme scheme;
private IpAddress ipAddress;
private int port;
+ private State state;
private String token;
private String caCertData;
private String clientCertData;
@@ -145,6 +169,7 @@
public K8sApiConfig build() {
checkArgument(scheme != null, NOT_NULL_MSG, "scheme");
checkArgument(ipAddress != null, NOT_NULL_MSG, "ipAddress");
+ checkArgument(state != null, NOT_NULL_MSG, "state");
if (scheme == HTTPS) {
checkArgument(token != null, NOT_NULL_MSG, "token");
@@ -153,7 +178,7 @@
checkArgument(clientKeyData != null, NOT_NULL_MSG, "clientKeyData");
}
- return new DefaultK8sApiConfig(scheme, ipAddress, port, token,
+ return new DefaultK8sApiConfig(scheme, ipAddress, port, state, token,
caCertData, clientCertData, clientKeyData);
}
@@ -176,6 +201,12 @@
}
@Override
+ public Builder state(State state) {
+ this.state = state;
+ return this;
+ }
+
+ @Override
public Builder token(String token) {
this.token = token;
return this;
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
index 76b845c..3e26a50 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/DefaultK8sNode.java
@@ -90,6 +90,18 @@
}
@Override
+ public K8sNode updateIntgBridge(DeviceId deviceId) {
+ return new Builder()
+ .hostname(hostname)
+ .type(type)
+ .intgBridge(deviceId)
+ .managementIp(managementIp)
+ .dataIp(dataIp)
+ .state(state)
+ .build();
+ }
+
+ @Override
public IpAddress managementIp() {
return managementIp;
}
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sApiConfig.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sApiConfig.java
index 0c63733..62d050d 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sApiConfig.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sApiConfig.java
@@ -38,6 +38,21 @@
}
/**
+ * Lists of API server connectivity states.
+ */
+ enum State {
+ /**
+ * Signifies that client is connected to k8s API server.
+ */
+ CONNECTED,
+
+ /**
+ * Signifies that client is dis-connected from k8s API server.
+ */
+ DISCONNECTED,
+ }
+
+ /**
* Returns the authentication scheme.
*
* @return authentication scheme
@@ -59,6 +74,21 @@
int port();
/**
+ * Returns the connectivity state to kubernetes API server.
+ *
+ * @return connectivity state to kubernetes API server
+ */
+ State state();
+
+ /**
+ * Returns new kubernetes API config instance with given state.
+ *
+ * @param newState updated state
+ * @return updated kubernetes API config
+ */
+ K8sApiConfig updateState(State newState);
+
+ /**
* Returns the token used for authenticating to API server.
*
* @return token value
@@ -123,6 +153,14 @@
Builder port(int port);
/**
+ * Returns kubernetes API server config builder with supplied state.
+ *
+ * @param state connectivity state
+ * @return kubernetes API config builder
+ */
+ Builder state(State state);
+
+ /**
* Returns kubernetes API server config builder with supplied token.
*
* @param token token for authentication
diff --git a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
index 4abc2cb..671b37a 100644
--- a/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
+++ b/apps/k8s-node/api/src/main/java/org/onosproject/k8snode/api/K8sNode.java
@@ -68,6 +68,14 @@
DeviceId intgBridge();
/**
+ * Returns new kubernetes node instance with given integration bridge.
+ *
+ * @param deviceId integration bridge device ID
+ * @return updated kubernetes node
+ */
+ K8sNode updateIntgBridge(DeviceId deviceId);
+
+ /**
* Returns the management network IP address of the node.
*
* @return ip address
diff --git a/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sApiConfigTest.java b/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sApiConfigTest.java
index 55e56f1..4abac0f 100644
--- a/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sApiConfigTest.java
+++ b/apps/k8s-node/api/src/test/java/org/onosproject/k8snode/api/DefaultK8sApiConfigTest.java
@@ -25,6 +25,8 @@
import static org.onlab.junit.ImmutableClassChecker.assertThatClassIsImmutable;
import static org.onosproject.k8snode.api.K8sApiConfig.Scheme.HTTP;
import static org.onosproject.k8snode.api.K8sApiConfig.Scheme.HTTPS;
+import static org.onosproject.k8snode.api.K8sApiConfig.State.CONNECTED;
+import static org.onosproject.k8snode.api.K8sApiConfig.State.DISCONNECTED;
/**
* Unit tests for DefaultK8sApiConfig.
@@ -73,6 +75,7 @@
.scheme(SCHEME_1)
.ipAddress(IP_ADDRESS_1)
.port(PORT_1)
+ .state(CONNECTED)
.token(TOKEN_1)
.caCertData(CA_CERT_DATA_1)
.clientCertData(CLIENT_CERT_DATA_1)
@@ -83,6 +86,7 @@
.scheme(SCHEME_1)
.ipAddress(IP_ADDRESS_1)
.port(PORT_1)
+ .state(CONNECTED)
.token(TOKEN_1)
.caCertData(CA_CERT_DATA_1)
.clientCertData(CLIENT_CERT_DATA_1)
@@ -93,6 +97,7 @@
.scheme(SCHEME_2)
.ipAddress(IP_ADDRESS_2)
.port(PORT_2)
+ .state(DISCONNECTED)
.token(TOKEN_2)
.caCertData(CA_CERT_DATA_2)
.clientCertData(CLIENT_CERT_DATA_2)
@@ -120,6 +125,7 @@
assertEquals(SCHEME_1, config.scheme());
assertEquals(IP_ADDRESS_1, config.ipAddress());
assertEquals(PORT_1, config.port());
+ assertEquals(CONNECTED, config.state());
assertEquals(TOKEN_1, config.token());
assertEquals(CA_CERT_DATA_1, config.caCertData());
assertEquals(CLIENT_CERT_DATA_1, config.clientCertData());
diff --git a/apps/k8s-node/app/BUILD b/apps/k8s-node/app/BUILD
index d740f68..ca5b850 100644
--- a/apps/k8s-node/app/BUILD
+++ b/apps/k8s-node/app/BUILD
@@ -3,6 +3,13 @@
"//protocols/ovsdb/api:onos-protocols-ovsdb-api",
"//protocols/ovsdb/rfc:onos-protocols-ovsdb-rfc",
"//apps/k8s-node/api:onos-apps-k8s-node-api",
+ "@k8s_client//jar",
+ "@k8s_model//jar",
+ "@okhttp//jar",
+ "@okio//jar",
+ "@logging_interceptor//jar",
+ "@jackson_dataformat_yaml//jar",
+ "@snakeyaml//jar",
]
TEST_DEPS = TEST_ADAPTERS + TEST_REST + [
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sApiConfigListCommand.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sApiConfigListCommand.java
index 6ede6da..b0b4bc2 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sApiConfigListCommand.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/cli/K8sApiConfigListCommand.java
@@ -37,7 +37,7 @@
description = "Lists all kubernetes API server configs registered to the service")
public class K8sApiConfigListCommand extends AbstractShellCommand {
- private static final String FORMAT = "%-10s%-25s%-10s";
+ private static final String FORMAT = "%-10s%-25s%-10s%-10s";
@Override
protected void doExecute() {
@@ -48,10 +48,10 @@
if (outputJson()) {
print("%s", json(configs));
} else {
- print(FORMAT, "Scheme", "IpAddress", "Port");
+ print(FORMAT, "Scheme", "IpAddress", "Port", "State");
for (K8sApiConfig config : configs) {
- print(FORMAT, config.scheme().name(),
- config.ipAddress().toString(), config.port());
+ print(FORMAT, config.scheme().name(), config.ipAddress().toString(),
+ config.port(), config.state().name());
}
print("Total %s API configs", configService.apiConfigs().size());
}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sApiConfigCodec.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sApiConfigCodec.java
index 5e0cb27..a65f47d 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sApiConfigCodec.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/codec/K8sApiConfigCodec.java
@@ -26,6 +26,7 @@
import static org.onlab.util.Tools.nullIsIllegal;
import static org.onosproject.k8snode.api.K8sApiConfig.Scheme.HTTPS;
+import static org.onosproject.k8snode.api.K8sApiConfig.State.DISCONNECTED;
/**
* Kubernetes API server config codec used for serializing and de-serializing JSON string.
@@ -35,6 +36,7 @@
private static final String SCHEME = "scheme";
private static final String IP_ADDRESS = "ipAddress";
private static final String PORT = "port";
+ private static final String STATE = "state";
private static final String TOKEN = "token";
private static final String CA_CERT_DATA = "caCertData";
private static final String CLIENT_CERT_DATA = "clientCertData";
@@ -47,7 +49,8 @@
ObjectNode node = context.mapper().createObjectNode()
.put(SCHEME, entity.scheme().name())
.put(IP_ADDRESS, entity.ipAddress().toString())
- .put(PORT, entity.port());
+ .put(PORT, entity.port())
+ .put(STATE, entity.state().name());
if (entity.scheme() == HTTPS) {
node.put(TOKEN, entity.token())
@@ -90,7 +93,8 @@
K8sApiConfig.Builder builder = DefaultK8sApiConfig.builder()
.scheme(scheme)
.ipAddress(ipAddress)
- .port(port);
+ .port(port)
+ .state(DISCONNECTED);
JsonNode tokenJson = json.get(TOKEN);
JsonNode caCertDataJson = json.get(CA_CERT_DATA);
@@ -112,7 +116,6 @@
clientKeyData = nullIsIllegal(clientKeyDataJson.asText(),
CLIENT_KEY_DATA + MISSING_MESSAGE);
-
} else {
if (tokenJson != null) {
token = tokenJson.asText();
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
new file mode 100644
index 0000000..7415395
--- /dev/null
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DefaultK8sApiConfigHandler.java
@@ -0,0 +1,202 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snode.impl;
+
+import io.fabric8.kubernetes.api.model.Node;
+import io.fabric8.kubernetes.api.model.NodeAddress;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import org.onlab.packet.IpAddress;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snode.api.DefaultK8sNode;
+import org.onosproject.k8snode.api.K8sApiConfig;
+import org.onosproject.k8snode.api.K8sApiConfigAdminService;
+import org.onosproject.k8snode.api.K8sApiConfigEvent;
+import org.onosproject.k8snode.api.K8sApiConfigListener;
+import org.onosproject.k8snode.api.K8sNode;
+import org.onosproject.k8snode.api.K8sNodeAdminService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snode.api.K8sNode.Type.MASTER;
+import static org.onosproject.k8snode.api.K8sNode.Type.MINION;
+import static org.onosproject.k8snode.api.K8sNodeService.APP_ID;
+import static org.onosproject.k8snode.api.K8sNodeState.INIT;
+import static org.onosproject.k8snode.util.K8sNodeUtil.k8sClient;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Handles the state of kubernetes API server configuration.
+ */
+@Component(immediate = true)
+public class DefaultK8sApiConfigHandler {
+
+ private final Logger log = getLogger(getClass());
+
+ private static final String INTERNAL_IP = "InternalIP";
+ private static final String K8S_ROLE = "node-role.kubernetes.io";
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sApiConfigAdminService k8sApiConfigAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sNodeAdminService k8sNodeAdminService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+
+ private final K8sApiConfigListener k8sApiConfigListener = new InternalK8sApiConfigListener();
+
+ private ApplicationId appId;
+ private NodeId localNode;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.getAppId(APP_ID);
+ localNode = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ k8sApiConfigAdminService.addListener(k8sApiConfigListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ k8sApiConfigAdminService.removeListener(k8sApiConfigListener);
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ /**
+ * Checks the validity of the given kubernetes API server configuration.
+ *
+ * @param config kubernetes API server configuration
+ * @return validity result
+ */
+ private boolean checkApiServerConfig(K8sApiConfig config) {
+ KubernetesClient k8sClient = k8sClient(config);
+ return k8sClient != null && k8sClient.getApiVersion() != null;
+ }
+
+ private void bootstrapK8sNodes(K8sApiConfig config) {
+ KubernetesClient k8sClient = k8sClient(config);
+
+ if (k8sClient == null) {
+ log.warn("Failed to connect to kubernetes API server");
+ return;
+ }
+
+ k8sClient.nodes().list().getItems().forEach(n ->
+ k8sNodeAdminService.createNode(buildK8sNode(n))
+ );
+ }
+
+ private K8sNode buildK8sNode(Node node) {
+ String hostname = node.getMetadata().getName();
+ IpAddress managementIp = null;
+ IpAddress dataIp = null;
+
+ for (NodeAddress nodeAddress:node.getStatus().getAddresses()) {
+ // we need to consider assigning managementIp and dataIp differently
+ // FIXME: ExternalIp is not considered currently
+ if (nodeAddress.getType().equals(INTERNAL_IP)) {
+ managementIp = IpAddress.valueOf(nodeAddress.getAddress());
+ dataIp = IpAddress.valueOf(nodeAddress.getAddress());
+ }
+ }
+
+ String roleStr = node.getMetadata().getLabels().keySet().stream()
+ .filter(l -> l.contains(K8S_ROLE))
+ .findFirst().orElse(null);
+
+ K8sNode.Type nodeType = MASTER;
+
+ if (roleStr != null) {
+ String role = roleStr.split("/")[1];
+ if (MASTER.name().equalsIgnoreCase(role)) {
+ nodeType = MASTER;
+ } else {
+ nodeType = MINION;
+ }
+ }
+
+ return DefaultK8sNode.builder()
+ .hostname(hostname)
+ .managementIp(managementIp)
+ .dataIp(dataIp)
+ .type(nodeType) // need to get correct node type
+ .state(INIT)
+ .build();
+ }
+
+ /**
+ * An internal kubernetes API server config listener.
+ * The notification is triggered by K8sApiConfigStore.
+ */
+ private class InternalK8sApiConfigListener implements K8sApiConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNode, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sApiConfigEvent event) {
+
+ switch (event.type()) {
+ case K8S_API_CONFIG_CREATED:
+ eventExecutor.execute(() -> processConfigCreation(event.subject()));
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void processConfigCreation(K8sApiConfig config) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (checkApiServerConfig(config)) {
+ K8sApiConfig newConfig = config.updateState(K8sApiConfig.State.CONNECTED);
+ k8sApiConfigAdminService.updateApiConfig(newConfig);
+ bootstrapK8sNodes(config);
+ }
+ }
+ }
+}
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sApiConfigStore.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sApiConfigStore.java
index 321aac0..e5a65ab 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sApiConfigStore.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/DistributedK8sApiConfigStore.java
@@ -73,6 +73,7 @@
.register(K8sApiConfig.class)
.register(DefaultK8sApiConfig.class)
.register(K8sApiConfig.Scheme.class)
+ .register(K8sApiConfig.State.class)
.register(Collection.class)
.build();
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
index d80ef21..0289422 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/impl/K8sNodeManager.java
@@ -33,6 +33,7 @@
import org.onosproject.k8snode.api.K8sNodeStoreDelegate;
import org.onosproject.net.DeviceId;
import org.onosproject.net.device.DeviceService;
+import org.onosproject.store.service.AtomicCounter;
import org.onosproject.store.service.StorageService;
import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
@@ -45,6 +46,7 @@
import java.util.Dictionary;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
@@ -56,6 +58,7 @@
import static org.onosproject.k8snode.api.K8sNodeState.COMPLETE;
import static org.onosproject.k8snode.impl.OsgiPropertyConstants.OVSDB_PORT;
import static org.onosproject.k8snode.impl.OsgiPropertyConstants.OVSDB_PORT_NUM_DEFAULT;
+import static org.onosproject.k8snode.util.K8sNodeUtil.genDpid;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -81,6 +84,10 @@
private static final String ERR_NULL_NODE = "Kubernetes node cannot be null";
private static final String ERR_NULL_HOSTNAME = "Kubernetes node hostname cannot be null";
+ private static final String ERR_NULL_DEVICE_ID = "Kubernetes node device ID cannot be null";
+
+ private static final String DEVICE_ID_COUNTER_NAME = "device-id-counter";
+ private static final String NOT_DUPLICATED_MSG = "% cannot be duplicated";
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected K8sNodeStore nodeStore;
@@ -108,6 +115,8 @@
private final K8sNodeStoreDelegate delegate = new K8sNodeManager.InternalNodeStoreDelegate();
+ private AtomicCounter deviceIdCounter;
+
private ApplicationId appId;
@Activate
@@ -117,6 +126,8 @@
leadershipService.runForLeadership(appId.name());
+ deviceIdCounter = storageService.getAtomicCounter(DEVICE_ID_COUNTER_NAME);
+
log.info("Started");
}
@@ -144,15 +155,48 @@
@Override
public void createNode(K8sNode node) {
checkNotNull(node, ERR_NULL_NODE);
- nodeStore.createNode(node);
- log.info(String.format(MSG_NODE, node.hostname(), MSG_CREATED));
+
+ K8sNode updatedNode;
+
+ if (node.intgBridge() == null) {
+ String deviceIdStr = genDpid(deviceIdCounter.incrementAndGet());
+ checkNotNull(deviceIdStr, ERR_NULL_DEVICE_ID);
+ updatedNode = node.updateIntgBridge(DeviceId.deviceId(deviceIdStr));
+ checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
+ NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ } else {
+ updatedNode = node;
+ checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
+ NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ }
+
+ nodeStore.createNode(updatedNode);
+ log.info(String.format(MSG_NODE, updatedNode.hostname(), MSG_CREATED));
}
@Override
public void updateNode(K8sNode node) {
checkNotNull(node, ERR_NULL_NODE);
- nodeStore.updateNode(node);
- log.info(String.format(MSG_NODE, node.hostname(), MSG_UPDATED));
+
+ K8sNode updatedNode;
+
+ K8sNode existingNode = nodeStore.node(node.hostname());
+ checkNotNull(existingNode, ERR_NULL_NODE);
+
+ DeviceId existDeviceId = nodeStore.node(node.hostname()).intgBridge();
+
+ if (node.intgBridge() == null) {
+ updatedNode = node.updateIntgBridge(existDeviceId);
+ checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
+ NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ } else {
+ updatedNode = node;
+ checkArgument(!hasIntgBridge(updatedNode.intgBridge(), updatedNode.hostname()),
+ NOT_DUPLICATED_MSG, updatedNode.intgBridge());
+ }
+
+ nodeStore.updateNode(updatedNode);
+ log.info(String.format(MSG_NODE, updatedNode.hostname(), MSG_UPDATED));
}
@Override
@@ -206,6 +250,15 @@
.findFirst().orElse(null);
}
+ private boolean hasIntgBridge(DeviceId deviceId, String hostname) {
+ Optional<K8sNode> existNode = nodeStore.nodes().stream()
+ .filter(n -> !n.hostname().equals(hostname))
+ .filter(n -> n.intgBridge().equals(deviceId))
+ .findFirst();
+
+ return existNode.isPresent();
+ }
+
private class InternalNodeStoreDelegate implements K8sNodeStoreDelegate {
@Override
diff --git a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/util/K8sNodeUtil.java b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/util/K8sNodeUtil.java
index 7fea9cf..2413f46 100644
--- a/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/util/K8sNodeUtil.java
+++ b/apps/k8s-node/app/src/main/java/org/onosproject/k8snode/util/K8sNodeUtil.java
@@ -17,6 +17,9 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
+import io.fabric8.kubernetes.client.ConfigBuilder;
+import io.fabric8.kubernetes.client.DefaultKubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClient;
import org.apache.commons.lang.StringUtils;
import org.onlab.packet.IpAddress;
import org.onosproject.k8snode.api.K8sApiConfig;
@@ -46,6 +49,10 @@
private static final String COLON_SLASH = "://";
private static final String COLON = ":";
+ private static final int HEX_LENGTH = 16;
+ private static final String OF_PREFIX = "of:";
+ private static final String ZERO = "0";
+
/**
* Prevents object installation from external.
*/
@@ -184,4 +191,52 @@
public static String endpoint(K8sApiConfig apiConfig) {
return endpoint(apiConfig.scheme(), apiConfig.ipAddress(), apiConfig.port());
}
+
+ /**
+ * Generates a DPID (of:0000000000000001) from an index value.
+ *
+ * @param index index value
+ * @return generated DPID
+ */
+ public static String genDpid(long index) {
+ if (index < 0) {
+ return null;
+ }
+
+ String hexStr = Long.toHexString(index);
+
+ StringBuilder zeroPadding = new StringBuilder();
+ for (int i = 0; i < HEX_LENGTH - hexStr.length(); i++) {
+ zeroPadding.append(ZERO);
+ }
+
+ return OF_PREFIX + zeroPadding.toString() + hexStr;
+ }
+
+ /**
+ * Obtains workable kubernetes client.
+ *
+ * @param config kubernetes API config
+ * @return kubernetes client
+ */
+ public static KubernetesClient k8sClient(K8sApiConfig config) {
+ if (config == null) {
+ log.warn("Kubernetes API server config is empty.");
+ return null;
+ }
+
+ String endpoint = endpoint(config);
+
+ ConfigBuilder configBuilder = new ConfigBuilder().withMasterUrl(endpoint);
+
+ if (config.scheme() == K8sApiConfig.Scheme.HTTPS) {
+ configBuilder.withTrustCerts(true)
+ .withOauthToken(config.token())
+ .withCaCertData(config.caCertData())
+ .withClientCertData(config.clientCertData())
+ .withClientKeyData(config.clientKeyData());
+ }
+
+ return new DefaultKubernetesClient(configBuilder.build());
+ }
}
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/codec/K8sApiConfigCodecTest.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/codec/K8sApiConfigCodecTest.java
index ece2ef8..cca918e 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/codec/K8sApiConfigCodecTest.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/codec/K8sApiConfigCodecTest.java
@@ -39,6 +39,7 @@
import static org.easymock.EasyMock.replay;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.notNullValue;
+import static org.onosproject.k8snode.api.K8sApiConfig.State.CONNECTED;
import static org.onosproject.k8snode.codec.K8sApiConfigJsonMatcher.matchesK8sApiConfig;
import static org.onosproject.net.NetTestTools.APP_ID;
@@ -79,6 +80,7 @@
.scheme(K8sApiConfig.Scheme.HTTPS)
.ipAddress(IpAddress.valueOf("10.10.10.23"))
.port(6443)
+ .state(CONNECTED)
.token("token")
.caCertData("caCertData")
.clientCertData("clientCertData")
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/codec/K8sApiConfigJsonMatcher.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/codec/K8sApiConfigJsonMatcher.java
index df1ac2f..2eb42b9 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/codec/K8sApiConfigJsonMatcher.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/codec/K8sApiConfigJsonMatcher.java
@@ -30,6 +30,7 @@
private static final String SCHEME = "scheme";
private static final String IP_ADDRESS = "ipAddress";
private static final String PORT = "port";
+ private static final String STATE = "state";
private static final String TOKEN = "token";
private static final String CA_CERT_DATA = "caCertData";
private static final String CLIENT_CERT_DATA = "clientCertData";
@@ -66,6 +67,16 @@
return false;
}
+ // check state
+ JsonNode jsonState = jsonNode.get(STATE);
+ String state = k8sApiConfig.state().name();
+ if (jsonState != null) {
+ if (!jsonState.asText().equals(state)) {
+ description.appendText("state was " + jsonState);
+ return false;
+ }
+ }
+
// check token
JsonNode jsonToken = jsonNode.get(TOKEN);
String token = k8sApiConfig.token();
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sApiConfigManagerTest.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sApiConfigManagerTest.java
index 90f56c3..f46feb8 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sApiConfigManagerTest.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sApiConfigManagerTest.java
@@ -40,6 +40,7 @@
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.onosproject.k8snode.api.K8sApiConfig.State.DISCONNECTED;
import static org.onosproject.k8snode.api.K8sApiConfigEvent.Type.K8S_API_CONFIG_CREATED;
import static org.onosproject.k8snode.api.K8sApiConfigEvent.Type.K8S_API_CONFIG_REMOVED;
import static org.onosproject.k8snode.util.K8sNodeUtil.endpoint;
@@ -74,11 +75,13 @@
.scheme(K8sApiConfig.Scheme.HTTP)
.ipAddress(IpAddress.valueOf("10.10.10.2"))
.port(6443)
+ .state(DISCONNECTED)
.build();
apiConfig2 = DefaultK8sApiConfig.builder()
.scheme(K8sApiConfig.Scheme.HTTPS)
.ipAddress(IpAddress.valueOf("10.10.10.3"))
.port(6443)
+ .state(DISCONNECTED)
.token("token")
.caCertData("caCertData")
.clientCertData("clientCertData")
@@ -88,6 +91,7 @@
.scheme(K8sApiConfig.Scheme.HTTP)
.ipAddress(IpAddress.valueOf("10.10.10.4"))
.port(8080)
+ .state(DISCONNECTED)
.build();
configStore = new DistributedK8sApiConfigStore();
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
index 066647b..a07d60e 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/impl/K8sNodeManagerTest.java
@@ -231,7 +231,7 @@
/**
* Checks if updating not existing node fails with proper exception.
*/
- @Test(expected = IllegalArgumentException.class)
+ @Test(expected = NullPointerException.class)
public void testUpdateNotExistingNode() {
target.updateNode(MINION_1);
}
diff --git a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java
index 75d4433..90f7d0d 100644
--- a/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java
+++ b/apps/k8s-node/app/src/test/java/org/onosproject/k8snode/web/K8sNodeWebResourceTest.java
@@ -49,6 +49,7 @@
import static org.easymock.EasyMock.verify;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
+import static org.onosproject.k8snode.api.K8sApiConfig.State.DISCONNECTED;
/**
* Unit test for Kubernetes node REST API.
@@ -100,6 +101,7 @@
.scheme(K8sApiConfig.Scheme.HTTPS)
.ipAddress(IpAddress.valueOf("10.134.34.223"))
.port(6443)
+ .state(DISCONNECTED)
.token("tokenMod")
.caCertData("caCertData")
.clientCertData("clientCertData")
diff --git a/lib/deps.json b/lib/deps.json
index bf7c4ead..c51141c 100644
--- a/lib/deps.json
+++ b/lib/deps.json
@@ -180,6 +180,7 @@
"jackson-annotations": "mvn:com.fasterxml.jackson.core:jackson-annotations:2.9.5",
"jackson-core": "mvn:com.fasterxml.jackson.core:jackson-core:2.9.5",
"jackson-databind": "mvn:com.fasterxml.jackson.core:jackson-databind:2.9.5",
+ "jackson-dataformat-yaml": "mvn:com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.5",
// Note: grpc imports "javax_annotation_api" has an http file, so we need to give a different name here
// See WORKSPACE
@@ -302,7 +303,7 @@
"jackson-coreutils": "mvn:com.github.fge:jackson-coreutils:1.6",
"btf": "mvn:com.github.fge:btf:1.2",
"msg-simple": "mvn:com.github.fge:msg-simple:1.1",
- "snakeyaml": "mvn:org.yaml:snakeyaml:1.15",
+ "snakeyaml": "mvn:org.yaml:snakeyaml:1.18",
// Kubernetes related jars
"k8s-client": "mvn:io.fabric8:kubernetes-client:4.1.1",
diff --git a/tools/build/bazel/generate_workspace.bzl b/tools/build/bazel/generate_workspace.bzl
index 791259f..0909948 100644
--- a/tools/build/bazel/generate_workspace.bzl
+++ b/tools/build/bazel/generate_workspace.bzl
@@ -466,6 +466,12 @@
jar_sha256 = "0fb4e079c118e752cc94c15ad22e6782b0dfc5dc09145f4813fb39d82e686047",
licenses = ["notice"],
jar_urls = ["http://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.9.5/jackson-databind-2.9.5.jar"], )
+ if "jackson_dataformat_yaml" not in native.existing_rules():
+ java_import_external(
+ name = "jackson_dataformat_yaml",
+ jar_sha256 = "adc1c495e9c7286bfa1d861ca07e06c8d0980057ad981b48b04f68eb8dcade2c",
+ licenses = ["notice"],
+ jar_urls = ["http://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.9.5/jackson-dataformat-yaml-2.9.5.jar"], )
if "javax_annotation_api_mvn" not in native.existing_rules():
java_import_external(
name = "javax_annotation_api_mvn",
@@ -1153,9 +1159,9 @@
if "snakeyaml" not in native.existing_rules():
java_import_external(
name = "snakeyaml",
- jar_sha256 = "79ea8aac6590f49ee8390c2f17ed9343079e85b44158a097b301dfee42af86ec",
+ jar_sha256 = "81bf4c29d0275dace75fadb5febf5384553422816256023efa83b2b15a9ced60",
licenses = ["notice"],
- jar_urls = ["http://repo1.maven.org/maven2/org/yaml/snakeyaml/1.15/snakeyaml-1.15.jar"], )
+ jar_urls = ["http://repo1.maven.org/maven2/org/yaml/snakeyaml/1.18/snakeyaml-1.18.jar"], )
if "k8s_client" not in native.existing_rules():
java_import_external(
name = "k8s_client",
@@ -1371,6 +1377,7 @@
artifact_map["@jackson_core//:jackson_core"] = "mvn:com.fasterxml.jackson.core:jackson-core:jar:2.9.5"
artifact_map["@jackson_databind//:jackson_databind"] = "mvn:com.fasterxml.jackson.core:jackson-databind:jar:2.9.5"
artifact_map["@javax_annotation_api_mvn//:javax_annotation_api_mvn"] = "mvn:javax.annotation:javax.annotation-api:jar:1.3"
+artifact_map["@jackson_dataformat_yaml//:jackson_dataformat_yaml"] = "mvn:com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:2.9.5"
artifact_map["@javax_inject//:javax_inject"] = "mvn:org.glassfish.hk2.external:javax.inject:jar:2.5.0-b42"
artifact_map["@javax_ws_rs_api//:javax_ws_rs_api"] = "mvn:javax.ws.rs:javax.ws.rs-api:jar:2.1"
artifact_map["@javax_activation//:javax_activation"] = "mvn:javax.activation:activation:jar:NON-OSGI:1.1.1"
@@ -1484,7 +1491,7 @@
artifact_map["@jackson_coreutils//:jackson_coreutils"] = "mvn:com.github.fge:jackson-coreutils:jar:1.6"
artifact_map["@btf//:btf"] = "mvn:com.github.fge:btf:jar:1.2"
artifact_map["@msg_simple//:msg_simple"] = "mvn:com.github.fge:msg-simple:jar:1.1"
-artifact_map["@snakeyaml//:snakeyaml"] = "mvn:org.yaml:snakeyaml:jar:1.15"
+artifact_map["@snakeyaml//:snakeyaml"] = "mvn:org.yaml:snakeyaml:jar:1.18"
artifact_map["@k8s_client//:k8s_client"] = "mvn:io.fabric8:kubernetes-client:jar:NON-OSGI:4.1.1"
artifact_map["@k8s_model//:k8s_model"] = "mvn:io.fabric8:kubernetes-model:jar:4.1.1"
artifact_map["@zjsonpatch//:zjsonpatch"] = "mvn:io.fabric8:zjsonpatch:jar:0.3.0"