Implement kubernetes pod, service and endpoints watcher
Change-Id: I0c364f370de9599b124f0e85f093099e8b4d97f3
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sEndpointsWatcher.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sEndpointsWatcher.java
new file mode 100644
index 0000000..371a34a
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sEndpointsWatcher.java
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.impl;
+
+import io.fabric8.kubernetes.api.model.Endpoints;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClientException;
+import io.fabric8.kubernetes.client.Watcher;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snetworking.api.K8sEndpointsAdminService;
+import org.onosproject.k8snode.api.K8sApiConfig;
+import org.onosproject.k8snode.api.K8sApiConfigService;
+import org.onosproject.mastership.MastershipService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.k8sClient;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Kubernetes endpoints watcher used for feeding endpoints information.
+ */
+@Component(immediate = true)
+public class K8sEndpointsWatcher {
+
+ private final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sEndpointsAdminService k8sEndpointsAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sApiConfigService k8sApiConfigService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+ private final Watcher<Endpoints>
+ internalEndpointsWatcher = new InternalK8sEndpointsWatcher();
+
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+
+ initWatcher();
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private void initWatcher() {
+ K8sApiConfig config =
+ k8sApiConfigService.apiConfigs().stream().findAny().orElse(null);
+ if (config == null) {
+ log.error("Failed to find valid kubernetes API configuration.");
+ return;
+ }
+
+ KubernetesClient client = k8sClient(config);
+
+ if (client == null) {
+ log.error("Failed to connect to kubernetes API server.");
+ return;
+ }
+
+ client.endpoints().watch(internalEndpointsWatcher);
+ }
+
+ private class InternalK8sEndpointsWatcher implements Watcher<Endpoints> {
+
+ @Override
+ public void eventReceived(Action action, Endpoints endpoints) {
+ switch (action) {
+ case ADDED:
+ eventExecutor.execute(() -> processAddition(endpoints));
+ break;
+ case MODIFIED:
+ eventExecutor.execute(() -> processModification(endpoints));
+ break;
+ case DELETED:
+ eventExecutor.execute(() -> processDeletion(endpoints));
+ break;
+ case ERROR:
+ log.warn("Failures processing endpoints manipulation.");
+ break;
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ @Override
+ public void onClose(KubernetesClientException e) {
+ log.info("Endpoints watcher OnClose: {}" + e);
+ }
+
+ private void processAddition(Endpoints endpoints) {
+ if (!isMaster()) {
+ return;
+ }
+
+ log.info("Process endpoints {} creating event from API server.",
+ endpoints.getMetadata().getName());
+
+ k8sEndpointsAdminService.createEndpoints(endpoints);
+ }
+
+ private void processModification(Endpoints endpoints) {
+ if (!isMaster()) {
+ return;
+ }
+
+ log.info("Process endpoints {} updating event from API server.",
+ endpoints.getMetadata().getName());
+
+ k8sEndpointsAdminService.updateEndpoints(endpoints);
+ }
+
+ private void processDeletion(Endpoints endpoints) {
+ if (!isMaster()) {
+ return;
+ }
+
+ log.info("Process endpoints {} removal event from API server.",
+ endpoints.getMetadata().getName());
+
+ k8sEndpointsAdminService.removeEndpoints(endpoints.getMetadata().getUid());
+ }
+
+ private boolean isMaster() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+ }
+}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodWatcher.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodWatcher.java
new file mode 100644
index 0000000..f00de40
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodWatcher.java
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.impl;
+
+import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClientException;
+import io.fabric8.kubernetes.client.Watcher;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snetworking.api.K8sPodAdminService;
+import org.onosproject.k8snode.api.K8sApiConfig;
+import org.onosproject.k8snode.api.K8sApiConfigService;
+import org.onosproject.mastership.MastershipService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.k8sClient;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Kubernetes pod watcher used for feeding pod information.
+ */
+@Component(immediate = true)
+public class K8sPodWatcher {
+
+ private final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sPodAdminService k8sPodAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sApiConfigService k8sApiConfigService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+ private final Watcher<Pod> internalK8sPodWatcher = new InternalK8sPodWatcher();
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+
+ initWatcher();
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private void initWatcher() {
+ K8sApiConfig config =
+ k8sApiConfigService.apiConfigs().stream().findAny().orElse(null);
+ if (config == null) {
+ log.error("Failed to find valid kubernetes API configuration.");
+ return;
+ }
+
+ KubernetesClient client = k8sClient(config);
+
+ if (client == null) {
+ log.error("Failed to connect to kubernetes API server.");
+ return;
+ }
+
+ client.pods().watch(internalK8sPodWatcher);
+ }
+
+ private class InternalK8sPodWatcher implements Watcher<Pod> {
+
+ @Override
+ public void eventReceived(Action action, Pod pod) {
+ switch (action) {
+ case ADDED:
+ eventExecutor.execute(() -> processAddition(pod));
+ break;
+ case MODIFIED:
+ eventExecutor.execute(() -> processModification(pod));
+ break;
+ case DELETED:
+ eventExecutor.execute(() -> processDeletion(pod));
+ break;
+ case ERROR:
+ log.warn("Failures processing pod manipulation.");
+ break;
+ default:
+ break;
+ }
+ }
+
+ @Override
+ public void onClose(KubernetesClientException e) {
+ log.info("Pod watcher OnClose: {}" + e);
+ }
+
+ private void processAddition(Pod pod) {
+ if (!isMaster()) {
+ return;
+ }
+
+ log.info("Process pod {} creating event from API server.",
+ pod.getMetadata().getName());
+
+ k8sPodAdminService.createPod(pod);
+ }
+
+ private void processModification(Pod pod) {
+ if (!isMaster()) {
+ return;
+ }
+
+ log.info("Process pod {} updating event from API server.",
+ pod.getMetadata().getName());
+
+ k8sPodAdminService.updatePod(pod);
+ }
+
+ private void processDeletion(Pod pod) {
+ if (!isMaster()) {
+ return;
+ }
+
+ log.info("Process pod {} removal event from API server.",
+ pod.getMetadata().getName());
+
+ k8sPodAdminService.removePod(pod.getMetadata().getUid());
+ }
+
+ private boolean isMaster() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+ }
+}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceWatcher.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceWatcher.java
new file mode 100644
index 0000000..41e12c2
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceWatcher.java
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.impl;
+
+import io.fabric8.kubernetes.api.model.Service;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClientException;
+import io.fabric8.kubernetes.client.Watcher;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snetworking.api.K8sServiceAdminService;
+import org.onosproject.k8snode.api.K8sApiConfig;
+import org.onosproject.k8snode.api.K8sApiConfigService;
+import org.onosproject.mastership.MastershipService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.k8sClient;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Kubernetes service watcher used for feeding service information.
+ */
+@Component(immediate = true)
+public class K8sServiceWatcher {
+
+ private final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sServiceAdminService k8sServiceAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sApiConfigService k8sApiConfigService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+
+ private final InternalK8sServiceWatcher
+ internalK8sServiceWatcher = new InternalK8sServiceWatcher();
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+
+ initWatcher();
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private void initWatcher() {
+ K8sApiConfig config =
+ k8sApiConfigService.apiConfigs().stream().findAny().orElse(null);
+ if (config == null) {
+ log.error("Failed to find valid kubernetes API configuration.");
+ return;
+ }
+
+ KubernetesClient client = k8sClient(config);
+
+ if (client == null) {
+ log.error("Failed to connect to kubernetes API server.");
+ return;
+ }
+
+ client.services().watch(internalK8sServiceWatcher);
+ }
+
+ private class InternalK8sServiceWatcher implements Watcher<Service> {
+
+ @Override
+ public void eventReceived(Action action, Service service) {
+ switch (action) {
+ case ADDED:
+ eventExecutor.execute(() -> processAddition(service));
+ break;
+ case MODIFIED:
+ eventExecutor.execute(() -> processModification(service));
+ break;
+ case DELETED:
+ eventExecutor.execute(() -> processDeletion(service));
+ break;
+ case ERROR:
+ log.warn("Failures processing service manipulation.");
+ break;
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ @Override
+ public void onClose(KubernetesClientException e) {
+ log.info("Service watcher OnClose: {}" + e);
+ }
+
+ private void processAddition(Service service) {
+ if (!isMaster()) {
+ return;
+ }
+
+ log.info("Process service {} creating event from API server.",
+ service.getMetadata().getName());
+
+ k8sServiceAdminService.createService(service);
+ }
+
+ private void processModification(Service service) {
+ if (!isMaster()) {
+ return;
+ }
+
+ log.info("Process service {} updating event from API server.",
+ service.getMetadata().getName());
+
+ k8sServiceAdminService.updateService(service);
+ }
+
+ private void processDeletion(Service service) {
+ if (!isMaster()) {
+ return;
+ }
+
+ log.info("Process service {} removal event from API server.",
+ service.getMetadata().getName());
+
+ k8sServiceAdminService.removeService(service.getMetadata().getUid());
+ }
+
+ private boolean isMaster() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+ }
+}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java
index 68a8d50..b6c2f69 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java
@@ -19,11 +19,16 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import io.fabric8.kubernetes.client.ConfigBuilder;
+import io.fabric8.kubernetes.client.DefaultKubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.net.util.SubnetUtils;
import org.onlab.packet.IpAddress;
import org.onosproject.cfg.ConfigProperty;
import org.onosproject.k8snetworking.api.K8sNetwork;
import org.onosproject.k8snetworking.api.K8sNetworkService;
+import org.onosproject.k8snode.api.K8sApiConfig;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.net.PortNumber;
import org.slf4j.Logger;
@@ -45,6 +50,9 @@
private static final Logger log = LoggerFactory.getLogger(K8sNetworkingUtil.class);
+ private static final String COLON_SLASH = "://";
+ private static final String COLON = ":";
+
private K8sNetworkingUtil() {
}
@@ -157,4 +165,62 @@
return allAddresses.stream()
.map(IpAddress::valueOf).collect(Collectors.toSet());
}
+
+ /**
+ * Generates endpoint URL by referring to scheme, ipAddress and port.
+ *
+ * @param scheme scheme
+ * @param ipAddress IP address
+ * @param port port number
+ * @return generated endpoint URL
+ */
+ public static String endpoint(K8sApiConfig.Scheme scheme, IpAddress ipAddress, int port) {
+ StringBuilder endpoint = new StringBuilder();
+ String protocol = StringUtils.lowerCase(scheme.name());
+
+ endpoint.append(protocol);
+ endpoint.append(COLON_SLASH);
+ endpoint.append(ipAddress.toString());
+ endpoint.append(COLON);
+ endpoint.append(port);
+
+ return endpoint.toString();
+ }
+
+ /**
+ * Generates endpoint URL by referring to scheme, ipAddress and port.
+ *
+ * @param apiConfig kubernetes API config
+ * @return generated endpoint URL
+ */
+ public static String endpoint(K8sApiConfig apiConfig) {
+ return endpoint(apiConfig.scheme(), apiConfig.ipAddress(), apiConfig.port());
+ }
+
+ /**
+ * Obtains workable kubernetes client.
+ *
+ * @param config kubernetes API config
+ * @return kubernetes client
+ */
+ public static KubernetesClient k8sClient(K8sApiConfig config) {
+ if (config == null) {
+ log.warn("Kubernetes API server config is empty.");
+ return null;
+ }
+
+ String endpoint = endpoint(config);
+
+ ConfigBuilder configBuilder = new ConfigBuilder().withMasterUrl(endpoint);
+
+ if (config.scheme() == K8sApiConfig.Scheme.HTTPS) {
+ configBuilder.withTrustCerts(true)
+ .withOauthToken(config.token())
+ .withCaCertData(config.caCertData())
+ .withClientCertData(config.clientCertData())
+ .withClientKeyData(config.clientKeyData());
+ }
+
+ return new DefaultKubernetesClient(configBuilder.build());
+ }
}