Add a kubernetes port and pod mapper with sync/purge rules CLIs
Change-Id: I89ed29e4357b84345f95fddf81ab7156715d7c82
diff --git a/apps/k8s-networking/BUILD b/apps/k8s-networking/BUILD
index ff83b1c..6ba4011 100644
--- a/apps/k8s-networking/BUILD
+++ b/apps/k8s-networking/BUILD
@@ -9,6 +9,8 @@
"@logging_interceptor//jar",
"@jackson_dataformat_yaml//jar",
"@snakeyaml//jar",
+ "@zjsonpatch//jar",
+ "@validation_api//jar",
]
onos_app(
diff --git a/apps/k8s-networking/app/BUILD b/apps/k8s-networking/app/BUILD
index 515cf0d..725e27d 100644
--- a/apps/k8s-networking/app/BUILD
+++ b/apps/k8s-networking/app/BUILD
@@ -7,6 +7,13 @@
"@commons_net//jar",
"@k8s_client//jar",
"@k8s_model//jar",
+ "@okhttp//jar",
+ "@okio//jar",
+ "@logging_interceptor//jar",
+ "@jackson_dataformat_yaml//jar",
+ "@snakeyaml//jar",
+ "@zjsonpatch//jar",
+ "@validation_api//jar",
]
TEST_DEPS = TEST_ADAPTERS + TEST_REST + [
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sPurgeRulesCommand.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sPurgeRulesCommand.java
new file mode 100644
index 0000000..1658a35
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sPurgeRulesCommand.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.cli;
+
+import org.apache.karaf.shell.api.action.Command;
+import org.apache.karaf.shell.api.action.lifecycle.Service;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.net.flow.FlowRuleService;
+
+import static java.lang.Thread.sleep;
+import static java.util.stream.StreamSupport.stream;
+import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+
+/**
+ * Purges all existing flow rules installed for kubernetes networks.
+ */
+@Service
+@Command(scope = "onos", name = "k8s-purge-rules",
+ description = "Purges all flow rules installed by kubernetes networking app")
+public class K8sPurgeRulesCommand extends AbstractShellCommand {
+
+ private static final long TIMEOUT_MS = 10000; // we wait 10s
+ private static final long SLEEP_MS = 2000; // we wait 2s for init each node
+
+ @Override
+ protected void doExecute() {
+ FlowRuleService flowRuleService = get(FlowRuleService.class);
+ CoreService coreService = get(CoreService.class);
+ ApplicationId appId = coreService.getAppId(K8S_NETWORKING_APP_ID);
+
+ if (appId == null) {
+ error("Failed to purge kubernetes networking flow rules.");
+ return;
+ }
+
+ flowRuleService.removeFlowRulesById(appId);
+ print("Successfully purged flow rules installed by kubernetes networking app.");
+
+ boolean result = true;
+ long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
+
+ // we make sure all flow rules are removed from the store
+ while (stream(flowRuleService.getFlowEntriesById(appId)
+ .spliterator(), false).count() > 0) {
+
+ long waitMs = timeoutExpiredMs - System.currentTimeMillis();
+
+ try {
+ sleep(SLEEP_MS);
+ } catch (InterruptedException e) {
+ log.error("Exception caused during rule purging...");
+ }
+
+ if (stream(flowRuleService.getFlowEntriesById(appId)
+ .spliterator(), false).count() == 0) {
+ break;
+ } else {
+ flowRuleService.removeFlowRulesById(appId);
+ print("Failed to purging flow rules, retrying rule purging...");
+ }
+
+ if (waitMs <= 0) {
+ result = false;
+ break;
+ }
+ }
+
+ if (result) {
+ print("Successfully purged flow rules!");
+ } else {
+ error("Failed to purge flow rules.");
+ }
+ }
+}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sPurgeState.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sPurgeStateCommand.java
similarity index 88%
rename from apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sPurgeState.java
rename to apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sPurgeStateCommand.java
index ccff90a..7b87d7e 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sPurgeState.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sPurgeStateCommand.java
@@ -19,6 +19,7 @@
import org.apache.karaf.shell.api.action.lifecycle.Service;
import org.onosproject.cli.AbstractShellCommand;
import org.onosproject.k8snetworking.api.K8sEndpointsAdminService;
+import org.onosproject.k8snetworking.api.K8sNetworkAdminService;
import org.onosproject.k8snetworking.api.K8sPodAdminService;
import org.onosproject.k8snetworking.api.K8sServiceAdminService;
@@ -28,10 +29,11 @@
@Service
@Command(scope = "onos", name = "k8s-purge-states",
description = "Purges all kubernetes states")
-public class K8sPurgeState extends AbstractShellCommand {
+public class K8sPurgeStateCommand extends AbstractShellCommand {
@Override
protected void doExecute() {
get(K8sPodAdminService.class).clear();
+ get(K8sNetworkAdminService.class).clear();
get(K8sEndpointsAdminService.class).clear();
get(K8sServiceAdminService.class).clear();
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sSyncRulesCommand.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sSyncRulesCommand.java
new file mode 100644
index 0000000..097292c
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sSyncRulesCommand.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.cli;
+
+import org.apache.karaf.shell.api.action.Command;
+import org.apache.karaf.shell.api.action.lifecycle.Service;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.k8snode.api.K8sNode;
+import org.onosproject.k8snode.api.K8sNodeAdminService;
+
+import static java.lang.Thread.sleep;
+import static org.onosproject.k8snode.api.K8sNodeState.COMPLETE;
+import static org.onosproject.k8snode.api.K8sNodeState.INIT;
+
+/**
+ * Synchronizes flow rules to provide connectivity for kubernetes pods.
+ */
+@Service
+@Command(scope = "onos", name = "k8s-sync-rules",
+ description = "Synchronizes all kubernetes flow rules")
+public class K8sSyncRulesCommand extends AbstractShellCommand {
+
+ private static final long SLEEP_MS = 3000; // we wait 3s for init each node
+ private static final long TIMEOUT_MS = 10000; // we wait 10s
+
+ private static final String SUCCESS_MSG = "Successfully synchronize flow rules for node %s!";
+ private static final String FAIL_MSG = "Failed to synchronize flow rules for node %s.";
+
+ @Override
+ protected void doExecute() {
+
+ K8sNodeAdminService adminService = get(K8sNodeAdminService.class);
+ if (adminService == null) {
+ error("Failed to re-install flow rules for kubernetes networking.");
+ return;
+ }
+
+ adminService.completeNodes().forEach(node ->
+ syncRulesBaseForNode(adminService, node));
+
+ print("Successfully requested re-installing flow rules.");
+ }
+
+ private void syncRulesBaseForNode(K8sNodeAdminService adminService,
+ K8sNode k8sNode) {
+ K8sNode updated = k8sNode.updateState(INIT);
+ adminService.updateNode(updated);
+
+ boolean result = true;
+ long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
+
+ while (adminService.node(k8sNode.hostname()).state() != COMPLETE) {
+
+ long waitMs = timeoutExpiredMs - System.currentTimeMillis();
+
+ try {
+ sleep(SLEEP_MS);
+ } catch (InterruptedException e) {
+ error("Exception caused during node synchronization...");
+ }
+
+ if (adminService.node(k8sNode.hostname()).state() == COMPLETE) {
+ break;
+ } else {
+ adminService.updateNode(updated);
+ print("Failed to synchronize flow rules, retrying...");
+ }
+
+ if (waitMs <= 0) {
+ result = false;
+ break;
+ }
+ }
+
+ if (result) {
+ print(SUCCESS_MSG, k8sNode.hostname());
+ } else {
+ error(FAIL_MSG, k8sNode.hostname());
+ }
+ }
+}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sSyncStateCommand.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sSyncStateCommand.java
index 1592755..170425c 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sSyncStateCommand.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/cli/K8sSyncStateCommand.java
@@ -21,15 +21,25 @@
import io.fabric8.kubernetes.client.KubernetesClient;
import org.apache.karaf.shell.api.action.Command;
import org.apache.karaf.shell.api.action.lifecycle.Service;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.k8snetworking.api.DefaultK8sPort;
import org.onosproject.k8snetworking.api.K8sEndpointsAdminService;
+import org.onosproject.k8snetworking.api.K8sNetworkAdminService;
import org.onosproject.k8snetworking.api.K8sPodAdminService;
+import org.onosproject.k8snetworking.api.K8sPort;
import org.onosproject.k8snetworking.api.K8sServiceAdminService;
import org.onosproject.k8snetworking.util.K8sNetworkingUtil;
import org.onosproject.k8snode.api.K8sApiConfig;
import org.onosproject.k8snode.api.K8sApiConfigService;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
import java.util.List;
+import java.util.Map;
+
+import static org.onosproject.k8snetworking.api.K8sPort.State.INACTIVE;
/**
* Synchronizes kubernetes states.
@@ -43,6 +53,13 @@
private static final String SERVICE_FORMAT = "%-50s%-30s%-30s";
private static final String ENDPOINTS_FORMAT = "%-50s%-50s%-20s";
+ private static final String PORT_ID = "portId";
+ private static final String DEVICE_ID = "deviceId";
+ private static final String PORT_NUMBER = "portNumber";
+ private static final String IP_ADDRESS = "ipAddress";
+ private static final String MAC_ADDRESS = "macAddress";
+ private static final String NETWORK_ID = "networkId";
+
@Override
protected void doExecute() {
K8sApiConfigService configService = get(K8sApiConfigService.class);
@@ -51,6 +68,8 @@
get(K8sServiceAdminService.class);
K8sEndpointsAdminService endpointsAdminService =
get(K8sEndpointsAdminService.class);
+ K8sNetworkAdminService networkAdminService =
+ get(K8sNetworkAdminService.class);
K8sApiConfig config =
configService.apiConfigs().stream().findAny().orElse(null);
@@ -96,6 +115,9 @@
} else {
podAdminService.createPod(pod);
}
+
+ syncPortFromPod(pod, networkAdminService);
+
printPod(pod);
});
}
@@ -139,4 +161,36 @@
pod.getStatus().getPodIP(),
containers.isEmpty() ? "" : containers);
}
+
+ private void syncPortFromPod(Pod pod, K8sNetworkAdminService adminService) {
+ Map<String, String> annotations = pod.getMetadata().getAnnotations();
+ if (annotations != null && !annotations.isEmpty() &&
+ annotations.get(PORT_ID) != null) {
+ String portId = annotations.get(PORT_ID);
+
+ K8sPort oldPort = adminService.port(portId);
+
+ String networkId = annotations.get(NETWORK_ID);
+ DeviceId deviceId = DeviceId.deviceId(annotations.get(DEVICE_ID));
+ PortNumber portNumber = PortNumber.portNumber(annotations.get(PORT_NUMBER));
+ IpAddress ipAddress = IpAddress.valueOf(annotations.get(IP_ADDRESS));
+ MacAddress macAddress = MacAddress.valueOf(annotations.get(MAC_ADDRESS));
+
+ K8sPort newPort = DefaultK8sPort.builder()
+ .portId(portId)
+ .networkId(networkId)
+ .deviceId(deviceId)
+ .ipAddress(ipAddress)
+ .macAddress(macAddress)
+ .portNumber(portNumber)
+ .state(INACTIVE)
+ .build();
+
+ if (oldPort == null) {
+ adminService.createPort(newPort);
+ } else {
+ adminService.updatePort(newPort);
+ }
+ }
+ }
}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sEndpointsWatcher.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sEndpointsWatcher.java
index 371a34a..683b070 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sEndpointsWatcher.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sEndpointsWatcher.java
@@ -25,7 +25,8 @@
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.k8snetworking.api.K8sEndpointsAdminService;
-import org.onosproject.k8snode.api.K8sApiConfig;
+import org.onosproject.k8snode.api.K8sApiConfigEvent;
+import org.onosproject.k8snode.api.K8sApiConfigListener;
import org.onosproject.k8snode.api.K8sApiConfigService;
import org.onosproject.mastership.MastershipService;
import org.osgi.service.component.annotations.Activate;
@@ -74,7 +75,8 @@
groupedThreads(this.getClass().getSimpleName(), "event-handler"));
private final Watcher<Endpoints>
internalEndpointsWatcher = new InternalK8sEndpointsWatcher();
-
+ private final InternalK8sApiConfigListener
+ internalK8sApiConfigListener = new InternalK8sApiConfigListener();
private ApplicationId appId;
private NodeId localNodeId;
@@ -84,36 +86,52 @@
appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
localNodeId = clusterService.getLocalNode().id();
leadershipService.runForLeadership(appId.name());
-
- initWatcher();
+ k8sApiConfigService.addListener(internalK8sApiConfigListener);
log.info("Started");
}
@Deactivate
protected void deactivate() {
+ k8sApiConfigService.removeListener(internalK8sApiConfigListener);
leadershipService.withdraw(appId.name());
eventExecutor.shutdown();
log.info("Stopped");
}
- private void initWatcher() {
- K8sApiConfig config =
- k8sApiConfigService.apiConfigs().stream().findAny().orElse(null);
- if (config == null) {
- log.error("Failed to find valid kubernetes API configuration.");
- return;
+ private class InternalK8sApiConfigListener implements K8sApiConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
}
- KubernetesClient client = k8sClient(config);
+ @Override
+ public void event(K8sApiConfigEvent event) {
- if (client == null) {
- log.error("Failed to connect to kubernetes API server.");
- return;
+ switch (event.type()) {
+ case K8S_API_CONFIG_UPDATED:
+ eventExecutor.execute(this::processConfigUpdating);
+ break;
+ case K8S_API_CONFIG_CREATED:
+ case K8S_API_CONFIG_REMOVED:
+ default:
+ // do nothing
+ break;
+ }
}
- client.endpoints().watch(internalEndpointsWatcher);
+ private void processConfigUpdating() {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ KubernetesClient client = k8sClient(k8sApiConfigService);
+
+ if (client != null) {
+ client.endpoints().watch(internalEndpointsWatcher);
+ }
+ }
}
private class InternalK8sEndpointsWatcher implements Watcher<Endpoints> {
@@ -125,7 +143,9 @@
eventExecutor.execute(() -> processAddition(endpoints));
break;
case MODIFIED:
- eventExecutor.execute(() -> processModification(endpoints));
+ // FIXME: there are too frequent endpoints update events
+ // issued from kubernetes API server, we disable update for now
+ // eventExecutor.execute(() -> processModification(endpoints));
break;
case DELETED:
eventExecutor.execute(() -> processDeletion(endpoints));
@@ -149,7 +169,7 @@
return;
}
- log.info("Process endpoints {} creating event from API server.",
+ log.trace("Process endpoints {} creating event from API server.",
endpoints.getMetadata().getName());
k8sEndpointsAdminService.createEndpoints(endpoints);
@@ -160,10 +180,13 @@
return;
}
- log.info("Process endpoints {} updating event from API server.",
+ log.trace("Process endpoints {} updating event from API server.",
endpoints.getMetadata().getName());
- k8sEndpointsAdminService.updateEndpoints(endpoints);
+ if (k8sEndpointsAdminService.endpoints(
+ endpoints.getMetadata().getUid()) != null) {
+ k8sEndpointsAdminService.updateEndpoints(endpoints);
+ }
}
private void processDeletion(Endpoints endpoints) {
@@ -171,7 +194,7 @@
return;
}
- log.info("Process endpoints {} removal event from API server.",
+ log.trace("Process endpoints {} removal event from API server.",
endpoints.getMetadata().getName());
k8sEndpointsAdminService.removeEndpoints(endpoints.getMetadata().getUid());
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodPortMapper.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodPortMapper.java
new file mode 100644
index 0000000..5239ee4
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodPortMapper.java
@@ -0,0 +1,204 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.impl;
+
+import com.google.common.collect.Maps;
+import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snetworking.api.K8sNetworkAdminService;
+import org.onosproject.k8snetworking.api.K8sNetworkEvent;
+import org.onosproject.k8snetworking.api.K8sNetworkListener;
+import org.onosproject.k8snetworking.api.K8sPodAdminService;
+import org.onosproject.k8snetworking.api.K8sPodEvent;
+import org.onosproject.k8snetworking.api.K8sPodListener;
+import org.onosproject.k8snetworking.api.K8sPort;
+import org.onosproject.k8snode.api.K8sApiConfigService;
+import org.onosproject.mastership.MastershipService;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.k8sClient;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Associates the kubernetes container port and pod.
+ */
+@Component(immediate = true)
+public class K8sPodPortMapper {
+
+ private final Logger log = getLogger(getClass());
+
+ private static final String PORT_ID = "portId";
+ private static final String DEVICE_ID = "deviceId";
+ private static final String PORT_NUMBER = "portNumber";
+ private static final String IP_ADDRESS = "ipAddress";
+ private static final String MAC_ADDRESS = "macAddress";
+ private static final String NETWORK_ID = "networkId";
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DriverService driverService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sNetworkAdminService k8sNetworkService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sPodAdminService k8sPodService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sApiConfigService k8sApiConfigService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+ private final InternalK8sNetworkListener k8sNetworkListener =
+ new InternalK8sNetworkListener();
+ private final InternalK8sPodListener k8sPodListener =
+ new InternalK8sPodListener();
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ k8sNetworkService.addListener(k8sNetworkListener);
+ k8sPodService.addListener(k8sPodListener);
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ k8sNetworkService.removeListener(k8sNetworkListener);
+ k8sPodService.removeListener(k8sPodListener);
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private class InternalK8sPodListener implements K8sPodListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sPodEvent event) {
+ switch (event.type()) {
+ case K8S_POD_CREATED:
+ case K8S_POD_UPDATED:
+ eventExecutor.execute(() -> processPodCreation(event.subject()));
+ break;
+ case K8S_POD_REMOVED:
+ default:
+ break;
+ }
+ }
+
+ private void processPodCreation(Pod pod) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ KubernetesClient client = k8sClient(k8sApiConfigService);
+
+ if (client == null) {
+ return;
+ }
+
+ // if the annotations were configured, we will not update it
+ if (pod.getMetadata().getAnnotations() != null) {
+ return;
+ }
+
+ Set<K8sPort> ports = k8sNetworkService.ports();
+
+ // TODO: we assume that POD IP is unique, there might be other
+ // variable which preserves better uniqueness
+ ports.stream().filter(p -> p.ipAddress().toString()
+ .equals(pod.getStatus().getPodIP()))
+ .forEach(p -> {
+ Map<String, String> annotations = Maps.newConcurrentMap();
+ annotations.put(PORT_ID, p.portId());
+ annotations.put(NETWORK_ID, p.networkId());
+ annotations.put(DEVICE_ID, p.deviceId().toString());
+ annotations.put(PORT_NUMBER, p.portNumber().toString());
+ annotations.put(IP_ADDRESS, p.ipAddress().toString());
+ annotations.put(MAC_ADDRESS, p.macAddress().toString());
+
+ client.pods().inNamespace(pod.getMetadata().getNamespace())
+ .withName(pod.getMetadata().getName())
+ .edit()
+ .editMetadata()
+ .addToAnnotations(annotations)
+ .endMetadata().done();
+ });
+ }
+ }
+
+ private class InternalK8sNetworkListener implements K8sNetworkListener {
+
+ private boolean isRelevantHelper(K8sNetworkEvent event) {
+ return mastershipService.isLocalMaster(event.port().deviceId());
+ }
+
+ @Override
+ public void event(K8sNetworkEvent event) {
+ switch (event.type()) {
+ case K8S_PORT_UPDATED:
+ case K8S_PORT_REMOVED:
+ // no need to process port removal event...
+ default:
+ break;
+ }
+ }
+ }
+}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodWatcher.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodWatcher.java
index f00de40..a5c15f4 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodWatcher.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sPodWatcher.java
@@ -25,7 +25,8 @@
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.k8snetworking.api.K8sPodAdminService;
-import org.onosproject.k8snode.api.K8sApiConfig;
+import org.onosproject.k8snode.api.K8sApiConfigEvent;
+import org.onosproject.k8snode.api.K8sApiConfigListener;
import org.onosproject.k8snode.api.K8sApiConfigService;
import org.onosproject.mastership.MastershipService;
import org.osgi.service.component.annotations.Activate;
@@ -73,6 +74,8 @@
private final ExecutorService eventExecutor = newSingleThreadExecutor(
groupedThreads(this.getClass().getSimpleName(), "event-handler"));
private final Watcher<Pod> internalK8sPodWatcher = new InternalK8sPodWatcher();
+ private final InternalK8sApiConfigListener
+ internalK8sApiConfigListener = new InternalK8sApiConfigListener();
private ApplicationId appId;
private NodeId localNodeId;
@@ -82,36 +85,52 @@
appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
localNodeId = clusterService.getLocalNode().id();
leadershipService.runForLeadership(appId.name());
-
- initWatcher();
+ k8sApiConfigService.addListener(internalK8sApiConfigListener);
log.info("Started");
}
@Deactivate
protected void deactivate() {
+ k8sApiConfigService.removeListener(internalK8sApiConfigListener);
leadershipService.withdraw(appId.name());
eventExecutor.shutdown();
log.info("Stopped");
}
- private void initWatcher() {
- K8sApiConfig config =
- k8sApiConfigService.apiConfigs().stream().findAny().orElse(null);
- if (config == null) {
- log.error("Failed to find valid kubernetes API configuration.");
- return;
+ private class InternalK8sApiConfigListener implements K8sApiConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
}
- KubernetesClient client = k8sClient(config);
+ @Override
+ public void event(K8sApiConfigEvent event) {
- if (client == null) {
- log.error("Failed to connect to kubernetes API server.");
- return;
+ switch (event.type()) {
+ case K8S_API_CONFIG_UPDATED:
+ eventExecutor.execute(this::processConfigUpdating);
+ break;
+ case K8S_API_CONFIG_CREATED:
+ case K8S_API_CONFIG_REMOVED:
+ default:
+ // do nothing
+ break;
+ }
}
- client.pods().watch(internalK8sPodWatcher);
+ private void processConfigUpdating() {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ KubernetesClient client = k8sClient(k8sApiConfigService);
+
+ if (client != null) {
+ client.pods().watch(internalK8sPodWatcher);
+ }
+ }
}
private class InternalK8sPodWatcher implements Watcher<Pod> {
@@ -146,7 +165,7 @@
return;
}
- log.info("Process pod {} creating event from API server.",
+ log.trace("Process pod {} creating event from API server.",
pod.getMetadata().getName());
k8sPodAdminService.createPod(pod);
@@ -157,10 +176,12 @@
return;
}
- log.info("Process pod {} updating event from API server.",
+ log.trace("Process pod {} updating event from API server.",
pod.getMetadata().getName());
- k8sPodAdminService.updatePod(pod);
+ if (k8sPodAdminService.pod(pod.getMetadata().getUid()) != null) {
+ k8sPodAdminService.updatePod(pod);
+ }
}
private void processDeletion(Pod pod) {
@@ -168,7 +189,7 @@
return;
}
- log.info("Process pod {} removal event from API server.",
+ log.trace("Process pod {} removal event from API server.",
pod.getMetadata().getName());
k8sPodAdminService.removePod(pod.getMetadata().getUid());
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceWatcher.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceWatcher.java
index 41e12c2..17f9946 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceWatcher.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sServiceWatcher.java
@@ -25,7 +25,8 @@
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.k8snetworking.api.K8sServiceAdminService;
-import org.onosproject.k8snode.api.K8sApiConfig;
+import org.onosproject.k8snode.api.K8sApiConfigEvent;
+import org.onosproject.k8snode.api.K8sApiConfigListener;
import org.onosproject.k8snode.api.K8sApiConfigService;
import org.onosproject.mastership.MastershipService;
import org.osgi.service.component.annotations.Activate;
@@ -75,6 +76,8 @@
private final InternalK8sServiceWatcher
internalK8sServiceWatcher = new InternalK8sServiceWatcher();
+ private final InternalK8sApiConfigListener
+ internalK8sApiConfigListener = new InternalK8sApiConfigListener();
private ApplicationId appId;
private NodeId localNodeId;
@@ -84,36 +87,52 @@
appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
localNodeId = clusterService.getLocalNode().id();
leadershipService.runForLeadership(appId.name());
-
- initWatcher();
+ k8sApiConfigService.addListener(internalK8sApiConfigListener);
log.info("Started");
}
@Deactivate
protected void deactivate() {
+ k8sApiConfigService.removeListener(internalK8sApiConfigListener);
leadershipService.withdraw(appId.name());
eventExecutor.shutdown();
log.info("Stopped");
}
- private void initWatcher() {
- K8sApiConfig config =
- k8sApiConfigService.apiConfigs().stream().findAny().orElse(null);
- if (config == null) {
- log.error("Failed to find valid kubernetes API configuration.");
- return;
+ private class InternalK8sApiConfigListener implements K8sApiConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
}
- KubernetesClient client = k8sClient(config);
+ @Override
+ public void event(K8sApiConfigEvent event) {
- if (client == null) {
- log.error("Failed to connect to kubernetes API server.");
- return;
+ switch (event.type()) {
+ case K8S_API_CONFIG_UPDATED:
+ eventExecutor.execute(this::processConfigUpdating);
+ break;
+ case K8S_API_CONFIG_CREATED:
+ case K8S_API_CONFIG_REMOVED:
+ default:
+ // do nothing
+ break;
+ }
}
- client.services().watch(internalK8sServiceWatcher);
+ private void processConfigUpdating() {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ KubernetesClient client = k8sClient(k8sApiConfigService);
+
+ if (client != null) {
+ client.services().watch(internalK8sServiceWatcher);
+ }
+ }
}
private class InternalK8sServiceWatcher implements Watcher<Service> {
@@ -149,7 +168,7 @@
return;
}
- log.info("Process service {} creating event from API server.",
+ log.trace("Process service {} creating event from API server.",
service.getMetadata().getName());
k8sServiceAdminService.createService(service);
@@ -160,10 +179,13 @@
return;
}
- log.info("Process service {} updating event from API server.",
+ log.trace("Process service {} updating event from API server.",
service.getMetadata().getName());
- k8sServiceAdminService.updateService(service);
+ if (k8sServiceAdminService.service(
+ service.getMetadata().getUid()) != null) {
+ k8sServiceAdminService.updateService(service);
+ }
}
private void processDeletion(Service service) {
@@ -171,7 +193,7 @@
return;
}
- log.info("Process service {} removal event from API server.",
+ log.trace("Process service {} removal event from API server.",
service.getMetadata().getName());
k8sServiceAdminService.removeService(service.getMetadata().getUid());
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java
index b6c2f69..2e838e8 100644
--- a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/util/K8sNetworkingUtil.java
@@ -29,6 +29,7 @@
import org.onosproject.k8snetworking.api.K8sNetwork;
import org.onosproject.k8snetworking.api.K8sNetworkService;
import org.onosproject.k8snode.api.K8sApiConfig;
+import org.onosproject.k8snode.api.K8sApiConfigService;
import org.onosproject.k8snode.api.K8sNode;
import org.onosproject.net.PortNumber;
import org.slf4j.Logger;
@@ -223,4 +224,28 @@
return new DefaultKubernetesClient(configBuilder.build());
}
+
+ /**
+ * Obtains workable kubernetes client.
+ *
+ * @param service kubernetes API service
+ * @return kubernetes client
+ */
+ public static KubernetesClient k8sClient(K8sApiConfigService service) {
+ K8sApiConfig config =
+ service.apiConfigs().stream().findAny().orElse(null);
+ if (config == null) {
+ log.error("Failed to find valid kubernetes API configuration.");
+ return null;
+ }
+
+ KubernetesClient client = k8sClient(config);
+
+ if (client == null) {
+ log.error("Failed to connect to kubernetes API server.");
+ return null;
+ }
+
+ return client;
+ }
}