[ONOS-7926] Implement IPAM service to allocate IP for Kubernetes POD
Change-Id: I32fd1fffb41ec728d0be092ac5a8f555179e7a9e
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/DistributedK8sIpamStore.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/DistributedK8sIpamStore.java
new file mode 100644
index 0000000..9142e24
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/DistributedK8sIpamStore.java
@@ -0,0 +1,195 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.impl;
+
+import com.google.common.collect.ImmutableSet;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snetworking.api.DefaultK8sIpam;
+import org.onosproject.k8snetworking.api.K8sIpam;
+import org.onosproject.k8snetworking.api.K8sIpamEvent;
+import org.onosproject.k8snetworking.api.K8sIpamStore;
+import org.onosproject.k8snetworking.api.K8sIpamStoreDelegate;
+import org.onosproject.store.AbstractStore;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.Versioned;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Implementation of kubernetes IP address management store using consistent map.
+ */
+@Component(immediate = true, service = K8sIpamStore.class)
+public class DistributedK8sIpamStore
+ extends AbstractStore<K8sIpamEvent, K8sIpamStoreDelegate>
+ implements K8sIpamStore {
+
+ private final Logger log = getLogger(getClass());
+
+ private static final String ERR_NOT_FOUND = " does not exist";
+ private static final String ERR_DUPLICATE = " already exists";
+ private static final String APP_ID = "org.onosproject.k8snetwork";
+
+ private static final KryoNamespace
+ SERIALIZER_K8S_IPAM = KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API)
+ .register(K8sIpam.class)
+ .register(DefaultK8sIpam.class)
+ .register(Collection.class)
+ .build();
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected StorageService storageService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+
+ private ConsistentMap<String, K8sIpam> allocatedStore;
+ private ConsistentMap<String, K8sIpam> availableStore;
+
+ @Activate
+ protected void activate() {
+ ApplicationId appId = coreService.registerApplication(APP_ID);
+ allocatedStore = storageService.<String, K8sIpam>consistentMapBuilder()
+ .withSerializer(Serializer.using(SERIALIZER_K8S_IPAM))
+ .withName("k8s-ipam-allocated-store")
+ .withApplicationId(appId)
+ .build();
+ availableStore = storageService.<String, K8sIpam>consistentMapBuilder()
+ .withSerializer(Serializer.using(SERIALIZER_K8S_IPAM))
+ .withName("k8s-ipam-available-store")
+ .withApplicationId(appId)
+ .build();
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ eventExecutor.shutdown();
+ log.info("Stopped");
+ }
+
+ @Override
+ public void createAllocatedIp(K8sIpam ipam) {
+ allocatedStore.compute(ipam.ipamId(), (ipamId, existing) -> {
+ final String error = ipam.ipamId() + ERR_DUPLICATE;
+ checkArgument(existing == null, error);
+ return ipam;
+ });
+ }
+
+ @Override
+ public void updateAllocatedIp(K8sIpam ipam) {
+ allocatedStore.compute(ipam.ipamId(), (ipamId, existing) -> {
+ final String error = ipam.ipamId() + ERR_NOT_FOUND;
+ checkArgument(existing != null, error);
+ return ipam;
+ });
+ }
+
+ @Override
+ public K8sIpam removeAllocatedIp(String ipamId) {
+ Versioned<K8sIpam> ipam = allocatedStore.remove(ipamId);
+ if (ipam == null) {
+ final String error = ipamId + ERR_NOT_FOUND;
+ throw new IllegalArgumentException(error);
+ }
+ return ipam.value();
+ }
+
+ @Override
+ public K8sIpam allocatedIp(String ipamId) {
+ return allocatedStore.asJavaMap().get(ipamId);
+ }
+
+ @Override
+ public Set<K8sIpam> allocatedIps() {
+ return ImmutableSet.copyOf(allocatedStore.asJavaMap().values());
+ }
+
+ @Override
+ public void createAvailableIp(K8sIpam ipam) {
+ availableStore.compute(ipam.ipamId(), (ipamId, existing) -> {
+ final String error = ipam.ipamId() + ERR_DUPLICATE;
+ checkArgument(existing == null, error);
+ return ipam;
+ });
+ }
+
+ @Override
+ public void updateAvailableIp(K8sIpam ipam) {
+ availableStore.compute(ipam.ipamId(), (ipamId, existing) -> {
+ final String error = ipam.ipamId() + ERR_NOT_FOUND;
+ checkArgument(existing != null, error);
+ return ipam;
+ });
+ }
+
+ @Override
+ public K8sIpam removeAvailableIp(String ipamId) {
+ Versioned<K8sIpam> ipam = availableStore.remove(ipamId);
+ if (ipam == null) {
+ final String error = ipamId + ERR_NOT_FOUND;
+ throw new IllegalArgumentException(error);
+ }
+ return ipam.value();
+ }
+
+ @Override
+ public K8sIpam availableIp(String ipamId) {
+ return availableStore.asJavaMap().get(ipamId);
+ }
+
+ @Override
+ public Set<K8sIpam> availableIps() {
+ return ImmutableSet.copyOf(availableStore.asJavaMap().values());
+ }
+
+ @Override
+ public void clear() {
+ allocatedStore.clear();
+ availableStore.clear();
+ }
+
+ @Override
+ public void clear(String networkId) {
+ Set<K8sIpam> ipams = allocatedStore.asJavaMap().values().stream()
+ .filter(i -> i.networkId().equals(networkId))
+ .collect(Collectors.toSet());
+ ipams.forEach(i -> allocatedStore.remove(i.ipamId()));
+ }
+}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sIpamHandler.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sIpamHandler.java
new file mode 100644
index 0000000..5591d2e
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sIpamHandler.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.impl;
+
+import org.onlab.packet.IpAddress;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.k8snetworking.api.K8sIpamAdminService;
+import org.onosproject.k8snetworking.api.K8sNetworkEvent;
+import org.onosproject.k8snetworking.api.K8sNetworkListener;
+import org.onosproject.k8snetworking.api.K8sNetworkService;
+import org.onosproject.mastership.MastershipService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.onosproject.k8snetworking.util.K8sNetworkingUtil.getSubnetIps;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Initializes and purges the kubernetes IPAM.
+ */
+@Component(immediate = true)
+public class K8sIpamHandler {
+
+ private final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sNetworkService k8sNetworkService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sIpamAdminService k8sIpamAdminService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+ private final InternalK8sNetworkListener k8sNetworkListener =
+ new InternalK8sNetworkListener();
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ k8sNetworkService.addListener(k8sNetworkListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ k8sNetworkService.removeListener(k8sNetworkListener);
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private class InternalK8sNetworkListener implements K8sNetworkListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(K8sNetworkEvent event) {
+ switch (event.type()) {
+ case K8S_NETWORK_CREATED:
+ eventExecutor.execute(() -> processNetworkAddition(event));
+ break;
+ case K8S_NETWORK_REMOVED:
+ eventExecutor.execute(() -> processNetworkRemoval(event));
+ break;
+ default:
+ break;
+ }
+ }
+
+ private void processNetworkAddition(K8sNetworkEvent event) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ Set<IpAddress> ips = getSubnetIps(event.subject().cidr());
+ k8sIpamAdminService.initializeIpPool(event.subject().networkId(), ips);
+ }
+
+ private void processNetworkRemoval(K8sNetworkEvent event) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ k8sIpamAdminService.purgeIpPool(event.subject().networkId());
+ }
+ }
+}
diff --git a/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sIpamManager.java b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sIpamManager.java
new file mode 100644
index 0000000..5501971
--- /dev/null
+++ b/apps/k8s-networking/app/src/main/java/org/onosproject/k8snetworking/impl/K8sIpamManager.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.k8snetworking.impl;
+
+import org.onlab.packet.IpAddress;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.event.ListenerRegistry;
+import org.onosproject.k8snetworking.api.DefaultK8sIpam;
+import org.onosproject.k8snetworking.api.K8sIpam;
+import org.onosproject.k8snetworking.api.K8sIpamAdminService;
+import org.onosproject.k8snetworking.api.K8sIpamEvent;
+import org.onosproject.k8snetworking.api.K8sIpamListener;
+import org.onosproject.k8snetworking.api.K8sIpamService;
+import org.onosproject.k8snetworking.api.K8sIpamStore;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.onosproject.k8snetworking.api.Constants.K8S_NETWORKING_APP_ID;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Provides implementation of administering and interfacing kubernetes IPAM.
+ */
+@Component(
+ immediate = true,
+ service = {K8sIpamAdminService.class, K8sIpamService.class}
+)
+public class K8sIpamManager
+ extends ListenerRegistry<K8sIpamEvent, K8sIpamListener>
+ implements K8sIpamAdminService, K8sIpamService {
+
+ protected final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected K8sIpamStore k8sIpamStore;
+
+ private ApplicationId appId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(K8S_NETWORKING_APP_ID);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ log.info("Stopped");
+ }
+
+ @Override
+ public IpAddress allocateIp(String networkId) {
+ IpAddress availableIp = availableIps(networkId).stream()
+ .findFirst().orElse(null);
+ if (availableIp != null) {
+ String ipamId = networkId + "-" + availableIp.toString();
+ k8sIpamStore.removeAvailableIp(ipamId);
+ k8sIpamStore.createAllocatedIp(
+ new DefaultK8sIpam(ipamId, availableIp, networkId));
+
+ log.info("Allocate a new IP {}", availableIp.toString());
+
+ return availableIp;
+ } else {
+ log.warn("No IPs are available for allocating.");
+ }
+ return null;
+ }
+
+ @Override
+ public boolean releaseIp(String networkId, IpAddress ipAddress) {
+ IpAddress releasedIp = allocatedIps(networkId).stream()
+ .filter(ip -> ip.equals(ipAddress))
+ .findFirst().orElse(null);
+ if (releasedIp != null) {
+ String ipamId = networkId + "-" + releasedIp.toString();
+ k8sIpamStore.removeAllocatedIp(ipamId);
+ k8sIpamStore.createAvailableIp(
+ new DefaultK8sIpam(ipamId, releasedIp, networkId));
+
+ log.info("Release the IP {}", releasedIp.toString());
+
+ return true;
+ } else {
+ log.warn("Failed to find requested IP {} for releasing...", ipAddress.toString());
+ }
+
+ return false;
+ }
+
+ @Override
+ public void initializeIpPool(String networkId, Set<IpAddress> ipAddresses) {
+ ipAddresses.forEach(ip -> {
+ String ipamId = networkId + "-" + ip;
+ K8sIpam ipam = new DefaultK8sIpam(ipamId, ip, networkId);
+ k8sIpamStore.createAvailableIp(ipam);
+ });
+ }
+
+ @Override
+ public void purgeIpPool(String networkId) {
+ k8sIpamStore.clear(networkId);
+ }
+
+ @Override
+ public Set<IpAddress> allocatedIps(String networkId) {
+ return k8sIpamStore.allocatedIps().stream()
+ .filter(i -> i.networkId().equals(networkId))
+ .map(K8sIpam::ipAddress).collect(Collectors.toSet());
+ }
+
+ @Override
+ public Set<IpAddress> availableIps(String networkId) {
+ return k8sIpamStore.availableIps().stream()
+ .filter(i -> i.networkId().equals(networkId))
+ .map(K8sIpam::ipAddress).collect(Collectors.toSet());
+ }
+}