Initial commit to implement external loadbalancer for kubernetes pods.
- Adds elb-related parameters in KubevirtNode class
- Defines KubernetesExternalLb and related service, manager and store.
Change-Id: I68476887af493c7daeaf1dca0320218f1764653f
(cherry picked from commit 734b5532e87da5e7f9ee9c5e63de2a3771fd8409)
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
new file mode 100644
index 0000000..fce69d7
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.impl;
+
+import com.google.common.collect.ImmutableSet;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnetworking.api.DefaultKubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbStore;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbStoreDelegate;
+import org.onosproject.store.AbstractStore;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.MapEvent;
+import org.onosproject.store.service.MapEventListener;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.Versioned;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED;
+import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED;
+import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Implementation of kubernetes external lb store using consistent map.
+ */
+@Component(immediate = true, service = KubernetesExternalLbStore.class)
+public class DistributedKubernetesExternalLbStore
+ extends AbstractStore<KubernetesExternalLbEvent, KubernetesExternalLbStoreDelegate>
+ implements KubernetesExternalLbStore {
+
+ private final Logger log = getLogger(getClass());
+
+ private static final String ERR_NOT_FOUND = " does not exist";
+ private static final String ERR_DUPLICATE = " already exists";
+ private static final String APP_ID = "org.onosproject.kubevirtnetwork";
+
+ private static final KryoNamespace SERIALIZER_KUBERNETES_EXTERNAL_LB = KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API)
+ .register(KubernetesExternalLb.class)
+ .register(DefaultKubernetesExternalLb.class)
+ .register(Collection.class)
+ .build();
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected StorageService storageService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+
+ private final MapEventListener<String, KubernetesExternalLb> lbMapEventListener =
+ new KubernetesExternalLbMapListener();
+
+ private ConsistentMap<String, KubernetesExternalLb> lbStore;
+
+ @Activate
+ protected void activate() {
+ ApplicationId appId = coreService.registerApplication(APP_ID);
+ lbStore = storageService.<String, KubernetesExternalLb>consistentMapBuilder()
+ .withSerializer(Serializer.using(SERIALIZER_KUBERNETES_EXTERNAL_LB))
+ .withName("kubernetes-lbstore")
+ .withApplicationId(appId)
+ .build();
+
+ lbStore.addListener(lbMapEventListener);
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ lbStore.removeListener(lbMapEventListener);
+ eventExecutor.shutdown();
+ log.info("Stopped");
+ }
+
+ @Override
+ public void createLoadBalancer(KubernetesExternalLb lb) {
+ lbStore.compute(lb.serviceName(), (seviceName, existing) -> {
+ final String error = lb.serviceName() + ERR_DUPLICATE;
+ checkArgument(existing == null, error);
+ return lb;
+ });
+ }
+
+ @Override
+ public void updateLoadBalancer(KubernetesExternalLb lb) {
+ lbStore.compute(lb.serviceName(), (seviceName, existing) -> {
+ final String error = lb.serviceName() + ERR_NOT_FOUND;
+ checkArgument(existing != null, error);
+ return lb;
+ });
+ }
+
+ @Override
+ public KubernetesExternalLb removeLoadBalancer(String serviceName) {
+
+ Versioned<KubernetesExternalLb> lb = lbStore.remove(serviceName);
+
+ if (lb == null) {
+ final String error = serviceName + ERR_NOT_FOUND;
+ throw new IllegalArgumentException(error);
+ }
+ return lb.value();
+ }
+
+ @Override
+ public KubernetesExternalLb loadBalancer(String serviceName) {
+ return lbStore.asJavaMap().get(serviceName);
+ }
+
+ @Override
+ public Set<KubernetesExternalLb> loadBalancers() {
+ return ImmutableSet.copyOf(lbStore.asJavaMap().values());
+ }
+
+ @Override
+ public void clear() {
+ lbStore.clear();
+ }
+
+ private class KubernetesExternalLbMapListener implements MapEventListener<String, KubernetesExternalLb> {
+ @Override
+ public void event(MapEvent<String, KubernetesExternalLb> event) {
+ switch (event.type()) {
+ case INSERT:
+ eventExecutor.execute(() ->
+ notifyDelegate(new KubernetesExternalLbEvent(
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED, event.newValue().value())));
+ break;
+ case UPDATE:
+ eventExecutor.execute(() ->
+ notifyDelegate(new KubernetesExternalLbEvent(
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED, event.newValue().value())));
+ break;
+ case REMOVE:
+ eventExecutor.execute(() ->
+ notifyDelegate(new KubernetesExternalLbEvent(
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED, event.newValue().value())));
+ break;
+ default:
+ //do nothing
+ break;
+ }
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbManager.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbManager.java
new file mode 100644
index 0000000..e68d289
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbManager.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.impl;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.event.ListenerRegistry;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbAdminService;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbListener;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbService;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbStore;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbStoreDelegate;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Set;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Provides implementation of administrating and interfacing kubernetes external lb.
+ */
+@Component(
+ immediate = true,
+ service = {KubernetesExternalLbAdminService.class, KubernetesExternalLbService.class}
+)
+public class KubernetesExternalLbManager
+ extends ListenerRegistry<KubernetesExternalLbEvent, KubernetesExternalLbListener>
+ implements KubernetesExternalLbAdminService, KubernetesExternalLbService {
+
+ protected final Logger log = getLogger(getClass());
+
+ private static final String MSG_LOAD_BALANCER = "Kubernetes external lb %s %s";
+ private static final String MSG_CREATED = "created";
+ private static final String MSG_UPDATED = "updated";
+ private static final String MSG_REMOVED = "removed";
+
+ private static final String ERR_NULL_LOAD_BALANCER = "Kubernetes external lb cannot be null";
+ private static final String ERR_NULL_LOAD_BALANCER_NAME = "Kubernetes external lb name cannot be null";
+ private static final String ERR_IN_USE = " still in use";
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbStore lbStore;
+
+ private final InternalKubernetesExternalLbStorageDelegate delegate =
+ new InternalKubernetesExternalLbStorageDelegate();
+
+ private ApplicationId appId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
+
+ lbStore.setDelegate(delegate);
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ lbStore.unsetDelegate(delegate);
+ log.info("Stopped");
+ }
+
+ @Override
+ public void createExternalLb(KubernetesExternalLb externalLb) {
+ checkNotNull(externalLb, ERR_NULL_LOAD_BALANCER);
+ checkArgument(!Strings.isNullOrEmpty(externalLb.serviceName()), ERR_NULL_LOAD_BALANCER_NAME);
+
+
+ lbStore.createLoadBalancer(externalLb);
+ log.info(String.format(MSG_LOAD_BALANCER, externalLb.serviceName(), MSG_CREATED));
+
+ }
+
+ @Override
+ public void updateExternalLb(KubernetesExternalLb externalLb) {
+ checkNotNull(externalLb, ERR_NULL_LOAD_BALANCER);
+ checkArgument(!Strings.isNullOrEmpty(externalLb.serviceName()), ERR_NULL_LOAD_BALANCER_NAME);
+
+ lbStore.updateLoadBalancer(externalLb);
+ log.info(String.format(MSG_LOAD_BALANCER, externalLb.serviceName(), MSG_UPDATED));
+ }
+
+ @Override
+ public void removeExternalLb(String serviceName) {
+ checkArgument(serviceName != null, ERR_NULL_LOAD_BALANCER_NAME);
+
+ synchronized (this) {
+ KubernetesExternalLb externalLb = lbStore.removeLoadBalancer(serviceName);
+
+ if (externalLb != null) {
+ log.info(String.format(MSG_LOAD_BALANCER, externalLb.serviceName(), MSG_REMOVED));
+ }
+ }
+ }
+
+ @Override
+ public void clear() {
+ lbStore.clear();
+ }
+
+ @Override
+ public KubernetesExternalLb loadBalancer(String serviceName) {
+ checkArgument(!Strings.isNullOrEmpty(serviceName), ERR_NULL_LOAD_BALANCER_NAME);
+
+ return lbStore.loadBalancer(serviceName);
+ }
+
+ @Override
+ public Set<KubernetesExternalLb> loadBalancers() {
+ return ImmutableSet.copyOf(lbStore.loadBalancers());
+ }
+
+ private class InternalKubernetesExternalLbStorageDelegate
+ implements KubernetesExternalLbStoreDelegate {
+
+ @Override
+ public void notify(KubernetesExternalLbEvent event) {
+ log.trace("send kubernetes external lb event {}", event);
+ process(event);
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
index de71f29..3496008 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
@@ -41,6 +41,7 @@
import org.onosproject.cfg.ConfigProperty;
import org.onosproject.kubevirtnetworking.api.DefaultKubevirtNetwork;
import org.onosproject.kubevirtnetworking.api.DefaultKubevirtPort;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
import org.onosproject.kubevirtnetworking.api.KubevirtHostRoute;
import org.onosproject.kubevirtnetworking.api.KubevirtIpPool;
import org.onosproject.kubevirtnetworking.api.KubevirtLoadBalancer;
@@ -542,6 +543,28 @@
}
/**
+ * Returns the gateway node for the specified kubernetes external lb.
+ * Among gateways, only one gateway would act as a gateway per external lb.
+ * Currently gateway node is selected based on modulo operation with router hashcode.
+ *
+ * @param nodeService kubevirt node service
+ * @param externalLb kubernetes external lb
+ * @return elected gateway node
+ */
+ public static KubevirtNode gatewayNodeForSpecifiedService(KubevirtNodeService nodeService,
+ KubernetesExternalLb externalLb) {
+ //TODO: enhance election logic for a better load balancing
+
+ int numOfGateways = nodeService.completeNodes(GATEWAY).size();
+ if (numOfGateways == 0) {
+ return null;
+ }
+
+ return (KubevirtNode) nodeService.completeNodes(GATEWAY)
+ .toArray()[externalLb.hashCode() % numOfGateways];
+ }
+
+ /**
* Returns the mac address of the router.
*
* @param router kubevirt router