Implement kubernetes external loadbalancer handler.
Change-Id: I0f3057d66769f0ca7db7d508483835cdd1ff1593
(cherry picked from commit 5ff76b7bd3ef45b8538464a6fc510995d120739e)
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
index 0675c93..6866537 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
@@ -28,6 +28,9 @@
public static final String KUBEVIRT_NETWORKING_APP_ID =
"org.onosproject.kubevirtnetworking";
public static final String DEFAULT_GATEWAY_MAC_STR = "fe:00:00:00:00:02";
+ public static final String KUBERNETES_EXTERNAL_LB_FAKE_MAC_STR = "de:00:00:00:00:01";
+ public static final MacAddress KUBERNETES_EXTERNAL_LB_FAKE_MAC =
+ MacAddress.valueOf(KUBERNETES_EXTERNAL_LB_FAKE_MAC_STR);
public static final MacAddress DEFAULT_GATEWAY_MAC =
MacAddress.valueOf(DEFAULT_GATEWAY_MAC_STR);
@@ -106,6 +109,8 @@
public static final int PRIORITY_STATEFUL_SNAT_RULE = 40500;
public static final int PRIORITY_FLOATING_IP_UPSTREAM_RULE = 40800;
public static final int PRIORITY_FLOATING_IP_DOWNSTREAM_RULE = 40700;
+ public static final int PRIORITY_ELB_UPSTREAM_RULE = 39000;
+ public static final int PRIORITY_ELB_DOWNSTREAM_RULE = 39500;
public static final int PRIORITY_INTERNAL_ROUTING_RULE = 41000;
public static final int PRIORITY_LB_RULE = 41500;
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
index 96ecf12..10e7cba 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
@@ -17,6 +17,8 @@
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
import java.util.Objects;
import java.util.Set;
@@ -28,24 +30,32 @@
*/
public final class DefaultKubernetesExternalLb implements KubernetesExternalLb {
- private static final String NOT_NULL_MSG = "Loadbalancer % cannot be null";
+ private static final String NOT_NULL_MSG = "External Loadbalancer % cannot be null";
private final String serviceName;
- private final String loadbalancerIp;
+ private final IpAddress loadbalancerIp;
private final Set<Integer> nodePortSet;
private final Set<Integer> portSet;
private final Set<String> endpointSet;
private final String electedGateway;
+ private final IpAddress loadbalancerGwIp;
+ private final MacAddress loadbalancerGwMac;
+ private final String electedWorker;
- public DefaultKubernetesExternalLb(String serviceName, String loadbalancerIp,
+ public DefaultKubernetesExternalLb(String serviceName, IpAddress loadbalancerIp,
Set<Integer> nodePortSet, Set<Integer> portSet,
- Set<String> endpointSet, String electedGateway) {
+ Set<String> endpointSet, String electedGateway,
+ String electedWorker,
+ IpAddress loadbalancerGwIp, MacAddress loadbalancerGwMac) {
this.serviceName = serviceName;
this.loadbalancerIp = loadbalancerIp;
this.nodePortSet = nodePortSet;
this.portSet = portSet;
this.endpointSet = endpointSet;
this.electedGateway = electedGateway;
+ this.electedWorker = electedWorker;
+ this.loadbalancerGwIp = loadbalancerGwIp;
+ this.loadbalancerGwMac = loadbalancerGwMac;
}
@Override
@@ -54,7 +64,7 @@
}
@Override
- public String loadBalancerIp() {
+ public IpAddress loadBalancerIp() {
return loadbalancerIp;
}
@@ -79,6 +89,21 @@
}
@Override
+ public String electedWorker() {
+ return electedWorker;
+ }
+
+ @Override
+ public IpAddress loadBalancerGwIp() {
+ return loadbalancerGwIp;
+ }
+
+ @Override
+ public MacAddress loadBalancerGwMac() {
+ return loadbalancerGwMac;
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) {
return true;
@@ -92,7 +117,10 @@
Objects.equals(nodePortSet, that.nodePortSet) &&
Objects.equals(portSet, that.portSet) &&
Objects.equals(endpointSet, that.endpointSet) &&
- Objects.equals(electedGateway, that.electedGateway);
+ Objects.equals(electedGateway, that.electedGateway) &&
+ Objects.equals(electedWorker, that.electedWorker) &&
+ Objects.equals(loadbalancerGwIp, that.loadbalancerGwIp) &&
+ Objects.equals(loadbalancerGwMac, that.loadbalancerGwMac);
}
@Override
@@ -104,8 +132,27 @@
.portSet(portSet)
.endpointSet(endpointSet)
.electedGateway(electedGateway)
+ .electedWorker(electedWorker)
+ .loadBalancerGwIp(loadbalancerGwIp)
+ .loadBalancerGwMac(loadbalancerGwMac)
.build();
}
+
+ @Override
+ public KubernetesExternalLb updateElectedWorker(String electedWorker) {
+ return DefaultKubernetesExternalLb.builder()
+ .serviceName(serviceName)
+ .loadBalancerIp(loadbalancerIp)
+ .nodePortSet(nodePortSet)
+ .portSet(portSet)
+ .endpointSet(endpointSet)
+ .electedGateway(electedGateway)
+ .electedWorker(electedWorker)
+ .loadBalancerGwIp(loadbalancerGwIp)
+ .loadBalancerGwMac(loadbalancerGwMac)
+ .build();
+ }
+
@Override
public int hashCode() {
return Objects.hash(serviceName, loadbalancerIp);
@@ -120,6 +167,9 @@
.add("port", portSet)
.add("endpointSet", endpointSet)
.add("electedGateway", electedGateway)
+ .add("electedWorker", electedWorker)
+ .add("loadbalancer gateway ip", loadbalancerGwIp)
+ .add("loadbalancer gateway Mac", loadbalancerGwMac)
.toString();
}
@@ -129,11 +179,14 @@
public static final class Builder implements KubernetesExternalLb.Builder {
private String serviceName;
- private String loadbalancerIp;
+ private IpAddress loadbalancerIp;
private Set<Integer> nodePortSet;
private Set<Integer> portSet;
private Set<String> endpointSet;
private String electedGateway;
+ private String electedWorker;
+ private IpAddress loadbalancerGwip;
+ private MacAddress loadbalancerGwMac;
private Builder() {
}
@@ -146,7 +199,8 @@
checkArgument(!portSet.isEmpty(), NOT_NULL_MSG, "portSet");
return new DefaultKubernetesExternalLb(serviceName, loadbalancerIp,
- nodePortSet, portSet, endpointSet, electedGateway);
+ nodePortSet, portSet, endpointSet, electedGateway, electedWorker,
+ loadbalancerGwip, loadbalancerGwMac);
}
@Override
@@ -156,7 +210,7 @@
}
@Override
- public Builder loadBalancerIp(String loadBalancerIp) {
+ public Builder loadBalancerIp(IpAddress loadBalancerIp) {
this.loadbalancerIp = loadBalancerIp;
return this;
}
@@ -179,9 +233,28 @@
return this;
}
+ @Override
public Builder electedGateway(String electedGateway) {
this.electedGateway = electedGateway;
return this;
}
+
+ @Override
+ public Builder electedWorker(String electedWorker) {
+ this.electedWorker = electedWorker;
+ return this;
+ }
+
+ @Override
+ public Builder loadBalancerGwIp(IpAddress loadbalancerGwip) {
+ this.loadbalancerGwip = loadbalancerGwip;
+ return this;
+ }
+
+ @Override
+ public Builder loadBalancerGwMac(MacAddress loadbalancerGwMac) {
+ this.loadbalancerGwMac = loadbalancerGwMac;
+ return this;
+ }
}
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java
index bc754db..4e902b0 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java
@@ -15,6 +15,9 @@
*/
package org.onosproject.kubevirtnetworking.api;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+
import java.util.Set;
/**
@@ -33,7 +36,7 @@
*
* @return load balancer IP
*/
- String loadBalancerIp();
+ IpAddress loadBalancerIp();
/**
* Returns the set of node port.
@@ -64,6 +67,13 @@
String electedGateway();
/**
+ * Returns the elected worker node for this service.
+ *
+ * @return worker node hostname
+ */
+ String electedWorker();
+
+ /**
* Updates the elected gateway node host name.
*
* @param electedGateway updated elected gateway node hostname
@@ -71,6 +81,31 @@
*/
KubernetesExternalLb updateElectedGateway(String electedGateway);
+ /**
+ * Updates the elected worker node host name.
+ *
+ * @param electedWorker updated elected worker node hostname
+ * @return kubernetes external lb with the updated worker node hostname
+ */
+ KubernetesExternalLb updateElectedWorker(String electedWorker);
+
+ /**
+ * Returns the gateway IP of load balancer.
+ * TEG would send outbound traffic to this gateway.
+ *
+ * @return load balancer gateway IP
+ */
+ IpAddress loadBalancerGwIp();
+
+ /**
+ * Returns the gateway MAC of load balancer.
+ * TEG would send outbound traffic to this gateway.
+ *
+ * @return load balancer gateway IP
+ */
+ MacAddress loadBalancerGwMac();
+
+
interface Builder {
/**
* Builds an immutable kubernetes external load balancer instance.
@@ -93,7 +128,7 @@
* @param loadBalancerIp external load balancer Ip
* @return external load balancer builder
*/
- Builder loadBalancerIp(String loadBalancerIp);
+ Builder loadBalancerIp(IpAddress loadBalancerIp);
/**
* Returns kubernetes external load balancer builder with supplied node port set.
@@ -123,8 +158,33 @@
* Returns kubernetes external load balancer builder with supplied elected gateway.
*
* @param gateway gateway node hostname
- * @return gateway node hostname
+ * @return external load balancer builder
*/
Builder electedGateway(String gateway);
+
+ /**
+ * Returns kubernetes external load balancer builder with supplied elected worker.
+ *
+ * @param worker worker node hostname
+ * @return external load balancer builder
+ */
+ Builder electedWorker(String worker);
+
+
+ /**
+ * Returns kubernetes external load balancer builder with supplied load balancer gateway Ip.
+ *
+ * @param loadBalancerGwIp gateway IP of the external load balancer
+ * @return external load balancer builder
+ */
+ Builder loadBalancerGwIp(IpAddress loadBalancerGwIp);
+
+ /**
+ * Returns kubernetes external load balancer builder with supplied load balancer gateway Mac.
+ *
+ * @param loadBalancerGwMac gateway Mac of the external load balancer
+ * @return external load balancer builder
+ */
+ Builder loadBalancerGwMac(MacAddress loadBalancerGwMac);
}
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbAdminService.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbAdminService.java
index c53d409..b5b05ed 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbAdminService.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbAdminService.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2021-present Open Networking Foundation
+ * Copyright 2022-present Open Networking Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbEvent.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbEvent.java
index 15f3905..69aebce 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbEvent.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbEvent.java
@@ -23,8 +23,26 @@
public class KubernetesExternalLbEvent
extends AbstractEvent<KubernetesExternalLbEvent.Type, KubernetesExternalLb> {
+ private final String oldGateway;
+ private final String oldWorker;
+
+
public KubernetesExternalLbEvent(Type type, KubernetesExternalLb subject) {
super(type, subject);
+ this.oldGateway = null;
+ this.oldWorker = null;
+ }
+
+ public KubernetesExternalLbEvent(Type type, KubernetesExternalLb subject, String oldGateway, String oldWorker) {
+ super(type, subject);
+ this.oldGateway = oldGateway;
+ this.oldWorker = oldWorker;
+ }
+
+ public KubernetesExternalLbEvent(Type type, KubernetesExternalLb subject, String oldWorker) {
+ super(type, subject);
+ this.oldGateway = null;
+ this.oldWorker = oldWorker;
}
/**
@@ -32,18 +50,46 @@
*/
public enum Type {
/**
- * Signifies that a new kubevirt load balancer is created.
+ * Signifies that a new kubernetex external load balancer is created.
*/
KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED,
/**
- * Signifies that a kubevirt load balancer is removed.
+ * Signifies that a kubernetex external load balancer is removed.
*/
KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED,
/**
- * Signifies that a kubevirt load balancer is updated.
+ * Signifies that a kubernetex external load balancer is updated.
*/
KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED,
+
+ /**
+ * Signifies that a kubernetes external load balancer gateway node is updated.
+ */
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_GATEWAY_CHANGED,
+
+ /**
+ * Signifies that a kubernetes external load balancer worker node is updated.
+ */
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_WORKER_CHANGED,
+ }
+
+ /**
+ * Returns the old gateway of the router event.
+ *
+ * @return gateway node hostname
+ */
+ public String oldGateway() {
+ return oldGateway;
+ }
+
+ /**
+ * Returns the old worker of the router event.
+ *
+ * @return worker node hostname
+ */
+ public String oldWorker() {
+ return oldWorker;
}
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbListener.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbListener.java
index 1a48764..5cbeb30 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbListener.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbListener.java
@@ -18,7 +18,7 @@
import org.onosproject.event.EventListener;
/**
- * Listener for kubernetes external lb event.
+ * Listener for kubernetes external load balancer event.
*/
public interface KubernetesExternalLbListener extends EventListener<KubernetesExternalLbEvent> {
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbService.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbService.java
index eeac4f2..dee7ae9 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbService.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbService.java
@@ -23,7 +23,7 @@
extends ListenerService<KubernetesExternalLbEvent, KubernetesExternalLbListener> {
/**
- * Returns the kubernetes external lb with the supplied service name.
+ * Returns the kubernetes external load balancer with the supplied service name.
*
* @param serviceName service name
* @return kubernetes external load balancer
@@ -31,9 +31,9 @@
KubernetesExternalLb loadBalancer(String serviceName);
/**
- * Returns all kubernetes external lb's registered in the service.
+ * Returns all kubernetes external load balancers registered in the service.
*
- * @return set of kubernetes external lb's
+ * @return set of kubernetes external load balancers
*/
Set<KubernetesExternalLb> loadBalancers();
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbStoreDelegate.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbStoreDelegate.java
index a7fba41..e7ff6ff 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbStoreDelegate.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbStoreDelegate.java
@@ -18,7 +18,7 @@
import org.onosproject.store.StoreDelegate;
/**
- * Kubernetes external lb store delegate abstraction.
+ * Kubernetes external load balancer store delegate abstraction.
*/
public interface KubernetesExternalLbStoreDelegate extends StoreDelegate<KubernetesExternalLbEvent> {
}
diff --git a/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java b/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java
index a4a661f..228bb9f 100644
--- a/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java
+++ b/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2021-present Open Networking Foundation
+ * Copyright 2022-present Open Networking Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@
import com.google.common.testing.EqualsTester;
import org.junit.Before;
import org.junit.Test;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
import java.util.Set;
@@ -31,8 +33,8 @@
public class DefaultKubernetesExternalLbTest {
private static final String SERVICE_NAME_1 = "service_name_1";
private static final String SERVICE_NAME_2 = "service_name_2";
- private static final String LOADBALANCER_IP_1 = "1.1.1.1";
- private static final String LOADBALANCER_IP_2 = "2.2.2.2";
+ private static final IpAddress LOADBALANCER_IP_1 = IpAddress.valueOf("1.1.1.2");
+ private static final IpAddress LOADBALANCER_IP_2 = IpAddress.valueOf("2.2.2.2");
private static final Set<Integer> NODE_PORT_SET_1 = Sets.newHashSet(Integer.valueOf(32080));
private static final Set<Integer> NODE_PORT_SET_2 = Sets.newHashSet(Integer.valueOf(33080));
private static final Set<Integer> PORT_SET_1 = Sets.newHashSet(Integer.valueOf(8080));
@@ -41,6 +43,13 @@
private static final Set<String> ENDPOINT_SET_2 = Sets.newHashSet(String.valueOf("1.1.2.2"));
private static final String ELECTED_GATEWAY_1 = "gateway1";
private static final String ELECTED_GATEWAY_2 = "gateway2";
+ private static final String ELECTED_WORKER_1 = "worker1";
+ private static final String ELECTED_WORKER_2 = "worker2";
+
+ private static final IpAddress LOADBALANCER_GW_IP_1 = IpAddress.valueOf("1.1.1.1");
+ private static final IpAddress LOADBALANCER_GW_IP_2 = IpAddress.valueOf("2.2.2.1");
+ private static final MacAddress LOADBALANCER_GW_MAC_1 = MacAddress.valueOf("aa:bb:cc:dd:ee:ff");
+ private static final MacAddress LOADBALANCER_GW_MAC_2 = MacAddress.valueOf("ff:ee:dd:cc:bb:aa");
private KubernetesExternalLb lb1;
private KubernetesExternalLb sameAsLb1;
@@ -66,6 +75,9 @@
.portSet(PORT_SET_1)
.endpointSet(ENDPOINT_SET_1)
.electedGateway(ELECTED_GATEWAY_1)
+ .electedWorker(ELECTED_WORKER_1)
+ .loadBalancerGwIp(LOADBALANCER_GW_IP_1)
+ .loadBalancerGwMac(LOADBALANCER_GW_MAC_1)
.build();
sameAsLb1 = DefaultKubernetesExternalLb.builder()
@@ -75,6 +87,9 @@
.portSet(PORT_SET_1)
.endpointSet(ENDPOINT_SET_1)
.electedGateway(ELECTED_GATEWAY_1)
+ .electedWorker(ELECTED_WORKER_1)
+ .loadBalancerGwIp(LOADBALANCER_GW_IP_1)
+ .loadBalancerGwMac(LOADBALANCER_GW_MAC_1)
.build();
lb2 = DefaultKubernetesExternalLb.builder()
@@ -84,6 +99,9 @@
.portSet(PORT_SET_2)
.endpointSet(ENDPOINT_SET_2)
.electedGateway(ELECTED_GATEWAY_2)
+ .electedWorker(ELECTED_WORKER_2)
+ .loadBalancerGwIp(LOADBALANCER_GW_IP_2)
+ .loadBalancerGwMac(LOADBALANCER_GW_MAC_2)
.build();
}
@@ -104,10 +122,10 @@
public void testConstruction() {
KubernetesExternalLb lb = lb1;
- assertEquals(SERVICE_NAME_1, lb1.serviceName());
- assertEquals(LOADBALANCER_IP_1, lb1.loadBalancerIp());
- assertEquals(NODE_PORT_SET_1, lb1.nodePortSet());
- assertEquals(PORT_SET_1, lb1.portSet());
- assertEquals(ENDPOINT_SET_1, lb1.endpointSet());
+ assertEquals(SERVICE_NAME_1, lb.serviceName());
+ assertEquals(LOADBALANCER_IP_1, lb.loadBalancerIp());
+ assertEquals(NODE_PORT_SET_1, lb.nodePortSet());
+ assertEquals(PORT_SET_1, lb.portSet());
+ assertEquals(ENDPOINT_SET_1, lb.endpointSet());
}
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java
new file mode 100644
index 0000000..47d4a71
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2021-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.cli;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
+import org.apache.karaf.shell.api.action.Command;
+import org.apache.karaf.shell.api.action.lifecycle.Service;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbService;
+
+import java.util.List;
+
+import static org.onosproject.kubevirtnetworking.api.Constants.CLI_IP_ADDRESS_LENGTH;
+import static org.onosproject.kubevirtnetworking.api.Constants.CLI_MAC_ADDRESS_LENGTH;
+import static org.onosproject.kubevirtnetworking.api.Constants.CLI_MARGIN_LENGTH;
+import static org.onosproject.kubevirtnetworking.api.Constants.CLI_NAME_LENGTH;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.genFormatString;
+
+/**
+ * Lists kubernetes services.
+ */
+@Service
+@Command(scope = "onos", name = "kubernetes-services",
+ description = "Lists all kubernetes services")
+public class KubernetesListServiceCommand extends AbstractShellCommand {
+
+ @Override
+ protected void doExecute() throws Exception {
+ KubernetesExternalLbService service = get(KubernetesExternalLbService.class);
+
+ List<KubernetesExternalLb> elbList = Lists.newArrayList(service.loadBalancers());
+
+ String format = genFormatString(ImmutableList.of(CLI_NAME_LENGTH, CLI_IP_ADDRESS_LENGTH,
+ CLI_NAME_LENGTH, CLI_NAME_LENGTH, CLI_IP_ADDRESS_LENGTH, CLI_MAC_ADDRESS_LENGTH));
+
+
+ print(format, "Service Name", "Loadbalancer IP", "Elected Gateway", "Elected Worker",
+ "Loadbalancer GW IP", "Loadbalancer GW MAC");
+
+ for (KubernetesExternalLb elb : elbList) {
+ String lbIp = elb.loadBalancerIp() == null ? "N/A" : elb.loadBalancerIp().toString();
+ String electedGw = elb.electedGateway() == null ? "N/A" : elb.electedGateway();
+ String electedWorker = elb.electedWorker() == null ? "N/A" : elb.electedWorker();
+ String lbGwIp = elb.loadBalancerGwIp() == null ? "N/A" : elb.loadBalancerGwIp().toString();
+ String lbGwMac = elb.loadBalancerGwMac() == null ? "N/A" : elb.loadBalancerGwMac().toString();
+
+ print(format, StringUtils.substring(elb.serviceName(), 0,
+ CLI_NAME_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(lbIp, 0,
+ CLI_IP_ADDRESS_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(electedGw, 0,
+ CLI_NAME_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(electedWorker, 0,
+ CLI_NAME_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(lbGwIp, 0,
+ CLI_IP_ADDRESS_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(lbGwMac, 0,
+ CLI_MAC_ADDRESS_LENGTH - CLI_MARGIN_LENGTH)
+ );
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
index fce69d7..1bd286e 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
@@ -16,6 +16,7 @@
package org.onosproject.kubevirtnetworking.impl;
import com.google.common.collect.ImmutableSet;
+import org.onlab.packet.IpAddress;
import org.onlab.util.KryoNamespace;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
@@ -47,12 +48,14 @@
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED;
+import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_GATEWAY_CHANGED;
import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED;
import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED;
+import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_WORKER_CHANGED;
import static org.slf4j.LoggerFactory.getLogger;
/**
- * Implementation of kubernetes external lb store using consistent map.
+ * Implementation of kubernetes external load balancer store using consistent map.
*/
@Component(immediate = true, service = KubernetesExternalLbStore.class)
public class DistributedKubernetesExternalLbStore
@@ -69,6 +72,7 @@
.register(KryoNamespaces.API)
.register(KubernetesExternalLb.class)
.register(DefaultKubernetesExternalLb.class)
+ .register(IpAddress.class)
.register(Collection.class)
.build();
@@ -161,19 +165,44 @@
KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED, event.newValue().value())));
break;
case UPDATE:
- eventExecutor.execute(() ->
- notifyDelegate(new KubernetesExternalLbEvent(
- KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED, event.newValue().value())));
+ eventExecutor.execute(() -> processMapUpdate(event));
break;
case REMOVE:
eventExecutor.execute(() ->
notifyDelegate(new KubernetesExternalLbEvent(
- KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED, event.newValue().value())));
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED, event.oldValue().value())));
break;
default:
//do nothing
break;
}
}
+
+ private void processMapUpdate(MapEvent<String, KubernetesExternalLb> event) {
+ log.debug("Kubernetes External LB updated");
+
+ KubernetesExternalLb oldValue = event.oldValue().value();
+ KubernetesExternalLb newValue = event.newValue().value();
+
+ if (oldValue.electedGateway() != null && newValue.electedGateway() != null &&
+ !oldValue.electedGateway().equals(newValue.electedGateway())) {
+ notifyDelegate(new KubernetesExternalLbEvent(
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_GATEWAY_CHANGED,
+ newValue, oldValue.electedGateway(), oldValue.electedWorker())
+ );
+ }
+
+ if (oldValue.electedWorker() != null && newValue.electedWorker() != null &&
+ !oldValue.electedWorker().equals(newValue.electedWorker())) {
+ notifyDelegate(new KubernetesExternalLbEvent(
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_WORKER_CHANGED,
+ newValue, oldValue.electedWorker())
+ );
+ }
+
+ notifyDelegate(new KubernetesExternalLbEvent(
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED, event.newValue().value()));
+
+ }
}
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
new file mode 100644
index 0000000..7b1c733
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
@@ -0,0 +1,460 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.impl;
+
+import org.onlab.packet.ARP;
+import org.onlab.packet.EthType;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.TpPort;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbAdminService;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbListener;
+import org.onosproject.kubevirtnetworking.api.KubevirtFlowRuleService;
+import org.onosproject.kubevirtnetworking.api.KubevirtGroupRuleService;
+import org.onosproject.kubevirtnetworking.util.RulePopulatorUtil;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.kubevirtnode.api.KubevirtNodeService;
+import org.onosproject.net.Device;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.onosproject.net.packet.PacketService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.GW_DROP_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.GW_ENTRY_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBERNETES_EXTERNAL_LB_FAKE_MAC;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ELB_DOWNSTREAM_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ELB_UPSTREAM_RULE;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.elbPatchPortNum;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.externalPatchPortNum;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.kubernetesElbMac;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.CT_NAT_SRC_FLAG;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Handles Kubernetes External load balancer.
+ */
+@Component(immediate = true)
+public class KubernetesExternalLbHandler {
+ protected final Logger log = getLogger(getClass());
+
+ private static final int TP_PORT_MINIMUM_NUM = 1025;
+ private static final int TP_PORT_MAXIMUM_NUM = 65535;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtApiConfigService configService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeService nodeService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtGroupRuleService groupRuleService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected PacketService packetService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtFlowRuleService flowService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbAdminService externalLbService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DriverService driverService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ private final InternalKubernetesExternalLbListener lbListener =
+ new InternalKubernetesExternalLbListener();
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ externalLbService.addListener(lbListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+ externalLbService.removeListener(lbListener);
+
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+
+ private class InternalKubernetesExternalLbListener implements KubernetesExternalLbListener {
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubernetesExternalLbEvent event) {
+ switch (event.type()) {
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED:
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED:
+ eventExecutor.execute(() -> processKubernetesExternalLbCreatedOrUpdated(
+ event.subject()));
+ break;
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_GATEWAY_CHANGED:
+ eventExecutor.execute(() -> processKubernetesExternalLbGatewayChanged(
+ event.subject(), event.oldGateway()));
+ break;
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_WORKER_CHANGED:
+ eventExecutor.execute(() -> processKubernetesExternalLbWorkerChanged(
+ event.subject(), event.oldWorker()));
+ break;
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED:
+ eventExecutor.execute(() -> processKubernetesExternalLbRemoved(
+ event.subject()));
+ break;
+ default:
+ //do nothing
+ break;
+ }
+ }
+
+ private void processKubernetesExternalLbCreatedOrUpdated(KubernetesExternalLb lb) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (lb.electedGateway() == null || lb.electedWorker() == null) {
+ log.warn("processKubernetesExternalLbCreatedOrUpdated called but electedGateway " +
+ "or electedWorker is null. Stop this task.");
+ return;
+ }
+
+ log.info("processKubernetesExternalLbCreatedOrUpdated and updated elb with elecedGateway: {}", lb);
+
+ setExternalLbRulesForService(lb, true);
+ }
+
+ private void processKubernetesExternalLbGatewayChanged(KubernetesExternalLb lb, String oldGatway) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (lb.electedWorker() == null || oldGatway == null) {
+ return;
+ }
+
+ log.info("processKubernetesExternalLbGatewayChanged with oldateway: {}", oldGatway);
+
+ setExternalLbRulesForService(lb.updateElectedGateway(oldGatway), false);
+
+ setExternalLbRulesForService(lb, true);
+ }
+
+ private void processKubernetesExternalLbWorkerChanged(KubernetesExternalLb lb, String oldWorker) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (lb.electedGateway() == null || oldWorker == null) {
+ return;
+ }
+
+ log.info("processKubernetesExternalLbWorkerChanged with oldworker: {}", oldWorker);
+
+ setExternalLbRulesForService(lb.updateElectedWorker(oldWorker), false);
+
+ setExternalLbRulesForService(lb, true);
+ }
+
+
+ private void processKubernetesExternalLbRemoved(KubernetesExternalLb lb) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (lb.electedGateway() == null) {
+ return;
+ }
+
+ setExternalLbRulesForService(lb, false);
+ }
+ }
+
+ private void setExternalLbRulesForService(KubernetesExternalLb lb, boolean install) {
+ if (lb.electedGateway() == null) {
+ return;
+ }
+
+ KubevirtNode gateway = nodeService.node(lb.electedGateway());
+
+ if (gateway == null) {
+ return;
+ }
+
+ setLoadbalanceIpArpResponseRules(lb, gateway, install);
+ setDownstreamRules(lb, gateway, install);
+ setUpstreamRules(lb, gateway, install);
+ }
+
+ private void setLoadbalanceIpArpResponseRules(KubernetesExternalLb lb, KubevirtNode gateway, boolean install) {
+
+ IpAddress loadBalancerIp = lb.loadBalancerIp();
+
+ if (loadBalancerIp == null) {
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(externalPatchPortNum(deviceService, gateway))
+ .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(loadBalancerIp.getIp4Address())
+ .build();
+
+ Device device = deviceService.getDevice(gateway.intgBridge());
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .extension(RulePopulatorUtil.buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpShaToThaExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpOp(ARP.OP_REPLY)
+ .setEthSrc(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .setArpSha(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .setArpSpa(loadBalancerIp.getIp4Address())
+ .setOutput(PortNumber.IN_PORT)
+ .build();
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_ARP_GATEWAY_RULE,
+ GW_ENTRY_TABLE,
+ install);
+ }
+
+ private void setDownstreamRules(KubernetesExternalLb lb, KubevirtNode gateway, boolean install) {
+
+ IpAddress loadBalancerIp = lb.loadBalancerIp();
+
+ if (loadBalancerIp == null) {
+ log.warn("setDownstreamRules called but loadBalancerIp is null. Stop this task.");
+ return;
+ }
+
+ MacAddress elbIntfMac = kubernetesElbMac(deviceService, gateway);
+ if (elbIntfMac == null) {
+ log.warn("setDownstreamRules called but elbIntfMac is null. Stop this task.");
+ return;
+ }
+
+ PortNumber elbBridgePortNum = elbPatchPortNum(deviceService, gateway);
+ if (elbBridgePortNum == null) {
+ log.warn("setDownstreamRules called but elbBridgePortNum is null. Stop this task.");
+ return;
+ }
+
+ KubernetesExternalLbInterface externalLbInterface = gateway.kubernetesExternalLbInterface();
+ if (externalLbInterface == null) {
+ log.warn("setDownstreamRules called but externalLbInterface is null. Stop this task.");
+ return;
+ }
+
+ KubevirtNode electedWorker = nodeService.node(lb.electedWorker());
+ if (electedWorker == null) {
+ log.warn("setDownstreamRules called but electedWorker is null. Stop this task.");
+ return;
+ }
+
+ lb.nodePortSet().forEach(nodeport -> {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthDst(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .matchIPDst(loadBalancerIp.toIpPrefix())
+ .matchIPProtocol(IPv4.PROTOCOL_TCP)
+ .matchTcpDst(TpPort.tpPort(nodeport.intValue()))
+ .build();
+
+ ExtensionTreatment natTreatment = RulePopulatorUtil
+ .niciraConnTrackTreatmentBuilder(driverService, gateway.intgBridge())
+ .commit(true)
+ .natFlag(CT_NAT_SRC_FLAG)
+ .natAction(true)
+ .natIp(externalLbInterface.externalLbIp())
+ .natPortMin(TpPort.tpPort(TP_PORT_MINIMUM_NUM))
+ .natPortMax(TpPort.tpPort(TP_PORT_MAXIMUM_NUM))
+ .build();
+
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .extension(natTreatment, gateway.intgBridge())
+ .setEthSrc(elbIntfMac)
+ .setEthDst(externalLbInterface.externalLbGwMac())
+ .setIpDst(electedWorker.dataIp())
+ .setOutput(elbBridgePortNum)
+ .build();
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_ELB_DOWNSTREAM_RULE,
+ GW_ENTRY_TABLE,
+ install);
+ });
+ }
+
+ private void setUpstreamRules(KubernetesExternalLb lb, KubevirtNode gateway, boolean install) {
+ IpAddress loadBalancerIp = lb.loadBalancerIp();
+
+ if (loadBalancerIp == null) {
+ log.warn("setUpstreamRules called but loadBalancerIp is null. Stop this task.");
+ return;
+ }
+
+ MacAddress elbIntfMac = kubernetesElbMac(deviceService, gateway);
+ if (elbIntfMac == null) {
+ log.warn("setUpstreamRules called but elbIntfMac is null. Stop this task.");
+ return;
+ }
+
+ PortNumber elbBridgePortNum = elbPatchPortNum(deviceService, gateway);
+ if (elbBridgePortNum == null) {
+ log.warn("setUpstreamRules called but elbBridgePortNum is null. Stop this task.");
+ return;
+ }
+
+ PortNumber externalPatchPortNum = externalPatchPortNum(deviceService, gateway);
+ if (externalPatchPortNum == null) {
+ log.warn("setUpstreamRules called but externalPatchPortNum is null. Stop this task.");
+ return;
+ }
+
+ KubernetesExternalLbInterface externalLbInterface = gateway.kubernetesExternalLbInterface();
+ if (externalLbInterface == null) {
+ log.warn("setUpstreamRules called but externalLbInterface is null. Stop this task.");
+ return;
+ }
+
+
+ KubevirtNode electedWorker = nodeService.node(lb.electedWorker());
+ if (electedWorker == null) {
+ log.warn("setDownstreamRules called but electedWorker is null. Stop this task.");
+ return;
+ }
+
+ lb.nodePortSet().forEach(nodePort -> {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(electedWorker.dataIp().toIpPrefix())
+ .matchIPDst(externalLbInterface.externalLbIp().toIpPrefix())
+ .matchIPProtocol(IPv4.PROTOCOL_TCP)
+ .matchTcpSrc(TpPort.tpPort(nodePort.intValue()));
+
+ ExtensionTreatment natTreatment = RulePopulatorUtil
+ .niciraConnTrackTreatmentBuilder(driverService, gateway.intgBridge())
+ .commit(false)
+ .natAction(true)
+ .table((short) GW_DROP_TABLE)
+ .build();
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setEthSrc(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .setIpSrc(lb.loadBalancerIp())
+ .setEthDst(lb.loadBalancerGwMac())
+ .extension(natTreatment, gateway.intgBridge())
+ .transition(GW_DROP_TABLE);
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_ELB_UPSTREAM_RULE,
+ GW_ENTRY_TABLE,
+ install);
+
+ sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(IPv4.PROTOCOL_TCP)
+ .matchTcpSrc(TpPort.tpPort(nodePort.intValue()));
+
+
+ tBuilder = DefaultTrafficTreatment.builder()
+ .setEthSrc(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .setIpSrc(lb.loadBalancerIp())
+ .setEthDst(lb.loadBalancerGwMac())
+ .setOutput(externalPatchPortNum);
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_ELB_UPSTREAM_RULE,
+ GW_DROP_TABLE,
+ install);
+ });
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
new file mode 100644
index 0000000..2aec742
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
@@ -0,0 +1,529 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.impl;
+
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.api.model.LoadBalancerIngress;
+import io.fabric8.kubernetes.api.model.LoadBalancerStatus;
+import io.fabric8.kubernetes.api.model.Service;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.Watcher;
+import io.fabric8.kubernetes.client.WatcherException;
+import io.fabric8.kubernetes.client.dsl.Resource;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnetworking.api.DefaultKubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbAdminService;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfig;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigListener;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigService;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigEvent;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigListener;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.kubevirtnode.api.KubevirtNodeEvent;
+import org.onosproject.kubevirtnode.api.KubevirtNodeListener;
+import org.onosproject.kubevirtnode.api.KubevirtNodeService;
+import org.onosproject.mastership.MastershipService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedService;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.k8sClient;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.workerNodeForSpecifiedService;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Kubernetes service watcher used for external loadbalancing among PODs.
+ */
+@Component(immediate = true)
+public class KubernetesServiceWatcher {
+ private final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtApiConfigService apiConfigService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbConfigService lbConfigService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbAdminService adminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeService nodeService;
+
+ private static final String KUBE_DASH_VIP = "kube-vip";
+ private static final String KUBE_VIP = "kubevip";
+ private static final String LOADBALANCER_IP = "loadBalancerIP";
+ private static final String TYPE_LOADBALANCER = "LoadBalancer";
+ private static final String KUBE_SYSTEM = "kube-system";
+ private static final String GATEWAY_IP = "gateway-ip";
+ private static final String GATEWAY_MAC = "gateway-mac";
+ private static final String DEFAULT = "default";
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+
+ private final InternalKubevirtApiConfigListener
+ apiConfigListener = new InternalKubevirtApiConfigListener();
+
+ private final InternalKubernetesServiceWatcher
+ serviceWatcher = new InternalKubernetesServiceWatcher();
+
+ private final InternalKubernetesExternalLbConfigListener
+ lbConfigListener = new InternalKubernetesExternalLbConfigListener();
+
+ private final InternalNodeEventListener
+ nodeEventListener = new InternalNodeEventListener();
+
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+
+ apiConfigService.addListener(apiConfigListener);
+ lbConfigService.addListener(lbConfigListener);
+ nodeService.addListener(nodeEventListener);
+
+ log.info("Started");
+ }
+
+
+ @Deactivate
+ protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+
+ apiConfigService.removeListener(apiConfigListener);
+ lbConfigService.removeListener(lbConfigListener);
+ nodeService.removeListener(nodeEventListener);
+
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private void instantiateWatcher() {
+ KubernetesClient client = k8sClient(apiConfigService);
+
+ if (client != null) {
+ client.services().inAnyNamespace().watch(serviceWatcher);
+ }
+ }
+
+ private class InternalKubernetesExternalLbConfigListener
+ implements KubernetesExternalLbConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubernetesExternalLbConfigEvent event) {
+ switch (event.type()) {
+ case KUBERNETES_EXTERNAL_LB_CONFIG_CREATED:
+ case KUBERNETES_EXTERNAL_LB_CONFIG_UPDATED:
+ eventExecutor.execute(this::processConfigUpdate);
+ break;
+ case KUBERNETES_EXTERNAL_LB_CONFIG_REMOVED:
+ default:
+ //do nothing
+ break;
+ }
+ }
+
+ private void processConfigUpdate() {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ addOrUpdateExternalLoadBalancers();
+ }
+ }
+
+ private class InternalKubevirtApiConfigListener implements KubevirtApiConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubevirtApiConfigEvent event) {
+
+ switch (event.type()) {
+ case KUBEVIRT_API_CONFIG_UPDATED:
+ eventExecutor.execute(this::processConfigUpdate);
+ break;
+ case KUBEVIRT_API_CONFIG_CREATED:
+ case KUBEVIRT_API_CONFIG_REMOVED:
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ private void processConfigUpdate() {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ instantiateWatcher();
+ addOrUpdateExternalLoadBalancers();
+ }
+ }
+
+ private class InternalKubernetesServiceWatcher implements Watcher<Service> {
+
+ @Override
+ public void eventReceived(Action action, Service service) {
+ switch (action) {
+ case ADDED:
+ log.info("Service event ADDED received ");
+ eventExecutor.execute(() -> processAddOrMod(service));
+ break;
+ case MODIFIED:
+ log.info("Service event MODIFIED received");
+ eventExecutor.execute(() -> processAddOrMod(service));
+ break;
+ case DELETED:
+ log.info("Service event DELETED received");
+ eventExecutor.execute(() -> processDeletion(service));
+ break;
+ case ERROR:
+ log.warn("Failures processing pod manipulation.");
+ break;
+ default:
+ break;
+ }
+ }
+
+ @Override
+ public void onClose(WatcherException e) {
+ // due to the bugs in fabric8, pod watcher might be closed,
+ // we will re-instantiate the pod watcher in this case
+ // FIXME: https://github.com/fabric8io/kubernetes-client/issues/2135
+ log.warn("Service watcher OnClose, re-instantiate the POD watcher...");
+ instantiateWatcher();
+ }
+
+ private void processAddOrMod(Service service) {
+ if (service == null || !isMaster()) {
+ return;
+ }
+
+ if (!configMapUpdated()) {
+ log.warn("Config map is not set yet. Stop this task");
+ return;
+ }
+
+ try {
+ if (addOrUpdateExternalLoadBalancer(service) &&
+ !isLoadBalancerStatusAlreadySet(service)) {
+ serviceStatusUpdate(service);
+ }
+ } catch (Exception e) {
+ log.error("Exception occurred because of {}", e.toString());
+ }
+ }
+
+ private void processDeletion(Service service) {
+ if (service == null || !isMaster()) {
+ return;
+ }
+ if (isLoadBalancerType(service) &&
+ isKubeVipCloudProviderLabelIsSet(service)) {
+ KubernetesExternalLb lb = adminService.loadBalancer(service.getMetadata().getName());
+
+ if (lb == null) {
+ return;
+ }
+
+ adminService.removeExternalLb(lb.serviceName());
+ }
+ }
+ private boolean isMaster() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+ }
+
+
+ //When api config or configmap updated, check every prerequisite and update all external load balancers
+ private void addOrUpdateExternalLoadBalancers() {
+ KubernetesClient client = k8sClient(apiConfigService);
+
+ client.services().inNamespace(DEFAULT).list()
+ .getItems().forEach(this::addOrUpdateExternalLoadBalancer);
+ }
+
+ private boolean addOrUpdateExternalLoadBalancer(Service service) {
+ if (isLoadBalancerType(service) &&
+ isKubeVipCloudProviderLabelIsSet(service)) {
+
+ KubernetesExternalLb lb = parseKubernetesExternalLb(service);
+ if (lb == null) {
+ log.warn("Failed to parse the kubernetes external lb");
+ return false;
+ }
+
+ KubevirtNode electedGatewayNode = gatewayNodeForSpecifiedService(nodeService, lb);
+ if (electedGatewayNode == null) {
+ log.warn("Service created but there's no gateway nodes ready. Stop this task.");
+ return false;
+ }
+
+ lb = lb.updateElectedGateway(electedGatewayNode.hostname());
+
+ KubevirtNode electedWorkerNode = workerNodeForSpecifiedService(nodeService, lb);
+ if (electedWorkerNode == null) {
+ log.warn("Service created but there's no worker nodes ready. Stop this task.");
+ return false;
+ }
+ lb = lb.updateElectedWorker(electedWorkerNode.hostname());
+
+ log.trace("processAddOrMod called and parsed lb is {}", lb);
+
+ if (adminService.loadBalancer(lb.serviceName()) == null) {
+ adminService.createExternalLb(lb);
+ } else {
+ adminService.updateExternalLb(lb);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private void serviceStatusUpdate(Service service) {
+ KubernetesClient client = k8sClient(apiConfigService);
+
+ String lbIp = service.getSpec().getLoadBalancerIP();
+ if (lbIp == null) {
+ return;
+ }
+
+ LoadBalancerIngress lbIngress = new LoadBalancerIngress(KUBE_VIP, lbIp, Lists.newArrayList());
+
+ service.getStatus().getLoadBalancer().setIngress(Lists.newArrayList(lbIngress));
+
+ //When a service is deleted, the event MODIFED is also along with DELETED event
+ //So filter out this MODIFIED events
+ if (client.services().withName(service.getMetadata().getName()) != null) {
+ client.services().patchStatus(service);
+ }
+ }
+
+ //Only process if the event when the kube-vip-cloud-provider label is set
+ // and loadbalancer status is not set.
+ private boolean isKubeVipCloudProviderLabelIsSet(Service service) {
+ log.trace("isKubeVipCloudProviderLabelIsSet called with labels {}", service.getMetadata().getLabels());
+ if (service.getMetadata().getLabels() == null) {
+ return false;
+ }
+
+ return service.getMetadata().getLabels().containsValue(KUBE_DASH_VIP);
+ }
+
+ private boolean isLoadBalancerStatusAlreadySet(Service service) {
+ log.trace("isLoadBalancerStatusAlreadySet called with status {}", service.getStatus());
+
+ LoadBalancerStatus lbStatus = service.getStatus().getLoadBalancer();
+ if (lbStatus.getIngress().isEmpty()) {
+ return false;
+ }
+
+ String lbIp = service.getSpec().getLoadBalancerIP();
+ if (lbIp == null) {
+ return false;
+ }
+
+ return lbStatus.getIngress().stream()
+ .filter(lbIngress -> Objects.equals(lbIngress.getIp(), lbIp))
+ .findAny().isPresent();
+ }
+
+ private boolean configMapUpdated() {
+ KubernetesExternalLbConfig config = lbConfigService.lbConfigs().stream().findAny().orElse(null);
+
+ if (config == null) {
+ return false;
+ }
+
+ return config.configName() != null && config.globalIpRange() != null &&
+ config.loadBalancerGwIp() != null && config.loadBalancerGwMac() != null;
+ }
+
+ //Only process if the event when the service type is LoadBalancer
+ private boolean isLoadBalancerType(Service service) {
+ return service.getSpec().getType().equals(TYPE_LOADBALANCER);
+ }
+
+ private KubernetesExternalLb parseKubernetesExternalLb(Service service) {
+ if (service.getMetadata() == null || service.getSpec() == null) {
+ return null;
+ }
+
+ String serviceName = service.getMetadata().getName();
+ if (serviceName == null) {
+ return null;
+ }
+
+ String lbIp = service.getSpec().getLoadBalancerIP();
+ if (lbIp == null) {
+ return null;
+ }
+
+ Set<Integer> nodePortSet = Sets.newHashSet();
+ Set<Integer> portSet = Sets.newHashSet();
+ Set<String> endpointSet = Sets.newHashSet();
+
+ service.getSpec().getPorts().forEach(servicePort -> {
+ nodePortSet.add(servicePort.getNodePort());
+ portSet.add(servicePort.getPort());
+ });
+
+ nodeService.completeNodes(WORKER).forEach(workerNode -> {
+ endpointSet.add(workerNode.dataIp().toString());
+ });
+
+ String loadbalancerGatewayIp = loadBalancerGatewayIp();
+
+ if (loadbalancerGatewayIp == null) {
+ log.error("Can't find the loadbalancer gateway ip in the kubevip configmap.." +
+ "Failed to parse kubernetes external lb and return null");
+ return null;
+ }
+
+ MacAddress loadBalancerGatewayMac = loadBalancerGatewayMac();
+
+ if (loadbalancerGatewayIp == null) {
+ log.error("Can't find the loadbalancer gateway mac in the kubevip configmap.." +
+ "Failed to parse kubernetes external lb and return null");
+ return null;
+ }
+
+ return DefaultKubernetesExternalLb.builder().serviceName(serviceName)
+ .loadBalancerIp(IpAddress.valueOf(lbIp))
+ .nodePortSet(nodePortSet)
+ .portSet(portSet)
+ .endpointSet(endpointSet)
+ .loadBalancerGwIp(IpAddress.valueOf(loadbalancerGatewayIp))
+ .loadBalancerGwMac(loadBalancerGatewayMac)
+ .build();
+ }
+
+ private String loadBalancerGatewayIp() {
+ KubernetesClient client = k8sClient(apiConfigService);
+
+ Resource<ConfigMap> kubeVipConfigMapResource =
+ client.configMaps().inNamespace(KUBE_SYSTEM).withName(KUBE_VIP);
+
+ if (kubeVipConfigMapResource == null) {
+ return null;
+ }
+
+ Map<String, String> kubeVipConfigMap = kubeVipConfigMapResource.get().getData();
+
+ if (!kubeVipConfigMap.containsKey(GATEWAY_IP)) {
+ return null;
+ }
+
+ return kubeVipConfigMap.get(GATEWAY_IP);
+ }
+
+ private MacAddress loadBalancerGatewayMac() {
+ KubernetesExternalLbConfig config = lbConfigService.lbConfigs().stream().findAny().orElse(null);
+
+ if (config == null) {
+ return null;
+ }
+
+ return config.loadBalancerGwMac();
+ }
+
+ private class InternalNodeEventListener implements KubevirtNodeListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubevirtNodeEvent event) {
+ switch (event.type()) {
+ case KUBEVIRT_NODE_COMPLETE:
+ eventExecutor.execute(() -> processNodeCompletion(event.subject()));
+ break;
+ case KUBEVIRT_NODE_INCOMPLETE:
+ case KUBEVIRT_NODE_REMOVED:
+ eventExecutor.execute(() -> processNodeDeletion(event.subject()));
+ break;
+ case KUBEVIRT_NODE_UPDATED:
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ private void processNodeCompletion(KubevirtNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ addOrUpdateExternalLoadBalancers();
+ }
+
+ private void processNodeDeletion(KubevirtNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ addOrUpdateExternalLoadBalancers();
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
index 5bc1548..1d0a2e9 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
@@ -45,6 +45,8 @@
import org.onosproject.kubevirtnetworking.api.KubevirtRouterAdminService;
import org.onosproject.kubevirtnetworking.api.KubevirtRouterEvent;
import org.onosproject.kubevirtnetworking.api.KubevirtRouterListener;
+import org.onosproject.kubevirtnetworking.util.RulePopulatorUtil;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtNodeEvent;
@@ -109,10 +111,12 @@
import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.FLAT;
import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.VLAN;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.elbPatchPortNum;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedRouter;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtNetwork;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtPort;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterMacAddress;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.kubernetesElbMac;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.portNumber;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.resolveHostname;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.segmentIdHex;
@@ -1324,6 +1328,12 @@
}
} else if (node.type().equals(GATEWAY)) {
updateGatewayNodeForRouter();
+
+ KubernetesExternalLbInterface externalLbInterface = node.kubernetesExternalLbInterface();
+
+ if (externalLbInterface != null) {
+ setElbInternalIpArpResponseRules(node, true);
+ }
}
}
@@ -1358,6 +1368,49 @@
kubevirtRouterService.updateRouter(router.updatedElectedGateway(newGwNode.hostname()));
});
}
+
+ private void setElbInternalIpArpResponseRules(KubevirtNode gateway, boolean install) {
+ KubernetesExternalLbInterface externalLbInterface = gateway.kubernetesExternalLbInterface();
+ if (externalLbInterface == null) {
+ return;
+ }
+
+ IpAddress eIp = externalLbInterface.externalLbIp();
+
+ MacAddress elbIntfMac = kubernetesElbMac(deviceService, gateway);
+ if (elbIntfMac == null) {
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(elbPatchPortNum(deviceService, gateway))
+ .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(eIp.getIp4Address())
+ .build();
+
+ Device device = deviceService.getDevice(gateway.intgBridge());
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .extension(RulePopulatorUtil.buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpShaToThaExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpOp(ARP.OP_REPLY)
+ .setEthSrc(elbIntfMac)
+ .setArpSha(elbIntfMac)
+ .setArpSpa(eIp.getIp4Address())
+ .setOutput(PortNumber.IN_PORT)
+ .build();
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_ARP_GATEWAY_RULE,
+ GW_ENTRY_TABLE,
+ install);
+ }
}
private class InternalKubevirtPortListener implements KubevirtPortListener {
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
index 3496008..539323f 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
@@ -53,6 +53,7 @@
import org.onosproject.kubevirtnetworking.api.KubevirtRouterService;
import org.onosproject.kubevirtnode.api.DefaultKubevirtNode;
import org.onosproject.kubevirtnode.api.DefaultKubevirtPhyInterface;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtApiConfig;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
import org.onosproject.kubevirtnode.api.KubevirtNode;
@@ -89,6 +90,7 @@
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.MASTER;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.OTHER;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
+import static org.onosproject.net.AnnotationKeys.PORT_MAC;
import static org.onosproject.net.AnnotationKeys.PORT_NAME;
/**
@@ -545,7 +547,7 @@
/**
* Returns the gateway node for the specified kubernetes external lb.
* Among gateways, only one gateway would act as a gateway per external lb.
- * Currently gateway node is selected based on modulo operation with router hashcode.
+ * Currently gateway node is selected based on modulo operation with external lb hashcode.
*
* @param nodeService kubevirt node service
* @param externalLb kubernetes external lb
@@ -565,6 +567,28 @@
}
/**
+ * Returns the worker node for the specified kubernetes external lb.
+ * Among worker nodes, only one worker would serve the traffic from and to the gateway.
+ * Currently worker node is selected based on modulo operation with external lb hashcode.
+ *
+ * @param nodeService kubevirt node service
+ * @param externalLb kubernetes external lb
+ * @return elected worker node
+ */
+ public static KubevirtNode workerNodeForSpecifiedService(KubevirtNodeService nodeService,
+ KubernetesExternalLb externalLb) {
+ //TODO: enhance election logic for a better load balancing
+
+ int numOfWorkers = nodeService.completeNodes(WORKER).size();
+ if (numOfWorkers == 0) {
+ return null;
+ }
+
+ return (KubevirtNode) nodeService.completeNodes(WORKER)
+ .toArray()[externalLb.hashCode() % numOfWorkers];
+ }
+
+ /**
* Returns the mac address of the router.
*
* @param router kubevirt router
@@ -659,6 +683,73 @@
}
/**
+ * Returns the external lb patch port number with specified gateway.
+ *
+ * @param deviceService device service
+ * @param gateway gateway node
+ * @return external lb bridge patch port number
+ */
+ public static PortNumber elbPatchPortNum(DeviceService deviceService, KubevirtNode gateway) {
+ KubernetesExternalLbInterface kubernetesExternalLbInterface =
+ gateway.kubernetesExternalLbInterface();
+
+ if (kubernetesExternalLbInterface == null) {
+ log.warn("No elb interface is attached to gateway {}", gateway.hostname());
+ return null;
+ }
+
+ String elbBridgeName = kubernetesExternalLbInterface.externalLbBridgeName();
+
+ String patchPortName = "int-to-" + elbBridgeName;
+
+ Port port = deviceService.getPorts(gateway.intgBridge()).stream()
+ .filter(p -> p.isEnabled() &&
+ Objects.equals(p.annotations().value(PORT_NAME), patchPortName))
+ .findAny().orElse(null);
+
+ return port != null ? port.number() : null;
+ }
+
+ /**
+ * Returns the external lb patch port Mac with specified gateway.
+ *
+ * @param deviceService device service
+ * @param gateway gateway node
+ * @return external lb bridge patch Mac Address
+ */
+ public static MacAddress kubernetesElbMac(DeviceService deviceService, KubevirtNode gateway) {
+
+ KubernetesExternalLbInterface kubernetesExternalLbInterface =
+ gateway.kubernetesExternalLbInterface();
+
+ if (kubernetesExternalLbInterface == null) {
+ log.warn("No elb interface is attached to gateway {}", gateway.hostname());
+ return null;
+ }
+
+ String elbBridgeName = kubernetesExternalLbInterface.externalLbBridgeName();
+
+ String patchPortName = "int-to-" + elbBridgeName;
+
+ Port port = deviceService.getPorts(gateway.intgBridge()).stream()
+ .filter(p -> p.isEnabled() &&
+ Objects.equals(p.annotations().value(PORT_NAME), patchPortName))
+ .findAny().orElse(null);
+
+ if (port == null) {
+ return null;
+ }
+
+ String portMacStr = port.annotations().value(PORT_MAC);
+
+ if (portMacStr == null) {
+ return null;
+ }
+
+ return MacAddress.valueOf(portMacStr);
+ }
+
+ /**
* Returns the kubevirt external network with specified router.
*
* @param networkService kubevirt network service
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/Constants.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/Constants.java
index 3d2f14c..0957fb7 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/Constants.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/Constants.java
@@ -49,4 +49,11 @@
public static final String DEFAULT_CLUSTER_NAME = "default";
public static final String SONA_PROJECT_DOMAIN = "sonaproject.github.io";
+
+ // CLI item length
+ public static final int CLI_NAME_LENGTH = 30;
+ public static final int CLI_IP_ADDRESSES_LENGTH = 50;
+ public static final int CLI_IP_ADDRESS_LENGTH = 25;
+ public static final int CLI_MAC_ADDRESS_LENGTH = 25;
+ public static final int CLI_MARGIN_LENGTH = 2;
}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbConfig.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbConfig.java
new file mode 100644
index 0000000..1147ac7
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbConfig.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+import com.google.common.base.MoreObjects;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+
+import java.util.Objects;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * Default implementation of kubernetes external lb config.
+ */
+public final class DefaultKubernetesExternalLbConfig implements KubernetesExternalLbConfig {
+
+ private static final String NOT_NULL_MSG = "ExternalLbConfig % cannot be null";
+
+ private final String configName;
+ private final IpAddress loadBalancerGwIp;
+ private final MacAddress loadBalancerGwMac;
+ private final String globalIpRange;
+
+ public DefaultKubernetesExternalLbConfig(String configName, IpAddress loadBalancerGwIp,
+ MacAddress loadBalancerGwMac, String globalIpRange) {
+ this.configName = configName;
+ this.loadBalancerGwIp = loadBalancerGwIp;
+ this.loadBalancerGwMac = loadBalancerGwMac;
+ this.globalIpRange = globalIpRange;
+ }
+
+ @Override
+ public String configName() {
+ return configName;
+ }
+
+ @Override
+ public IpAddress loadBalancerGwIp() {
+ return loadBalancerGwIp;
+ }
+
+ @Override
+ public MacAddress loadBalancerGwMac() {
+ return loadBalancerGwMac;
+ }
+
+ @Override
+ public String globalIpRange() {
+ return globalIpRange;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ DefaultKubernetesExternalLbConfig that = (DefaultKubernetesExternalLbConfig) o;
+
+ return Objects.equals(configName, that.configName) &&
+ Objects.equals(loadBalancerGwIp, that.loadBalancerGwIp) &&
+ Objects.equals(globalIpRange, that.globalIpRange);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(configName, loadBalancerGwIp, globalIpRange);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("configName", configName)
+ .add("loadBalancerGwIp", loadBalancerGwIp)
+ .add("loadBalancerGwMac", loadBalancerGwMac)
+ .add("globalIpRange", globalIpRange)
+ .toString();
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ @Override
+ public KubernetesExternalLbConfig updateLbGatewayMac(MacAddress gatewayMac) {
+ return DefaultKubernetesExternalLbConfig.builder()
+ .configName(configName)
+ .loadBalancerGwIp(loadBalancerGwIp)
+ .loadBalancerGwMac(loadBalancerGwMac)
+ .globalIpRange(globalIpRange)
+ .build();
+ }
+
+ public static final class Builder implements KubernetesExternalLbConfig.Builder {
+
+ private String configName;
+ private IpAddress loadBalancerGwIp;
+ private MacAddress loadBalancerGwMac;
+ private String globalIpRange;
+
+ private Builder() {
+ }
+
+ @Override
+ public KubernetesExternalLbConfig build() {
+ checkArgument(configName != null, NOT_NULL_MSG, "configName");
+ checkArgument(loadBalancerGwIp != null, NOT_NULL_MSG, "loadBalancerGwIp");
+ checkArgument(globalIpRange != null, NOT_NULL_MSG, "globalIpRange");
+
+ return new DefaultKubernetesExternalLbConfig(configName, loadBalancerGwIp,
+ loadBalancerGwMac, globalIpRange);
+ }
+
+ @Override
+ public Builder configName(String configName) {
+ this.configName = configName;
+ return this;
+ }
+
+ @Override
+ public Builder loadBalancerGwIp(IpAddress loadBalancerGwIp) {
+ this.loadBalancerGwIp = loadBalancerGwIp;
+ return this;
+ }
+
+ @Override
+ public KubernetesExternalLbConfig.Builder loadBalancerGwMac(MacAddress loadBalancerGwMac) {
+ this.loadBalancerGwMac = loadBalancerGwMac;
+ return this;
+ }
+
+ @Override
+ public Builder globalIpRange(String globalIpRange) {
+ this.globalIpRange = globalIpRange;
+ return this;
+ }
+ }
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbInterface.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbInterface.java
new file mode 100644
index 0000000..fe53b53
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbInterface.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+import com.google.common.base.MoreObjects;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+
+import java.util.Objects;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+public class DefaultKubernetesExternalLbInterface implements KubernetesExternalLbInterface {
+
+ private static final String NOT_NULL_MSG = "KubernetesExternalLbInterface % cannot be null";
+
+ private String elbBridgeName;
+ private IpAddress elbIp;
+ private IpAddress elbGwIp;
+ private MacAddress elbGwMac;
+
+ public DefaultKubernetesExternalLbInterface(String elbBridgeName, IpAddress elbIp,
+ IpAddress elbGwIp, MacAddress elbGwMac) {
+ this.elbBridgeName = elbBridgeName;
+ this.elbIp = elbIp;
+ this.elbGwIp = elbGwIp;
+ this.elbGwMac = elbGwMac;
+ }
+
+ @Override
+ public String externalLbBridgeName() {
+ return elbBridgeName;
+ }
+
+ @Override
+ public IpAddress externalLbIp() {
+ return elbIp;
+ }
+
+ @Override
+ public IpAddress externalLbGwIp() {
+ return elbGwIp;
+ }
+
+ @Override
+ public MacAddress externalLbGwMac() {
+ return elbGwMac;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ DefaultKubernetesExternalLbInterface that = (DefaultKubernetesExternalLbInterface) o;
+
+ return Objects.equals(elbBridgeName, that.elbBridgeName) &&
+ Objects.equals(elbIp, that.elbIp) &&
+ Objects.equals(elbGwIp, that.elbGwIp);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("elbBridgeName", elbBridgeName)
+ .add("elbIp", elbIp)
+ .add("elbGwIp", elbGwIp)
+ .add("elbGwMac", elbGwMac)
+ .toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(elbBridgeName, elbIp, elbGwIp);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static final class Builder implements KubernetesExternalLbInterface.Builder {
+ private String elbBridgeName;
+ private IpAddress elbIp;
+ private IpAddress elbGwIp;
+ private MacAddress elbGwMac;
+
+ private Builder() {
+ }
+
+ @Override
+ public KubernetesExternalLbInterface build() {
+ checkArgument(elbBridgeName != null, NOT_NULL_MSG, "externalLbBridgeName");
+ checkArgument(elbIp != null, NOT_NULL_MSG, "externalLbIp");
+ checkArgument(elbGwIp != null, NOT_NULL_MSG, "externalLbGwIp");
+ checkArgument(elbGwMac != null, NOT_NULL_MSG, "externalLbGwMac");
+
+ return new DefaultKubernetesExternalLbInterface(elbBridgeName, elbIp, elbGwIp, elbGwMac);
+ }
+
+ @Override
+ public Builder externalLbBridgeName(String elbBridgeName) {
+ this.elbBridgeName = elbBridgeName;
+ return this;
+ }
+
+ @Override
+ public Builder externalLbIp(IpAddress elbIp) {
+ this.elbIp = elbIp;
+ return this;
+ }
+
+ @Override
+ public Builder externallbGwIp(IpAddress elbGwIp) {
+ this.elbGwIp = elbGwIp;
+ return this;
+ }
+
+ @Override
+ public Builder externalLbGwMac(MacAddress elbGwMac) {
+ this.elbGwMac = elbGwMac;
+ return this;
+ }
+ }
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java
index 16f7034..2c3d0be 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java
@@ -59,10 +59,7 @@
private final KubevirtNodeState state;
private final Collection<KubevirtPhyInterface> phyIntfs;
private final String gatewayBridgeName;
- private final String elbBridgeName;
- private final IpAddress elbIp;
- private final IpAddress elbGwIp;
- private final MacAddress elbGwMac;
+ private final KubernetesExternalLbInterface kubernetesExternalLbIntf;
/**
* A default constructor of kubevirt node.
@@ -77,18 +74,15 @@
* @param state node state
* @param phyIntfs physical interfaces
* @param gatewayBridgeName gateway bridge name
- * @param elbBridgeName elb bridge name
- * @param elbIp elb IP address
- * @param elbGwIp elb gw IP address
- * @param elbGwMac elb gw MAC address
+ * @param kubernetesExternalLbIntf kubernetesExternalLbIntf
*/
protected DefaultKubevirtNode(String clusterName, String hostname, Type type,
DeviceId intgBridge, DeviceId tunBridge,
IpAddress managementIp, IpAddress dataIp,
KubevirtNodeState state,
Collection<KubevirtPhyInterface> phyIntfs,
- String gatewayBridgeName, String elbBridgeName, IpAddress elbIp,
- IpAddress elbGwIp, MacAddress elbGwMac) {
+ String gatewayBridgeName,
+ KubernetesExternalLbInterface kubernetesExternalLbIntf) {
this.clusterName = clusterName;
this.hostname = hostname;
this.type = type;
@@ -99,10 +93,7 @@
this.state = state;
this.phyIntfs = phyIntfs;
this.gatewayBridgeName = gatewayBridgeName;
- this.elbBridgeName = elbBridgeName;
- this.elbIp = elbIp;
- this.elbGwIp = elbGwIp;
- this.elbGwMac = elbGwMac;
+ this.kubernetesExternalLbIntf = kubernetesExternalLbIntf;
}
@Override
@@ -163,10 +154,7 @@
.state(newState)
.phyIntfs(phyIntfs)
.gatewayBridgeName(gatewayBridgeName)
- .elbBridgeName(elbBridgeName)
- .elbIp(elbIp)
- .elbGwIp(elbGwIp)
- .elbGwMac(elbGwMac)
+ .kubernetesExternalLbInterface(kubernetesExternalLbIntf)
.build();
}
@@ -183,10 +171,7 @@
.state(state)
.phyIntfs(phyIntfs)
.gatewayBridgeName(gatewayBridgeName)
- .elbBridgeName(elbBridgeName)
- .elbIp(elbIp)
- .elbGwIp(elbGwIp)
- .elbGwMac(elbGwMac)
+ .kubernetesExternalLbInterface(kubernetesExternalLbIntf)
.build();
}
@@ -203,10 +188,7 @@
.state(state)
.phyIntfs(phyIntfs)
.gatewayBridgeName(gatewayBridgeName)
- .elbBridgeName(elbBridgeName)
- .elbIp(elbIp)
- .elbGwIp(elbGwIp)
- .elbGwMac(elbGwMac)
+ .kubernetesExternalLbInterface(kubernetesExternalLbIntf)
.build();
}
@@ -259,23 +241,8 @@
}
@Override
- public String elbBridgeName() {
- return elbBridgeName;
- }
-
- @Override
- public IpAddress elbIp() {
- return elbIp;
- }
-
- @Override
- public IpAddress elbGwIp() {
- return elbGwIp;
- }
-
- @Override
- public MacAddress elbGwMac() {
- return elbGwMac;
+ public KubernetesExternalLbInterface kubernetesExternalLbInterface() {
+ return kubernetesExternalLbIntf;
}
private PortNumber tunnelPort(String tunnelType) {
@@ -343,10 +310,7 @@
.state(node.state())
.phyIntfs(node.phyIntfs())
.gatewayBridgeName(node.gatewayBridgeName())
- .elbBridgeName(node.elbBridgeName())
- .elbIp(node.elbIp())
- .elbGwIp(node.elbGwIp())
- .elbGwMac(node.elbGwMac());
+ .kubernetesExternalLbInterface(node.kubernetesExternalLbInterface());
}
@Override
@@ -386,10 +350,7 @@
.add("state", state)
.add("phyIntfs", phyIntfs)
.add("gatewayBridgeName", gatewayBridgeName)
- .add("elbBridgeName", elbBridgeName)
- .add("elbIp", elbIp)
- .add("elbGwIp", elbGwIp)
- .add("elbGwMac", elbGwMac)
+ .add("kubernetesExternalLbInterface", kubernetesExternalLbIntf)
.toString();
}
@@ -409,6 +370,7 @@
private IpAddress elbIp;
private IpAddress elbGwIp;
private MacAddress elbGwMac;
+ private KubernetesExternalLbInterface kubernetesExternalLbInterface;
// private constructor not intended to use from external
private Builder() {
@@ -436,10 +398,7 @@
state,
phyIntfs,
gatewayBridgeName,
- elbBridgeName,
- elbIp,
- elbGwIp,
- elbGwMac
+ kubernetesExternalLbInterface
);
}
@@ -504,26 +463,8 @@
}
@Override
- public Builder elbBridgeName(String elbBridgeName) {
- this.elbBridgeName = elbBridgeName;
- return this;
- }
-
- @Override
- public Builder elbIp(IpAddress elbIp) {
- this.elbIp = elbIp;
- return this;
- }
-
- @Override
- public Builder elbGwIp(IpAddress elbGwIp) {
- this.elbGwIp = elbGwIp;
- return this;
- }
-
- @Override
- public Builder elbGwMac(MacAddress elbGwMac) {
- this.elbGwMac = elbGwMac;
+ public Builder kubernetesExternalLbInterface(KubernetesExternalLbInterface kubernetesExternalLbInterface) {
+ this.kubernetesExternalLbInterface = kubernetesExternalLbInterface;
return this;
}
}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfig.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfig.java
new file mode 100644
index 0000000..a0f1e90
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfig.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+
+/**
+ * Representation of configuration used in Kubernetes External Lb service.
+ */
+public interface KubernetesExternalLbConfig {
+
+ /**
+ * Returns the name of kubernetes external lb config.
+ * This is defined in the configmap.
+ *
+ * @return config name
+ */
+ String configName();
+ /**
+ * Returns the gateway IP of load balancer.
+ * TEG would send outbound traffic to this gateway.
+ *
+ * @return load balancer gateway IP
+ */
+ IpAddress loadBalancerGwIp();
+
+ /**
+ * Returns the gateway MAC of load balancer.
+ * TEG would send outbound traffic to this gateway.
+ *
+ * @return load balancer gateway IP
+ */
+ MacAddress loadBalancerGwMac();
+
+ /**
+ * Returns the global IP range used in external LB.
+ * Each service of type LoadBalancer would get the public IP out of those.
+ * Format: "223.39.6.85-223.39.6.90"
+ *
+ * @return global Ip range
+ */
+ String globalIpRange();
+
+ /**
+ * Returns the KubernetesExternalLbConfig with updated external lb gateway mac address.
+ *
+ * @param gatewayMac external lb gateway mac address
+ * @return KubernetesExternalLbConfig
+ */
+ KubernetesExternalLbConfig updateLbGatewayMac(MacAddress gatewayMac);
+
+ interface Builder {
+ /**
+ * Builds an immutable kubernal external lb config instance.
+ *
+ * @return kubernetes external lb config
+ */
+ KubernetesExternalLbConfig build();
+
+ /**
+ * Returns kubernetes external lb config builder with supplied config name.
+ *
+ * @param configName config name
+ * @return kubernetes external lb config builder
+ */
+ Builder configName(String configName);
+
+ /**
+ * Returns kubernetes external lb config builder with supplied loadbalancer gw Ip.
+ *
+ * @param loadBalancerGwIp loadbalancer gw Ip
+ * @return kubernetes external lb config builder
+ */
+ Builder loadBalancerGwIp(IpAddress loadBalancerGwIp);
+
+ /**
+ * Returns kubernetes external lb config builder with supplied loadbalancer gw Mac.
+ *
+ * @param loadBalancerGwMac loadbalancer gw Mac
+ * @return kubernetes external lb config builder
+ */
+ Builder loadBalancerGwMac(MacAddress loadBalancerGwMac);
+
+ /**
+ * Returns kubernetes external lb config builder with supplied global Ip range.
+ *
+ * @param globalIpRange global Ip range
+ * @return kubernetes external lb config builder
+ */
+ Builder globalIpRange(String globalIpRange);
+
+ }
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigAdminService.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigAdminService.java
new file mode 100644
index 0000000..36e163f
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigAdminService.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+/**
+ * Service for administering inventory of Kubernetes External Lb Configs.
+ */
+public interface KubernetesExternalLbConfigAdminService extends KubernetesExternalLbConfigService {
+
+ /**
+ * Creates a new kubernetes external lb config.
+ *
+ * @param lbConfig kubernetes external lb config
+ */
+ void createKubernetesExternalLbConfig(KubernetesExternalLbConfig lbConfig);
+
+ /**
+ * Updates a new kubernetes external lb config.
+ *
+ * @param lbConfig kubernetes external lb config
+ */
+ void updateKubernetesExternalLbConfig(KubernetesExternalLbConfig lbConfig);
+
+ /**
+ * Removes a new kubernetes external lb config.
+ *
+ * @param configName kubernetes external lb config
+ */
+ void removeKubernetesExternalLbConfig(String configName);
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigEvent.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigEvent.java
new file mode 100644
index 0000000..c42ed31
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigEvent.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+import org.onosproject.event.AbstractEvent;
+
+public class KubernetesExternalLbConfigEvent
+ extends AbstractEvent<KubernetesExternalLbConfigEvent.Type, KubernetesExternalLbConfig> {
+
+ public KubernetesExternalLbConfigEvent(Type type, KubernetesExternalLbConfig subject) {
+ super(type, subject);
+ }
+
+ /**
+ * Kubernetes external lb config events.
+ */
+ public enum Type {
+ /**
+ * Signifies that a new config is created.
+ */
+ KUBERNETES_EXTERNAL_LB_CONFIG_CREATED,
+
+ /**
+ * Signifies that a new config is updated.
+ */
+ KUBERNETES_EXTERNAL_LB_CONFIG_UPDATED,
+
+ /**
+ * Signifies that a new config is removed.
+ */
+ KUBERNETES_EXTERNAL_LB_CONFIG_REMOVED
+ }
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigListener.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigListener.java
new file mode 100644
index 0000000..a91d048
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigListener.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+import org.onosproject.event.EventListener;
+
+/**
+ * Listener for kubernetes external lb config event.
+ */
+public interface KubernetesExternalLbConfigListener extends EventListener<KubernetesExternalLbConfigEvent> {
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigService.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigService.java
new file mode 100644
index 0000000..32e07e5
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigService.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+import org.onosproject.event.ListenerService;
+
+import java.util.Set;
+
+/**
+ * Service for administering the inventory of kubernetes external lb configs.
+ */
+public interface KubernetesExternalLbConfigService
+ extends ListenerService<KubernetesExternalLbConfigEvent, KubernetesExternalLbConfigListener> {
+ String APP_ID = "org.onosproject.kubevirtnode";
+
+ /**
+ * Returns the kubernetes external lb config with the given config name.
+ *
+ * @param configName config name
+ * @return kubernetes external lb config
+ */
+ KubernetesExternalLbConfig lbConfig(String configName);
+
+ /**
+ * Returns all kubernetes external lb configs.
+ *
+ * @return set of kubernetes external lb configs
+ */
+ Set<KubernetesExternalLbConfig> lbConfigs();
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigStore.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigStore.java
new file mode 100644
index 0000000..9a1e366
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigStore.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+import org.onosproject.store.Store;
+
+import java.util.Set;
+
+/**
+ * Manages inventory of kubernetes external lb config states; not intended for direct use.
+ */
+public interface KubernetesExternalLbConfigStore
+ extends Store<KubernetesExternalLbConfigEvent, KubernetesExternalLbConfigStoreDelegate> {
+
+ /**
+ * Creates a new kubernetes external lb config.
+ *
+ * @param lbConfig kubernetes external lb config
+ */
+ void createExternalLbConfig(KubernetesExternalLbConfig lbConfig);
+
+ /**
+ * Updates a new kubernetes external lb config.
+ *
+ * @param lbConfig kubernetes external lb config
+ */
+ void updateExternalLbConfig(KubernetesExternalLbConfig lbConfig);
+
+ /**
+ * Removes a new kubernetes external lb config.
+ *
+ * @param configName kubernetes external lb config
+ * @return removed kubernetes external lb config
+ */
+ KubernetesExternalLbConfig removeExternalLbConfig(String configName);
+
+ /**
+ * Returns the kubernetes external lb config with the given config name.
+ *
+ * @param configName config name
+ * @return kubernetes external lb config
+ */
+ KubernetesExternalLbConfig externalLbConfig(String configName);
+
+ /**
+ * Returns all kubernetes external lb configs.
+ *
+ * @return set of kubernetes external lb configs
+ */
+ Set<KubernetesExternalLbConfig> externalLbConfigs();
+
+ /**
+ * Removes all kubernetes external lb configs.
+ */
+ void clear();
+
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigStoreDelegate.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigStoreDelegate.java
new file mode 100644
index 0000000..12e5e11
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbConfigStoreDelegate.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+import org.onosproject.store.StoreDelegate;
+
+/**
+ * Kubernetes external lb config store delegate abstraction.
+ */
+public interface KubernetesExternalLbConfigStoreDelegate extends StoreDelegate<KubernetesExternalLbConfigEvent> {
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbInterface.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbInterface.java
new file mode 100644
index 0000000..df9081f
--- /dev/null
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubernetesExternalLbInterface.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.api;
+
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+
+/**
+ * Representation of a Kubernetes external load balancer interface for kubevirt node.
+ */
+public interface KubernetesExternalLbInterface {
+
+ /**
+ * Returns the name of the elb bridge.
+ * Using this bridge, TEG internally communicates with data IP's in worker nodes.
+ *
+ * @return gateway bridge name
+ */
+ String externalLbBridgeName();
+
+ /**
+ * Returns the internal Ip Address of TEG for kubernetes external lb purpose.
+ *
+ * @return elb ip address
+ */
+ IpAddress externalLbIp();
+
+ /**
+ * Returns the gateway IP of the elb IP.
+ *
+ * @return elb gw ip address
+ */
+ IpAddress externalLbGwIp();
+
+ /**
+ * Returns the mac address of the elb gw.
+ *
+ * @return elb gw mac address
+ */
+ MacAddress externalLbGwMac();
+
+
+ interface Builder {
+
+ /**
+ * Builds an immutable kubernetes external load balancer interface instance.
+ *
+ * @return external load balancer interface instance
+ */
+ KubernetesExternalLbInterface build();
+
+ /**
+ * Returns kubernetes external load balancer interface builder with supplied elb bridge name.
+ *
+ * @param elbBridgeName elb bridge name
+ * @return kubernetes external load balancer interface builder
+ */
+ Builder externalLbBridgeName(String elbBridgeName);
+
+ /**
+ * Returns kubernetes external load balancer interface builder with supplied supplied elb Ip address.
+ *
+ * @param elbIp elb ip address
+ * @return kubernetes external load balancer interface builder
+ */
+ Builder externalLbIp(IpAddress elbIp);
+
+ /**
+ * Returns kubernetes external load balancer interface builder with supplied supplied elb gw Ip address.
+ *
+ * @param elbGwIp elb gw ip address
+ * @return kubernetes external load balancer interface builder
+ */
+ Builder externallbGwIp(IpAddress elbGwIp);
+
+ /**
+ * Returns kubernetes external load balancer interface builder with supplied supplied elb gw MAC address.
+ *
+ * @param elbGwMac elb gw mac address
+ * @return kubernetes external load balancer interface builder
+ */
+ Builder externalLbGwMac(MacAddress elbGwMac);
+ }
+}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java
index 0015105..502784d 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java
@@ -16,7 +16,6 @@
package org.onosproject.kubevirtnode.api;
import org.onlab.packet.IpAddress;
-import org.onlab.packet.MacAddress;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
@@ -190,34 +189,11 @@
String gatewayBridgeName();
/**
- * Returns the name of the elb bridge.
- * Using this bridge, TEG internally communicates with data IP's in worker nodes.
+ * Returns the kubernetesExternalLbInterface.
*
- * @return gateway bridge name
+ * @return kubernetesExternalLbInterface
*/
- String elbBridgeName();
-
- /**
- * Returns the internal Ip Address of TEG for kubernetes external lb purpose.
- *
- * @return elb ip address
- */
- IpAddress elbIp();
-
- /**
- * Returns the gateway IP of the elb IP.
- *
- * @return elb gw ip address
- */
- IpAddress elbGwIp();
-
- /**
- * Returns the mac address of the elb gw.
- *
- * @return elb gw mac address
- */
- MacAddress elbGwMac();
-
+ KubernetesExternalLbInterface kubernetesExternalLbInterface();
/**
* Builder of new node entity.
@@ -310,36 +286,13 @@
*/
KubevirtNode.Builder gatewayBridgeName(String gatewayBridgeName);
- /**
- * Returns kubevirt node builder with supplied elb bridge name.
- *
- * @param elbBridgeName elb bridge name
- * @return kubevirt node builder
- */
- KubevirtNode.Builder elbBridgeName(String elbBridgeName);
/**
- * Returns kubevirt node builder with supplied supplied elb Ip address.
+ * Returns kubevirt node builder with supplied supplied kubernetesExternalLbInterface.
*
- * @param elbIp elb ip address
+ * @param kubernetesExternalLbInterface kubernetesExternalLbInterface
* @return kubevirt node builder
*/
- KubevirtNode.Builder elbIp(IpAddress elbIp);
-
- /**
- * Returns kubevirt node builder with supplied supplied elb gw Ip address.
- *
- * @param elbGwIp elb gw ip address
- * @return kubevirt node builder
- */
- KubevirtNode.Builder elbGwIp(IpAddress elbGwIp);
-
- /**
- * Returns kubevirt node builder with supplied supplied elb gw MAC address.
- *
- * @param elbGwMac elb gw mac address
- * @return kubevirt node builder
- */
- KubevirtNode.Builder elbGwMac(MacAddress elbGwMac);
+ KubevirtNode.Builder kubernetesExternalLbInterface(KubernetesExternalLbInterface kubernetesExternalLbInterface);
}
}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/cli/KubernetesExternalLbConfigCommand.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/cli/KubernetesExternalLbConfigCommand.java
new file mode 100644
index 0000000..6eeb220
--- /dev/null
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/cli/KubernetesExternalLbConfigCommand.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.cli;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang.StringUtils;
+import org.apache.karaf.shell.api.action.Command;
+import org.apache.karaf.shell.api.action.lifecycle.Service;
+import org.onlab.packet.IpAddress;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfig;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigService;
+
+import static org.onosproject.kubevirtnode.api.Constants.CLI_IP_ADDRESSES_LENGTH;
+import static org.onosproject.kubevirtnode.api.Constants.CLI_IP_ADDRESS_LENGTH;
+import static org.onosproject.kubevirtnode.api.Constants.CLI_MAC_ADDRESS_LENGTH;
+import static org.onosproject.kubevirtnode.api.Constants.CLI_MARGIN_LENGTH;
+import static org.onosproject.kubevirtnode.api.Constants.CLI_NAME_LENGTH;
+import static org.onosproject.kubevirtnode.util.KubevirtNodeUtil.genFormatString;
+
+/**
+ * Lists all Kubernetes External LB config registered to the service.
+ */
+@Service
+@Command(scope = "onos", name = "kubernetes-lb-configs",
+ description = "Lists all Kubernetes External LB config registered to the service")
+public class KubernetesExternalLbConfigCommand extends AbstractShellCommand {
+
+ private static final String KUBE_VIP = "kubevip";
+
+ @Override
+ protected void doExecute() throws Exception {
+ KubernetesExternalLbConfigService service = get(KubernetesExternalLbConfigService.class);
+
+ String format = genFormatString(ImmutableList.of(CLI_NAME_LENGTH,
+ org.onosproject.kubevirtnode.api.Constants.CLI_IP_ADDRESS_LENGTH,
+ CLI_MAC_ADDRESS_LENGTH, CLI_IP_ADDRESSES_LENGTH));
+
+ KubernetesExternalLbConfig lbConfig = service.lbConfig(KUBE_VIP);
+
+ if (lbConfig == null) {
+ print("LB config not found!");
+ } else {
+ print(format, "ConfigName", "Gateway IP", "Gateway MAC", "Global-Range");
+
+ String configName = lbConfig.configName();
+ IpAddress gatewayIp = lbConfig.loadBalancerGwIp();
+ String gatewayMac = lbConfig.loadBalancerGwMac() == null ? "N/A" : lbConfig.loadBalancerGwMac().toString();
+ String globalRange = lbConfig.globalIpRange() == null ? "N/A" : lbConfig.globalIpRange();
+
+ print(format, StringUtils.substring(configName, 0,
+ CLI_NAME_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(gatewayIp.toString(), 0,
+ CLI_IP_ADDRESS_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(gatewayMac, 0,
+ CLI_MAC_ADDRESS_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(globalRange, 0,
+ CLI_IP_ADDRESSES_LENGTH - CLI_MARGIN_LENGTH)
+ );
+ }
+ }
+}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/cli/KubevirtShowNodeCommand.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/cli/KubevirtShowNodeCommand.java
index 5927530..39a18f0 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/cli/KubevirtShowNodeCommand.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/cli/KubevirtShowNodeCommand.java
@@ -67,7 +67,7 @@
}
private void printNode(KubevirtNode node) {
- print("node: %s", node.toString());
+ print("Node: %s", node.toString());
print("Name: %s", node.hostname());
print(" Type: %s", node.type());
print(" State: %s", node.state());
@@ -88,11 +88,11 @@
print(" GatewayBridgeName: %s", node.gatewayBridgeName());
}
- if (node.elbBridgeName() != null && node.elbIp() != null) {
- print(" ElbBridgeName: %s", node.elbBridgeName());
- print(" ElbIp: %s", node.elbIp().toString());
- print(" ElbGwIp: %s", node.elbGwIp().toString());
- print(" ElbGwMac: %s", node.elbGwMac().toString());
+ if (node.kubernetesExternalLbInterface() != null) {
+ print(" ElbBridgeName: %s", node.kubernetesExternalLbInterface().externalLbBridgeName());
+ print(" ElbIp: %s", node.kubernetesExternalLbInterface().externalLbIp().toString());
+ print(" ElbGwIp: %s", node.kubernetesExternalLbInterface().externalLbGwIp().toString());
+ print(" ElbGwMac: %s", node.kubernetesExternalLbInterface().externalLbGwMac().toString());
}
}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbInterfaceCodec.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbInterfaceCodec.java
new file mode 100644
index 0000000..cdd4402
--- /dev/null
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbInterfaceCodec.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.codec;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onosproject.codec.CodecContext;
+import org.onosproject.codec.JsonCodec;
+import org.onosproject.kubevirtnode.api.DefaultKubernetesExternalLbInterface;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
+import org.slf4j.Logger;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onlab.util.Tools.nullIsIllegal;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Kubernetes external load balancer interface codec used for serializing and de-serializing JSON string.
+ */
+public class KubernetesExternalLbInterfaceCodec extends JsonCodec<KubernetesExternalLbInterface> {
+ private final Logger log = getLogger(getClass());
+
+ private static final String ELB_BRIDGE_NAME = "externalLbBridgeName";
+ private static final String ELB_IP = "externalLbIp";
+ private static final String ELB_GW_IP = "externalLbGwIp";
+ private static final String ELB_GW_MAC = "externalLbGwMac";
+
+ private static final String MISSING_MESSAGE = " is required in KubernetesExternalLbInterfaceCodec";
+
+ @Override
+ public ObjectNode encode(KubernetesExternalLbInterface externalLbInterface, CodecContext context) {
+ checkNotNull(externalLbInterface, "checkNotNull cannot be null");
+
+ ObjectNode result = context.mapper().createObjectNode()
+ .put(ELB_BRIDGE_NAME, externalLbInterface.externalLbBridgeName())
+ .put(ELB_IP, externalLbInterface.externalLbIp().toString())
+ .put(ELB_GW_IP, externalLbInterface.externalLbGwIp().toString())
+ .put(ELB_GW_MAC, externalLbInterface.externalLbGwMac().toString());
+
+ return result;
+ }
+
+ @Override
+ public KubernetesExternalLbInterface decode(ObjectNode json, CodecContext context) {
+ if (json == null || !json.isObject()) {
+ return null;
+ }
+
+ String elbBridgeName = nullIsIllegal(json.get(ELB_BRIDGE_NAME).asText(),
+ ELB_BRIDGE_NAME + MISSING_MESSAGE);
+
+ String elbIp = nullIsIllegal(json.get(ELB_IP).asText(),
+ ELB_IP + MISSING_MESSAGE);
+
+ String elbGwIp = nullIsIllegal(json.get(ELB_GW_IP).asText(),
+ ELB_GW_IP + MISSING_MESSAGE);
+
+ String elbGwMac = nullIsIllegal(json.get(ELB_GW_MAC).asText(),
+ ELB_GW_MAC + MISSING_MESSAGE);
+
+ KubernetesExternalLbInterface externalLbInterface = DefaultKubernetesExternalLbInterface.builder()
+ .externalLbBridgeName(elbBridgeName)
+ .externallbGwIp(IpAddress.valueOf(elbGwIp))
+ .externalLbIp(IpAddress.valueOf(elbIp))
+ .externalLbGwMac(MacAddress.valueOf(elbGwMac))
+ .build();
+
+ return externalLbInterface;
+ }
+}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubevirtNodeCodec.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubevirtNodeCodec.java
index ec6f6bf..5267c77 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubevirtNodeCodec.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/codec/KubevirtNodeCodec.java
@@ -19,10 +19,10 @@
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.onlab.packet.IpAddress;
-import org.onlab.packet.MacAddress;
import org.onosproject.codec.CodecContext;
import org.onosproject.codec.JsonCodec;
import org.onosproject.kubevirtnode.api.DefaultKubevirtNode;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtNodeState;
import org.onosproject.kubevirtnode.api.KubevirtPhyInterface;
@@ -53,12 +53,9 @@
private static final String STATE = "state";
private static final String PHYSICAL_INTERFACES = "phyIntfs";
private static final String GATEWAY_BRIDGE_NAME = "gatewayBridgeName";
- private static final String ELB_BRIDGE_NAME = "elbBridgeName";
- private static final String ELB_IP = "elbIp";
- private static final String ELB_GW_IP = "elbGwIp";
- private static final String ELB_GW_MAC = "elbGwMac";
+ private static final String KUBERNETES_EXTERNAL_LB_INTERFACE = "kubernetesExternalLbInterface";
- private static final String MISSING_MESSAGE = " is required in OpenstackNode";
+ private static final String MISSING_MESSAGE = " is required in KubevirtNode";
@Override
public ObjectNode encode(KubevirtNode node, CodecContext context) {
@@ -101,21 +98,11 @@
result.put(GATEWAY_BRIDGE_NAME, node.gatewayBridgeName());
}
- //serialize elb bridge and ip address if exist
- if (node.elbBridgeName() != null) {
- result.put(ELB_BRIDGE_NAME, node.elbBridgeName());
- }
-
- if (node.elbIp() != null) {
- result.put(ELB_IP, node.elbIp().toString());
- }
-
- if (node.elbGwIp() != null) {
- result.put(ELB_GW_IP, node.elbGwIp().toString());
- }
-
- if (node.elbGwMac() != null) {
- result.put(ELB_GW_MAC, node.elbGwMac().toString());
+ // serialize kubernetex external load balancer interface if exist
+ if (node.kubernetesExternalLbInterface() != null) {
+ ObjectNode elbIntfJson = context.codec(KubernetesExternalLbInterface.class)
+ .encode(node.kubernetesExternalLbInterface(), context);
+ result.put(KUBERNETES_EXTERNAL_LB_INTERFACE, elbIntfJson.toString());
}
return result;
@@ -173,24 +160,16 @@
nodeBuilder.gatewayBridgeName(externalBridgeJson.asText());
}
- JsonNode elbBridgeJson = json.get(ELB_BRIDGE_NAME);
- if (elbBridgeJson != null) {
- nodeBuilder.elbBridgeName(elbBridgeJson.asText());
- }
+ JsonNode elbIntfJson = json.get(KUBERNETES_EXTERNAL_LB_INTERFACE);
- JsonNode elbIpJson = json.get(ELB_IP);
- if (elbIpJson != null) {
- nodeBuilder.elbIp(IpAddress.valueOf(elbIpJson.asText()));
- }
+ if (elbIntfJson != null) {
+ final JsonCodec<KubernetesExternalLbInterface>
+ kubernetesExternalLbInterfaceCodecJsonCodec = context.codec(KubernetesExternalLbInterface.class);
+ ObjectNode elbIntfObjNode = elbIntfJson.deepCopy();
- JsonNode elbGwIpJson = json.get(ELB_GW_IP);
- if (elbIpJson != null) {
- nodeBuilder.elbGwIp(IpAddress.valueOf(elbGwIpJson.asText()));
- }
+ nodeBuilder.kubernetesExternalLbInterface(
+ kubernetesExternalLbInterfaceCodecJsonCodec.decode(elbIntfObjNode, context));
- JsonNode elbGwMacJson = json.get(ELB_GW_MAC);
- if (elbIpJson != null) {
- nodeBuilder.elbGwMac(MacAddress.valueOf(elbGwMacJson.asText()));
}
log.trace("node is {}", nodeBuilder.build());
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubernetesExternalLbConfigStore.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubernetesExternalLbConfigStore.java
new file mode 100644
index 0000000..16a816e
--- /dev/null
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubernetesExternalLbConfigStore.java
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.impl;
+
+import com.google.common.collect.ImmutableSet;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnode.api.DefaultKubernetesExternalLbConfig;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfig;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigStore;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigStoreDelegate;
+import org.onosproject.store.AbstractStore;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.MapEvent;
+import org.onosproject.store.service.MapEventListener;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.onosproject.store.service.Versioned;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent.Type
+ .KUBERNETES_EXTERNAL_LB_CONFIG_CREATED;
+import static org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent.Type
+ .KUBERNETES_EXTERNAL_LB_CONFIG_REMOVED;
+import static org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent.Type
+ .KUBERNETES_EXTERNAL_LB_CONFIG_UPDATED;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Implementation of kubernetes external lb config store using consistent map.
+ */
+@Component(immediate = true, service = KubernetesExternalLbConfigStore.class)
+public class DistributedKubernetesExternalLbConfigStore
+ extends AbstractStore<KubernetesExternalLbConfigEvent, KubernetesExternalLbConfigStoreDelegate>
+ implements KubernetesExternalLbConfigStore {
+
+ private final Logger log = getLogger(getClass());
+
+ private static final String ERR_NOT_FOUND = " does not exist";
+ private static final String ERR_DUPLICATE = " already exists";
+ private static final String APP_ID = "org.onosproject.kubevirtnode";
+
+ private static final KryoNamespace
+ SERIALIZER_KUBERNETES_EXTERNAL_LB_CONFIG = KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API)
+ .register(KubernetesExternalLbConfig.class)
+ .register(DefaultKubernetesExternalLbConfig.class)
+ .register(IpAddress.class)
+ .register(MacAddress.class)
+ .register(Collection.class)
+ .build();
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected StorageService storageService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler", log));
+
+ private final MapEventListener<String, KubernetesExternalLbConfig> lbConfigMapEventListener =
+ new KubernetesExternalLbConfigMapListener();
+
+ private ConsistentMap<String, KubernetesExternalLbConfig> lbConfigStore;
+
+ @Activate
+ protected void activate() {
+ ApplicationId appId = coreService.registerApplication(APP_ID);
+ lbConfigStore = storageService.<String, KubernetesExternalLbConfig>consistentMapBuilder()
+ .withSerializer(Serializer.using(SERIALIZER_KUBERNETES_EXTERNAL_LB_CONFIG))
+ .withName("kubernetes-lbconfigstore")
+ .withApplicationId(appId)
+ .build();
+
+ lbConfigStore.addListener(lbConfigMapEventListener);
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ lbConfigStore.removeListener(lbConfigMapEventListener);
+ eventExecutor.shutdown();
+ log.info("Stopped");
+ }
+
+ @Override
+ public void createExternalLbConfig(KubernetesExternalLbConfig lbConfig) {
+ lbConfigStore.compute(lbConfig.configName(), (configName, existing) -> {
+ final String error = lbConfig.configName() + ERR_DUPLICATE;
+ checkArgument(existing == null, error);
+ return lbConfig;
+ });
+ }
+
+ @Override
+ public void updateExternalLbConfig(KubernetesExternalLbConfig lbConfig) {
+ lbConfigStore.compute(lbConfig.configName(), (configName, existing) -> {
+ final String error = lbConfig.configName() + ERR_NOT_FOUND;
+ checkArgument(existing != null, error);
+ return lbConfig;
+ });
+ }
+
+ @Override
+ public KubernetesExternalLbConfig removeExternalLbConfig(String configName) {
+
+ Versioned<KubernetesExternalLbConfig> lbConfig = lbConfigStore.remove(configName);
+
+ if (lbConfig == null) {
+ final String error = configName + ERR_NOT_FOUND;
+ throw new IllegalArgumentException(error);
+ }
+
+ return lbConfig.value();
+ }
+
+ @Override
+ public KubernetesExternalLbConfig externalLbConfig(String configName) {
+ return lbConfigStore.asJavaMap().get(configName);
+ }
+
+ @Override
+ public Set<KubernetesExternalLbConfig> externalLbConfigs() {
+
+ return ImmutableSet.copyOf(lbConfigStore.asJavaMap().values());
+ }
+
+ @Override
+ public void clear() {
+ lbConfigStore.clear();
+ }
+
+ private class KubernetesExternalLbConfigMapListener
+ implements MapEventListener<String, KubernetesExternalLbConfig> {
+
+ @Override
+ public void event(MapEvent<String, KubernetesExternalLbConfig> event) {
+ switch (event.type()) {
+ case INSERT:
+ eventExecutor.execute(() ->
+ notifyDelegate(new KubernetesExternalLbConfigEvent(
+ KUBERNETES_EXTERNAL_LB_CONFIG_CREATED, event.newValue().value())));
+ break;
+ case UPDATE:
+ eventExecutor.execute(() ->
+ notifyDelegate(new KubernetesExternalLbConfigEvent(
+ KUBERNETES_EXTERNAL_LB_CONFIG_UPDATED, event.newValue().value())));
+ break;
+ case REMOVE:
+ eventExecutor.execute(() ->
+ notifyDelegate(new KubernetesExternalLbConfigEvent(
+ KUBERNETES_EXTERNAL_LB_CONFIG_REMOVED, event.oldValue().value())));
+ break;
+ default:
+ //do nothing
+ break;
+ }
+ }
+ }
+}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubevirtNodeStore.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubevirtNodeStore.java
index d0bde71..47a28e1 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubevirtNodeStore.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubevirtNodeStore.java
@@ -19,8 +19,10 @@
import org.onlab.util.KryoNamespace;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnode.api.DefaultKubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.DefaultKubevirtNode;
import org.onosproject.kubevirtnode.api.DefaultKubevirtPhyInterface;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtNodeEvent;
import org.onosproject.kubevirtnode.api.KubevirtNodeState;
@@ -77,6 +79,8 @@
.register(DefaultKubevirtNode.class)
.register(KubevirtPhyInterface.class)
.register(DefaultKubevirtPhyInterface.class)
+ .register(KubernetesExternalLbInterface.class)
+ .register(DefaultKubernetesExternalLbInterface.class)
.register(KubevirtNode.Type.class)
.register(KubevirtNodeState.class)
.register(Collection.class)
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubernetesConfigMapWatcher.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubernetesConfigMapWatcher.java
new file mode 100644
index 0000000..bac5e03
--- /dev/null
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubernetesConfigMapWatcher.java
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.impl;
+
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.Watcher;
+import io.fabric8.kubernetes.client.WatcherException;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnode.api.DefaultKubernetesExternalLbConfig;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfig;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigAdminService;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfig;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigEvent;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigListener;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
+import org.onosproject.kubevirtnode.api.KubevirtNodeService;
+import org.onosproject.mastership.MastershipService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnode.api.KubevirtNodeService.APP_ID;
+import static org.onosproject.kubevirtnode.util.KubevirtNodeUtil.k8sClient;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Kubernetes configmap watcher used for external loadbalancing among PODs.
+ */
+@Component(immediate = true)
+public class KubernetesConfigMapWatcher {
+ private final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtApiConfigService configService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbConfigAdminService adminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeService nodeService;
+
+ private static final String KUBE_DASH_VIP = "kube-vip";
+ private static final String KUBE_VIP = "kubevip";
+ private static final String LOADBALANCER_IP = "loadBalancerIP";
+ private static final String TYPE_LOADBALANCER = "LoadBalancer";
+ private static final String KUBE_SYSTEM = "kube-system";
+ private static final String GATEWAY_IP = "gateway-ip";
+ private static final String GATEWAY_MAC = "gateway-mac";
+ private static final String RANGE_GLOBAL = "range-global";
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+
+ private final InternalKubevirtApiConfigListener
+ configListener = new InternalKubevirtApiConfigListener();
+
+ private final InternalKubernetesConfigMapWatcher
+ mapWatcher = new InternalKubernetesConfigMapWatcher();
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ configService.addListener(configListener);
+
+ log.info("Started");
+ }
+
+
+ @Deactivate
+ protected void deactivate() {
+ configService.removeListener(configListener);
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+
+ private void instantiateWatcher() {
+ KubevirtApiConfig config = configService.apiConfig();
+ if (config == null) {
+ return;
+ }
+ KubernetesClient client = k8sClient(config);
+
+ if (client != null) {
+ client.configMaps().inNamespace(KUBE_SYSTEM).withName(KUBE_VIP).watch(mapWatcher);
+ }
+ }
+
+ private class InternalKubernetesConfigMapWatcher implements Watcher<ConfigMap> {
+
+ private boolean isMaster() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+
+ @Override
+ public void eventReceived(Action action, ConfigMap configMap) {
+ switch (action) {
+ case ADDED:
+ log.info("ConfigMap event ADDED received");
+ eventExecutor.execute(() -> processAddOrMod(configMap));
+ break;
+ case MODIFIED:
+ log.info("ConfigMap event MODIFIED received");
+ eventExecutor.execute(() -> processAddOrMod(configMap));
+ break;
+ case DELETED:
+ log.info("ConfigMap event DELETED received");
+ eventExecutor.execute(() -> processDeletion(configMap));
+ break;
+ case ERROR:
+ log.warn("Failures processing pod manipulation.");
+ break;
+ default:
+ break;
+ }
+ }
+
+ @Override
+ public void onClose(WatcherException e) {
+ // due to the bugs in fabric8, pod watcher might be closed,
+ // we will re-instantiate the pod watcher in this case
+ // FIXME: https://github.com/fabric8io/kubernetes-client/issues/2135
+ log.warn("Configmap watcher OnClose, re-instantiate the POD watcher...");
+ instantiateWatcher();
+ }
+
+ private void processAddOrMod(ConfigMap configMap) {
+ if (configMap == null || !isMaster()) {
+ return;
+ }
+
+ String configName = configMap.getMetadata().getName();
+ if (!configName.equals(KUBE_VIP)) {
+ return;
+ }
+
+ KubernetesExternalLbConfig lbConfig = parseKubernetesExternalLbConfig(configMap, configName);
+
+ if (lbConfig == null) {
+ return;
+ }
+
+ log.info("Kubernetes external LB config inserted/updated {}", lbConfig);
+
+ if (adminService.lbConfig(configName) == null) {
+ adminService.createKubernetesExternalLbConfig(lbConfig);
+ } else {
+ adminService.updateKubernetesExternalLbConfig(lbConfig);
+ }
+ }
+
+ private void processDeletion(ConfigMap configMap) {
+ if (configMap == null || !isMaster()) {
+ return;
+ }
+
+ String configName = configMap.getMetadata().getName();
+ if (!configName.equals(KUBE_VIP)) {
+ return;
+ }
+
+ KubernetesExternalLbConfig lbConfig = adminService.lbConfig(configName);
+
+ if (lbConfig == null) {
+ return;
+ }
+
+ adminService.removeKubernetesExternalLbConfig(configName);
+ }
+
+ private KubernetesExternalLbConfig parseKubernetesExternalLbConfig(ConfigMap configMap, String configName) {
+ if (configMap.getData().get(GATEWAY_IP) == null || configMap.getData().get(RANGE_GLOBAL) == null) {
+ return null;
+ }
+
+ KubernetesExternalLbConfig.Builder lbConfigBuilder = DefaultKubernetesExternalLbConfig.builder();
+
+ try {
+ lbConfigBuilder.configName(configName)
+ .loadBalancerGwIp(IpAddress.valueOf(configMap.getData().get(GATEWAY_IP)))
+ .globalIpRange(configMap.getData().get(RANGE_GLOBAL));
+
+ if (configMap.getData().containsKey(GATEWAY_MAC)) {
+ lbConfigBuilder.loadBalancerGwMac(MacAddress.valueOf(configMap.getData().get(GATEWAY_MAC)));
+ }
+
+ } catch (IllegalArgumentException e) {
+ log.error("Exception occurred because of {}", e.toString());
+ }
+
+ return lbConfigBuilder.build();
+ }
+ }
+
+ private class InternalKubevirtApiConfigListener implements KubevirtApiConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubevirtApiConfigEvent event) {
+
+ switch (event.type()) {
+ case KUBEVIRT_API_CONFIG_UPDATED:
+ eventExecutor.execute(this::processConfigUpdate);
+ break;
+ case KUBEVIRT_API_CONFIG_CREATED:
+ case KUBEVIRT_API_CONFIG_REMOVED:
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ private void processConfigUpdate() {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ instantiateWatcher();
+ }
+ }
+}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubernetesExternalLbConfigManager.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubernetesExternalLbConfigManager.java
new file mode 100644
index 0000000..0562ee9
--- /dev/null
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubernetesExternalLbConfigManager.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.impl;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.event.ListenerRegistry;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfig;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigAdminService;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigListener;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigService;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigStore;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigStoreDelegate;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Set;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Provides implementation of administrating and interfacing kubernetes external lb config.
+ */
+@Component(
+ immediate = true,
+ service = {KubernetesExternalLbConfigAdminService.class, KubernetesExternalLbConfigService.class}
+)
+public class KubernetesExternalLbConfigManager
+ extends ListenerRegistry<KubernetesExternalLbConfigEvent, KubernetesExternalLbConfigListener>
+ implements KubernetesExternalLbConfigAdminService, KubernetesExternalLbConfigService {
+ protected final Logger log = getLogger(getClass());
+
+ private static final String MSG_LOAD_BALANCER_CONFIG = "Kubernetes external lb config %s %s";
+ private static final String MSG_CREATED = "created";
+ private static final String MSG_UPDATED = "updated";
+ private static final String MSG_REMOVED = "removed";
+
+ private static final String ERR_NULL_LOAD_BALANCER_CONFIG = "Kubernetes external lb config cannot be null";
+ private static final String ERR_NULL_LOAD_BALANCER_CONFIG_NAME
+ = "Kubernetes external lb config name cannot be null";
+ private static final String ERR_IN_USE = " still in use";
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbConfigStore lbConfigStore;
+
+ private final InternalKubernetesExternalLbConfigStorageDelegate delegate =
+ new InternalKubernetesExternalLbConfigStorageDelegate();
+
+ private ApplicationId appId;
+
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(APP_ID);
+
+ lbConfigStore.setDelegate(delegate);
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ lbConfigStore.unsetDelegate(delegate);
+ log.info("Stopped");
+ }
+
+
+ @Override
+ public void createKubernetesExternalLbConfig(KubernetesExternalLbConfig lbConfig) {
+ checkNotNull(lbConfig, ERR_NULL_LOAD_BALANCER_CONFIG);
+ checkArgument(!Strings.isNullOrEmpty(lbConfig.configName()), ERR_NULL_LOAD_BALANCER_CONFIG_NAME);
+
+ lbConfigStore.createExternalLbConfig(lbConfig);
+ log.info(String.format(MSG_LOAD_BALANCER_CONFIG, lbConfig.configName(), MSG_CREATED));
+ }
+
+ @Override
+ public void updateKubernetesExternalLbConfig(KubernetesExternalLbConfig lbConfig) {
+ checkNotNull(lbConfig, ERR_NULL_LOAD_BALANCER_CONFIG);
+ checkArgument(!Strings.isNullOrEmpty(lbConfig.configName()), ERR_NULL_LOAD_BALANCER_CONFIG_NAME);
+
+ lbConfigStore.updateExternalLbConfig(lbConfig);
+ log.info(String.format(MSG_LOAD_BALANCER_CONFIG, lbConfig.configName(), MSG_UPDATED));
+ }
+
+ @Override
+ public void removeKubernetesExternalLbConfig(String configName) {
+
+ checkArgument(configName != null, ERR_NULL_LOAD_BALANCER_CONFIG_NAME);
+
+ synchronized (this) {
+ KubernetesExternalLbConfig lbConfig = lbConfigStore.removeExternalLbConfig(configName);
+
+ if (lbConfig != null) {
+ log.info(String.format(MSG_LOAD_BALANCER_CONFIG, lbConfig.configName(), MSG_REMOVED));
+ }
+ }
+ }
+
+ @Override
+ public KubernetesExternalLbConfig lbConfig(String configName) {
+ checkArgument(configName != null, ERR_NULL_LOAD_BALANCER_CONFIG_NAME);
+
+ return lbConfigStore.externalLbConfig(configName);
+ }
+
+ @Override
+ public Set<KubernetesExternalLbConfig> lbConfigs() {
+ return ImmutableSet.copyOf(lbConfigStore.externalLbConfigs());
+ }
+
+ private class InternalKubernetesExternalLbConfigStorageDelegate
+ implements KubernetesExternalLbConfigStoreDelegate {
+
+ @Override
+ public void notify(KubernetesExternalLbConfigEvent event) {
+ log.trace("send kubernetes external lb config event {}", event);
+ process(event);
+ }
+ }
+}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubevirtNodeWatcher.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubevirtNodeWatcher.java
index 5ff2856..3949d79 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubevirtNodeWatcher.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubevirtNodeWatcher.java
@@ -186,7 +186,7 @@
}
KubevirtNode kubevirtNode = buildKubevirtNode(node);
- log.info("buildKubevirtNode: {}", kubevirtNode.toString());
+ log.info("buildKubevirtNode: {}", kubevirtNode);
if (kubevirtNode.type() == WORKER || kubevirtNode.type() == GATEWAY) {
if (!kubevirtNodeAdminService.hasNode(kubevirtNode.hostname())) {
kubevirtNodeAdminService.createNode(kubevirtNode);
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/util/KubevirtNodeUtil.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/util/KubevirtNodeUtil.java
index 2589be5..a5a5157 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/util/KubevirtNodeUtil.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/util/KubevirtNodeUtil.java
@@ -15,6 +15,8 @@
*/
package org.onosproject.kubevirtnode.util;
+import com.eclipsesource.json.JsonArray;
+import com.eclipsesource.json.JsonObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -27,12 +29,12 @@
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.apache.commons.lang.StringUtils;
-import com.eclipsesource.json.JsonArray;
-import com.eclipsesource.json.JsonObject;
import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
+import org.onosproject.kubevirtnode.api.DefaultKubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.DefaultKubevirtNode;
import org.onosproject.kubevirtnode.api.DefaultKubevirtPhyInterface;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtApiConfig;
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtNodeState;
@@ -85,11 +87,11 @@
private static final String DATA_IP_KEY = SONA_PROJECT_DOMAIN + "/data-ip";
private static final String GATEWAY_CONFIG_KEY = SONA_PROJECT_DOMAIN + "/gateway-config";
private static final String GATEWAY_BRIDGE_NAME = "gatewayBridgeName";
- private static final String ELB_CONFIG_KEY = SONA_PROJECT_DOMAIN + "/elb-config";
- private static final String ELB_BRIDGE_NAME = "elbBridgeName";
- private static final String ELB_IP_KEY = SONA_PROJECT_DOMAIN + "/elb-ip";
- private static final String ELB_GW_IP_KEY = SONA_PROJECT_DOMAIN + "/elb-gw-ip";
- private static final String ELB_GW_MAC_KEY = SONA_PROJECT_DOMAIN + "/elb-gw-mac";
+ private static final String EXTERNAL_LB_CONFIG_KEY = SONA_PROJECT_DOMAIN + "/externalLb-config";
+ private static final String EXTERNAL_LB_BRIDGE_NAME = "externalLbBridgeName";
+ private static final String EXTERNAL_LB_IP_KEY = SONA_PROJECT_DOMAIN + "/externalLb-ip";
+ private static final String EXTERNAL_LB_GATEWAY_IP_KEY = SONA_PROJECT_DOMAIN + "/externalLb-gateway-ip";
+ private static final String EXTERNAL_LB_GATEWAY_MAC_KEY = SONA_PROJECT_DOMAIN + "/externalLb-gateway-mac";
private static final String NETWORK_KEY = "network";
private static final String INTERFACE_KEY = "interface";
private static final String PHYS_BRIDGE_ID = "physBridgeId";
@@ -393,15 +395,17 @@
Set<KubevirtPhyInterface> phys = new HashSet<>();
String gatewayBridgeName = null;
- String elbConfig = annots.get(ELB_CONFIG_KEY);
- String elbIpStr = annots.get(ELB_IP_KEY);
- String elbGwIpStr = annots.get(ELB_GW_IP_KEY);
- String elbGwMacStr = annots.get(ELB_GW_MAC_KEY);
+ String elbConfig = annots.get(EXTERNAL_LB_CONFIG_KEY);
+ String elbIpStr = annots.get(EXTERNAL_LB_IP_KEY);
+ String elbGwIpStr = annots.get(EXTERNAL_LB_GATEWAY_IP_KEY);
+ String elbGwMacStr = annots.get(EXTERNAL_LB_GATEWAY_MAC_KEY);
String elbBridgeName = null;
IpAddress elbIp = null;
IpAddress elbGwIp = null;
MacAddress elbGwMac = null;
+ KubernetesExternalLbInterface kubernetesExternalLbInterface = null;
+
try {
if (physnetConfig != null) {
JsonArray configJson = JsonArray.readFrom(physnetConfig);
@@ -443,15 +447,20 @@
if (elbConfig != null && elbIpStr != null && elbGwIpStr != null) {
JsonNode elbJsonNode = new ObjectMapper().readTree(elbConfig);
- elbBridgeName = elbJsonNode.get(ELB_BRIDGE_NAME).asText();
+ elbBridgeName = elbJsonNode.get(EXTERNAL_LB_BRIDGE_NAME).asText();
elbIp = IpAddress.valueOf(elbIpStr);
elbGwIp = IpAddress.valueOf(elbGwIpStr);
- }
- if (elbGwMacStr != null) {
- elbGwMac = MacAddress.valueOf(elbGwMacStr);
- } else {
- //TODO: add the logic that retrieves the MAC address of the elb gw ip.
+ if (elbGwMacStr != null) {
+ elbGwMac = MacAddress.valueOf(elbGwMacStr);
+ }
+
+ kubernetesExternalLbInterface = DefaultKubernetesExternalLbInterface.builder()
+ .externalLbBridgeName(elbBridgeName)
+ .externalLbIp(elbIp)
+ .externallbGwIp(elbGwIp)
+ .externalLbGwMac(elbGwMac)
+ .build();
}
}
} catch (JsonProcessingException e) {
@@ -483,10 +492,7 @@
.state(KubevirtNodeState.ON_BOARDED)
.phyIntfs(phys)
.gatewayBridgeName(gatewayBridgeName)
- .elbBridgeName(elbBridgeName)
- .elbIp(elbIp)
- .elbGwIp(elbGwIp)
- .elbGwMac(elbGwMac)
+ .kubernetesExternalLbInterface(kubernetesExternalLbInterface)
.build();
}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/web/KubevirtNodeCodecRegister.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/web/KubevirtNodeCodecRegister.java
index 55d740f..00a1007 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/web/KubevirtNodeCodecRegister.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/web/KubevirtNodeCodecRegister.java
@@ -16,9 +16,11 @@
package org.onosproject.kubevirtnode.web;
import org.onosproject.codec.CodecService;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtApiConfig;
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtPhyInterface;
+import org.onosproject.kubevirtnode.codec.KubernetesExternalLbInterfaceCodec;
import org.onosproject.kubevirtnode.codec.KubevirtApiConfigCodec;
import org.onosproject.kubevirtnode.codec.KubevirtNodeCodec;
import org.onosproject.kubevirtnode.codec.KubevirtPhyInterfaceCodec;
@@ -47,6 +49,7 @@
codecService.registerCodec(KubevirtNode.class, new KubevirtNodeCodec());
codecService.registerCodec(KubevirtPhyInterface.class, new KubevirtPhyInterfaceCodec());
codecService.registerCodec(KubevirtApiConfig.class, new KubevirtApiConfigCodec());
+ codecService.registerCodec(KubernetesExternalLbInterface.class, new KubernetesExternalLbInterfaceCodec());
log.info("Started");
}
@@ -56,6 +59,7 @@
codecService.unregisterCodec(KubevirtNode.class);
codecService.unregisterCodec(KubevirtPhyInterface.class);
codecService.unregisterCodec(KubevirtApiConfig.class);
+ codecService.unregisterCodec(KubernetesExternalLbInterface.class);
log.info("Stopped");
}
diff --git a/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbIntfCodecTest.java b/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbIntfCodecTest.java
new file mode 100644
index 0000000..6917c35
--- /dev/null
+++ b/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbIntfCodecTest.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.codec;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.hamcrest.MatcherAssert;
+import org.junit.Before;
+import org.junit.Test;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onosproject.codec.JsonCodec;
+import org.onosproject.codec.impl.MockCodecContext;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnode.api.DefaultKubernetesExternalLbInterface;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.onosproject.kubevirtnode.codec.KubernetesExternalLbIntfJsonMatcher.matchesKubernetesElbIntf;
+import static org.onosproject.net.NetTestTools.APP_ID;
+
+/**
+ * Unit tests for KubernetesExternalLbInterface codec.
+ */
+public class KubernetesExternalLbIntfCodecTest {
+ MockCodecContext context;
+
+ private static final String REST_APP_ID = "org.onosproject.rest";
+ JsonCodec<KubernetesExternalLbInterface> kubernetesElbIntfCodec;
+
+ final CoreService mockCoreService = createMock(CoreService.class);
+
+ @Before
+ public void setUp() {
+ context = new MockCodecContext();
+ kubernetesElbIntfCodec = new KubernetesExternalLbInterfaceCodec();
+
+ assertThat(kubernetesElbIntfCodec, notNullValue());
+
+ expect(mockCoreService.registerApplication(REST_APP_ID))
+ .andReturn(APP_ID).anyTimes();
+ replay(mockCoreService);
+ context.registerService(CoreService.class, mockCoreService);
+ }
+
+ /**
+ * Tests encoding.
+ */
+ @Test
+ public void testEncode() {
+ KubernetesExternalLbInterface externalLbInterface =
+ DefaultKubernetesExternalLbInterface.builder()
+ .externalLbBridgeName("elbnetwork")
+ .externalLbIp(IpAddress.valueOf("10.10.10.2"))
+ .externallbGwIp(IpAddress.valueOf("10.10.10.1"))
+ .externalLbGwMac(MacAddress.valueOf("AA:BB:CC:DD:EE:FF"))
+ .build();
+
+ ObjectNode nodeJson = kubernetesElbIntfCodec.encode(externalLbInterface, context);
+ assertThat(nodeJson, matchesKubernetesElbIntf(externalLbInterface));
+ }
+
+ /**
+ * Tests decoding.
+ */
+ @Test
+ public void testDecode() throws IOException {
+ KubernetesExternalLbInterface externalLbInterface = getElbIntf("KubernetesExternalLbIntf.json");
+
+ assertEquals("elbnet", externalLbInterface.externalLbBridgeName());
+ assertEquals("10.10.10.2", externalLbInterface.externalLbIp().toString());
+ assertEquals("10.10.10.1", externalLbInterface.externalLbGwIp().toString());
+ assertEquals("AA:BB:CC:DD:EE:FF", externalLbInterface.externalLbGwMac().toString());
+ }
+
+ private KubernetesExternalLbInterface getElbIntf(String resourceName) throws IOException {
+ InputStream jsonStream = KubernetesExternalLbIntfCodecTest.class.getResourceAsStream(resourceName);
+ JsonNode json = context.mapper().readTree(jsonStream);
+ MatcherAssert.assertThat(json, notNullValue());
+
+ KubernetesExternalLbInterface externalLbInterface = kubernetesElbIntfCodec.decode(
+ (ObjectNode) json, context);
+ assertThat(externalLbInterface, notNullValue());
+
+ return externalLbInterface;
+ }
+
+}
diff --git a/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbIntfJsonMatcher.java b/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbIntfJsonMatcher.java
new file mode 100644
index 0000000..9744f62
--- /dev/null
+++ b/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubernetesExternalLbIntfJsonMatcher.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnode.codec;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.hamcrest.Description;
+import org.hamcrest.TypeSafeDiagnosingMatcher;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
+
+/**
+ * Hamcrest matcher for KubernetesExternalLbInterface.
+ */
+public final class KubernetesExternalLbIntfJsonMatcher extends TypeSafeDiagnosingMatcher<JsonNode> {
+
+ private final KubernetesExternalLbInterface externalLbInterface;
+
+ private static final String ELB_BRIDGE_NAME = "externalLbBridgeName";
+ private static final String ELB_IP = "externalLbIp";
+ private static final String ELB_GW_IP = "externalLbGwIp";
+ private static final String ELB_GW_MAC = "externalLbGwMac";
+
+ private KubernetesExternalLbIntfJsonMatcher(KubernetesExternalLbInterface externalLbInterface) {
+ this.externalLbInterface = externalLbInterface;
+ }
+
+ @Override
+ protected boolean matchesSafely(JsonNode jsonNode, Description description) {
+ // check externalLbBridgeName
+ String jsonElbBridgeName = jsonNode.get(ELB_BRIDGE_NAME).asText();
+ String elbBridgeName = externalLbInterface.externalLbBridgeName();
+ if (!jsonElbBridgeName.equals(elbBridgeName)) {
+ description.appendText("externalLbBridgeName was " + elbBridgeName);
+ return false;
+ }
+
+ // check externalLbIp
+ String jsonElbIp = jsonNode.get(ELB_IP).asText();
+ String elbIp = externalLbInterface.externalLbIp().toString();
+ if (!jsonElbIp.equals(elbIp)) {
+ description.appendText("externalLbIp was " + elbIp);
+ return false;
+ }
+
+ // check externalLbGwIp
+ String jsonElbGwIp = jsonNode.get(ELB_GW_IP).asText();
+ String elbGwIp = externalLbInterface.externalLbGwIp().toString();
+ if (!jsonElbGwIp.equals(elbGwIp)) {
+ description.appendText("externalLbGwIp was " + elbGwIp);
+ return false;
+ }
+
+ // check externalLbGwMac
+ String jsonElbGwMac = jsonNode.get(ELB_GW_MAC).asText();
+ String elbGwMac = externalLbInterface.externalLbGwMac().toString();
+ if (!jsonElbGwMac.equals(elbGwMac)) {
+ description.appendText("externalLbGwMac was " + elbGwMac);
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText(externalLbInterface.toString());
+ }
+
+ /**
+ * Factory to allocate a kubernetes external lb interface matcher.
+ *
+ * @param externalLbInterface kubernetes external lb interface we are looking for
+ * @return matcher
+ */
+ public static KubernetesExternalLbIntfJsonMatcher matchesKubernetesElbIntf(
+ KubernetesExternalLbInterface externalLbInterface) {
+ return new KubernetesExternalLbIntfJsonMatcher(externalLbInterface);
+ }
+}
diff --git a/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubevirtNodeCodecTest.java b/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubevirtNodeCodecTest.java
index 8f89152..e201554 100644
--- a/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubevirtNodeCodecTest.java
+++ b/apps/kubevirt-node/app/src/test/java/org/onosproject/kubevirtnode/codec/KubevirtNodeCodecTest.java
@@ -28,8 +28,10 @@
import org.onosproject.codec.JsonCodec;
import org.onosproject.codec.impl.CodecManager;
import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnode.api.DefaultKubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.DefaultKubevirtNode;
import org.onosproject.kubevirtnode.api.DefaultKubevirtPhyInterface;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtNodeState;
import org.onosproject.kubevirtnode.api.KubevirtPhyInterface;
@@ -57,6 +59,7 @@
JsonCodec<KubevirtNode> kubevirtNodeCodec;
JsonCodec<KubevirtPhyInterface> kubevirtPhyInterfaceJsonCodec;
+ JsonCodec<KubernetesExternalLbInterface> kubernetesExternalLbInterfaceJsonCodec;
final CoreService mockCoreService = createMock(CoreService.class);
private static final String REST_APP_ID = "org.onosproject.rest";
@@ -70,9 +73,11 @@
context = new MockCodecContext();
kubevirtNodeCodec = new KubevirtNodeCodec();
kubevirtPhyInterfaceJsonCodec = new KubevirtPhyInterfaceCodec();
+ kubernetesExternalLbInterfaceJsonCodec = new KubernetesExternalLbInterfaceCodec();
assertThat(kubevirtNodeCodec, notNullValue());
assertThat(kubevirtPhyInterfaceJsonCodec, notNullValue());
+ assertThat(kubernetesExternalLbInterfaceJsonCodec, notNullValue());
expect(mockCoreService.registerApplication(REST_APP_ID))
.andReturn(APP_ID).anyTimes();
@@ -106,8 +111,6 @@
.tunBridge(DeviceId.deviceId("br-tun"))
.dataIp(IpAddress.valueOf("20.20.20.2"))
.phyIntfs(ImmutableList.of(phyIntf1, phyIntf2))
- .elbBridgeName("elbnet")
- .elbIp(IpAddress.valueOf("10.10.10.1"))
.build();
ObjectNode nodeJson = kubevirtNodeCodec.encode(node, context);
@@ -157,6 +160,15 @@
*/
@Test
public void testKubevirtGatweayNodeEncode() {
+
+ KubernetesExternalLbInterface kubernetesExternalLbInterface =
+ DefaultKubernetesExternalLbInterface.builder()
+ .externalLbBridgeName("elbnetwork")
+ .externalLbIp(IpAddress.valueOf("10.10.10.2"))
+ .externallbGwIp(IpAddress.valueOf("10.10.10.1"))
+ .externalLbGwMac(MacAddress.valueOf("aa:bb:cc:dd:ee:ff"))
+ .build();
+
KubevirtNode node = DefaultKubevirtNode.builder()
.hostname("gateway")
.type(KubevirtNode.Type.GATEWAY)
@@ -166,10 +178,7 @@
.tunBridge(DeviceId.deviceId("br-tun"))
.dataIp(IpAddress.valueOf("20.20.20.2"))
.gatewayBridgeName("gateway")
- .elbBridgeName("elbnet")
- .elbIp(IpAddress.valueOf("192.168.0.2"))
- .elbGwIp(IpAddress.valueOf("192.168.0.1"))
- .elbGwMac(MacAddress.valueOf("AA:BB:CC:DD:EE:FF"))
+ .kubernetesExternalLbInterface(kubernetesExternalLbInterface)
.build();
ObjectNode nodeJson = kubevirtNodeCodec.encode(node, context);
@@ -186,6 +195,8 @@
public void testKubevirtGatewayNodeDecode() throws IOException {
KubevirtNode node = getKubevirtNode("KubevirtGatewayNode.json");
+ KubernetesExternalLbInterface externalLbInterface = node.kubernetesExternalLbInterface();
+
assertThat(node.hostname(), is("gateway-01"));
assertThat(node.type().name(), is("GATEWAY"));
assertThat(node.managementIp().toString(), is("172.16.130.4"));
@@ -193,10 +204,10 @@
assertThat(node.intgBridge().toString(), is("of:00000000000000a1"));
assertThat(node.tunBridge().toString(), is("of:00000000000000a2"));
assertThat(node.gatewayBridgeName(), is("gateway"));
- assertThat(node.elbBridgeName(), is("elbnet"));
- assertThat(node.elbIp().toString(), is("192.168.0.2"));
- assertThat(node.elbGwIp().toString(), is("192.168.0.1"));
- assertThat(node.elbGwMac().toString(), is("AA:BB:CC:DD:EE:FF"));
+ assertThat(externalLbInterface.externalLbBridgeName(), is("elbnet"));
+ assertThat(externalLbInterface.externalLbIp().toString(), is("10.10.10.2"));
+ assertThat(externalLbInterface.externalLbGwIp().toString(), is("10.10.10.1"));
+ assertThat(externalLbInterface.externalLbGwMac().toString(), is("AA:BB:CC:DD:EE:FF"));
}
/**
@@ -227,6 +238,10 @@
return (JsonCodec<T>) kubevirtPhyInterfaceJsonCodec;
}
+ if (entityClass == KubernetesExternalLbInterface.class) {
+ return (JsonCodec<T>) kubernetesExternalLbInterfaceJsonCodec;
+ }
+
return manager.getCodec(entityClass);
}
diff --git a/apps/kubevirt-node/app/src/test/resources/org/onosproject/kubevirtnode/codec/KubernetesExternalLbIntf.json b/apps/kubevirt-node/app/src/test/resources/org/onosproject/kubevirtnode/codec/KubernetesExternalLbIntf.json
new file mode 100644
index 0000000..697c416
--- /dev/null
+++ b/apps/kubevirt-node/app/src/test/resources/org/onosproject/kubevirtnode/codec/KubernetesExternalLbIntf.json
@@ -0,0 +1,6 @@
+{
+ "externalLbBridgeName": "elbnet",
+ "externalLbIp": "10.10.10.2",
+ "externalLbGwIp": "10.10.10.1",
+ "externalLbGwMac": "aa:bb:cc:dd:ee:ff"
+}
\ No newline at end of file
diff --git a/apps/kubevirt-node/app/src/test/resources/org/onosproject/kubevirtnode/codec/KubevirtGatewayNode.json b/apps/kubevirt-node/app/src/test/resources/org/onosproject/kubevirtnode/codec/KubevirtGatewayNode.json
index 85a11a7..ec81558 100644
--- a/apps/kubevirt-node/app/src/test/resources/org/onosproject/kubevirtnode/codec/KubevirtGatewayNode.json
+++ b/apps/kubevirt-node/app/src/test/resources/org/onosproject/kubevirtnode/codec/KubevirtGatewayNode.json
@@ -6,8 +6,10 @@
"integrationBridge": "of:00000000000000a1",
"tunnelBridge": "of:00000000000000a2",
"gatewayBridgeName": "gateway",
- "elbBridgeName": "elbnet",
- "elbIp": "192.168.0.2",
- "elbGwIp": "192.168.0.1",
- "elbGwMac": "aa:bb:cc:dd:ee:ff"
+ "kubernetesExternalLbInterface": {
+ "externalLbBridgeName": "elbnet",
+ "externalLbIp": "10.10.10.2",
+ "externalLbGwIp": "10.10.10.1",
+ "externalLbGwMac": "aa:bb:cc:dd:ee:ff"
+ }
}
\ No newline at end of file