Implement kubernetes external loadbalancer handler.
Change-Id: I0f3057d66769f0ca7db7d508483835cdd1ff1593
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
index 0675c93..6866537 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
@@ -28,6 +28,9 @@
public static final String KUBEVIRT_NETWORKING_APP_ID =
"org.onosproject.kubevirtnetworking";
public static final String DEFAULT_GATEWAY_MAC_STR = "fe:00:00:00:00:02";
+ public static final String KUBERNETES_EXTERNAL_LB_FAKE_MAC_STR = "de:00:00:00:00:01";
+ public static final MacAddress KUBERNETES_EXTERNAL_LB_FAKE_MAC =
+ MacAddress.valueOf(KUBERNETES_EXTERNAL_LB_FAKE_MAC_STR);
public static final MacAddress DEFAULT_GATEWAY_MAC =
MacAddress.valueOf(DEFAULT_GATEWAY_MAC_STR);
@@ -106,6 +109,8 @@
public static final int PRIORITY_STATEFUL_SNAT_RULE = 40500;
public static final int PRIORITY_FLOATING_IP_UPSTREAM_RULE = 40800;
public static final int PRIORITY_FLOATING_IP_DOWNSTREAM_RULE = 40700;
+ public static final int PRIORITY_ELB_UPSTREAM_RULE = 39000;
+ public static final int PRIORITY_ELB_DOWNSTREAM_RULE = 39500;
public static final int PRIORITY_INTERNAL_ROUTING_RULE = 41000;
public static final int PRIORITY_LB_RULE = 41500;
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
index 96ecf12..10e7cba 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
@@ -17,6 +17,8 @@
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
import java.util.Objects;
import java.util.Set;
@@ -28,24 +30,32 @@
*/
public final class DefaultKubernetesExternalLb implements KubernetesExternalLb {
- private static final String NOT_NULL_MSG = "Loadbalancer % cannot be null";
+ private static final String NOT_NULL_MSG = "External Loadbalancer % cannot be null";
private final String serviceName;
- private final String loadbalancerIp;
+ private final IpAddress loadbalancerIp;
private final Set<Integer> nodePortSet;
private final Set<Integer> portSet;
private final Set<String> endpointSet;
private final String electedGateway;
+ private final IpAddress loadbalancerGwIp;
+ private final MacAddress loadbalancerGwMac;
+ private final String electedWorker;
- public DefaultKubernetesExternalLb(String serviceName, String loadbalancerIp,
+ public DefaultKubernetesExternalLb(String serviceName, IpAddress loadbalancerIp,
Set<Integer> nodePortSet, Set<Integer> portSet,
- Set<String> endpointSet, String electedGateway) {
+ Set<String> endpointSet, String electedGateway,
+ String electedWorker,
+ IpAddress loadbalancerGwIp, MacAddress loadbalancerGwMac) {
this.serviceName = serviceName;
this.loadbalancerIp = loadbalancerIp;
this.nodePortSet = nodePortSet;
this.portSet = portSet;
this.endpointSet = endpointSet;
this.electedGateway = electedGateway;
+ this.electedWorker = electedWorker;
+ this.loadbalancerGwIp = loadbalancerGwIp;
+ this.loadbalancerGwMac = loadbalancerGwMac;
}
@Override
@@ -54,7 +64,7 @@
}
@Override
- public String loadBalancerIp() {
+ public IpAddress loadBalancerIp() {
return loadbalancerIp;
}
@@ -79,6 +89,21 @@
}
@Override
+ public String electedWorker() {
+ return electedWorker;
+ }
+
+ @Override
+ public IpAddress loadBalancerGwIp() {
+ return loadbalancerGwIp;
+ }
+
+ @Override
+ public MacAddress loadBalancerGwMac() {
+ return loadbalancerGwMac;
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) {
return true;
@@ -92,7 +117,10 @@
Objects.equals(nodePortSet, that.nodePortSet) &&
Objects.equals(portSet, that.portSet) &&
Objects.equals(endpointSet, that.endpointSet) &&
- Objects.equals(electedGateway, that.electedGateway);
+ Objects.equals(electedGateway, that.electedGateway) &&
+ Objects.equals(electedWorker, that.electedWorker) &&
+ Objects.equals(loadbalancerGwIp, that.loadbalancerGwIp) &&
+ Objects.equals(loadbalancerGwMac, that.loadbalancerGwMac);
}
@Override
@@ -104,8 +132,27 @@
.portSet(portSet)
.endpointSet(endpointSet)
.electedGateway(electedGateway)
+ .electedWorker(electedWorker)
+ .loadBalancerGwIp(loadbalancerGwIp)
+ .loadBalancerGwMac(loadbalancerGwMac)
.build();
}
+
+ @Override
+ public KubernetesExternalLb updateElectedWorker(String electedWorker) {
+ return DefaultKubernetesExternalLb.builder()
+ .serviceName(serviceName)
+ .loadBalancerIp(loadbalancerIp)
+ .nodePortSet(nodePortSet)
+ .portSet(portSet)
+ .endpointSet(endpointSet)
+ .electedGateway(electedGateway)
+ .electedWorker(electedWorker)
+ .loadBalancerGwIp(loadbalancerGwIp)
+ .loadBalancerGwMac(loadbalancerGwMac)
+ .build();
+ }
+
@Override
public int hashCode() {
return Objects.hash(serviceName, loadbalancerIp);
@@ -120,6 +167,9 @@
.add("port", portSet)
.add("endpointSet", endpointSet)
.add("electedGateway", electedGateway)
+ .add("electedWorker", electedWorker)
+ .add("loadbalancer gateway ip", loadbalancerGwIp)
+ .add("loadbalancer gateway Mac", loadbalancerGwMac)
.toString();
}
@@ -129,11 +179,14 @@
public static final class Builder implements KubernetesExternalLb.Builder {
private String serviceName;
- private String loadbalancerIp;
+ private IpAddress loadbalancerIp;
private Set<Integer> nodePortSet;
private Set<Integer> portSet;
private Set<String> endpointSet;
private String electedGateway;
+ private String electedWorker;
+ private IpAddress loadbalancerGwip;
+ private MacAddress loadbalancerGwMac;
private Builder() {
}
@@ -146,7 +199,8 @@
checkArgument(!portSet.isEmpty(), NOT_NULL_MSG, "portSet");
return new DefaultKubernetesExternalLb(serviceName, loadbalancerIp,
- nodePortSet, portSet, endpointSet, electedGateway);
+ nodePortSet, portSet, endpointSet, electedGateway, electedWorker,
+ loadbalancerGwip, loadbalancerGwMac);
}
@Override
@@ -156,7 +210,7 @@
}
@Override
- public Builder loadBalancerIp(String loadBalancerIp) {
+ public Builder loadBalancerIp(IpAddress loadBalancerIp) {
this.loadbalancerIp = loadBalancerIp;
return this;
}
@@ -179,9 +233,28 @@
return this;
}
+ @Override
public Builder electedGateway(String electedGateway) {
this.electedGateway = electedGateway;
return this;
}
+
+ @Override
+ public Builder electedWorker(String electedWorker) {
+ this.electedWorker = electedWorker;
+ return this;
+ }
+
+ @Override
+ public Builder loadBalancerGwIp(IpAddress loadbalancerGwip) {
+ this.loadbalancerGwip = loadbalancerGwip;
+ return this;
+ }
+
+ @Override
+ public Builder loadBalancerGwMac(MacAddress loadbalancerGwMac) {
+ this.loadbalancerGwMac = loadbalancerGwMac;
+ return this;
+ }
}
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java
index bc754db..4e902b0 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java
@@ -15,6 +15,9 @@
*/
package org.onosproject.kubevirtnetworking.api;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+
import java.util.Set;
/**
@@ -33,7 +36,7 @@
*
* @return load balancer IP
*/
- String loadBalancerIp();
+ IpAddress loadBalancerIp();
/**
* Returns the set of node port.
@@ -64,6 +67,13 @@
String electedGateway();
/**
+ * Returns the elected worker node for this service.
+ *
+ * @return worker node hostname
+ */
+ String electedWorker();
+
+ /**
* Updates the elected gateway node host name.
*
* @param electedGateway updated elected gateway node hostname
@@ -71,6 +81,31 @@
*/
KubernetesExternalLb updateElectedGateway(String electedGateway);
+ /**
+ * Updates the elected worker node host name.
+ *
+ * @param electedWorker updated elected worker node hostname
+ * @return kubernetes external lb with the updated worker node hostname
+ */
+ KubernetesExternalLb updateElectedWorker(String electedWorker);
+
+ /**
+ * Returns the gateway IP of load balancer.
+ * TEG would send outbound traffic to this gateway.
+ *
+ * @return load balancer gateway IP
+ */
+ IpAddress loadBalancerGwIp();
+
+ /**
+ * Returns the gateway MAC of load balancer.
+ * TEG would send outbound traffic to this gateway.
+ *
+ * @return load balancer gateway IP
+ */
+ MacAddress loadBalancerGwMac();
+
+
interface Builder {
/**
* Builds an immutable kubernetes external load balancer instance.
@@ -93,7 +128,7 @@
* @param loadBalancerIp external load balancer Ip
* @return external load balancer builder
*/
- Builder loadBalancerIp(String loadBalancerIp);
+ Builder loadBalancerIp(IpAddress loadBalancerIp);
/**
* Returns kubernetes external load balancer builder with supplied node port set.
@@ -123,8 +158,33 @@
* Returns kubernetes external load balancer builder with supplied elected gateway.
*
* @param gateway gateway node hostname
- * @return gateway node hostname
+ * @return external load balancer builder
*/
Builder electedGateway(String gateway);
+
+ /**
+ * Returns kubernetes external load balancer builder with supplied elected worker.
+ *
+ * @param worker worker node hostname
+ * @return external load balancer builder
+ */
+ Builder electedWorker(String worker);
+
+
+ /**
+ * Returns kubernetes external load balancer builder with supplied load balancer gateway Ip.
+ *
+ * @param loadBalancerGwIp gateway IP of the external load balancer
+ * @return external load balancer builder
+ */
+ Builder loadBalancerGwIp(IpAddress loadBalancerGwIp);
+
+ /**
+ * Returns kubernetes external load balancer builder with supplied load balancer gateway Mac.
+ *
+ * @param loadBalancerGwMac gateway Mac of the external load balancer
+ * @return external load balancer builder
+ */
+ Builder loadBalancerGwMac(MacAddress loadBalancerGwMac);
}
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbAdminService.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbAdminService.java
index c53d409..b5b05ed 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbAdminService.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbAdminService.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2021-present Open Networking Foundation
+ * Copyright 2022-present Open Networking Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbEvent.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbEvent.java
index 15f3905..69aebce 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbEvent.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbEvent.java
@@ -23,8 +23,26 @@
public class KubernetesExternalLbEvent
extends AbstractEvent<KubernetesExternalLbEvent.Type, KubernetesExternalLb> {
+ private final String oldGateway;
+ private final String oldWorker;
+
+
public KubernetesExternalLbEvent(Type type, KubernetesExternalLb subject) {
super(type, subject);
+ this.oldGateway = null;
+ this.oldWorker = null;
+ }
+
+ public KubernetesExternalLbEvent(Type type, KubernetesExternalLb subject, String oldGateway, String oldWorker) {
+ super(type, subject);
+ this.oldGateway = oldGateway;
+ this.oldWorker = oldWorker;
+ }
+
+ public KubernetesExternalLbEvent(Type type, KubernetesExternalLb subject, String oldWorker) {
+ super(type, subject);
+ this.oldGateway = null;
+ this.oldWorker = oldWorker;
}
/**
@@ -32,18 +50,46 @@
*/
public enum Type {
/**
- * Signifies that a new kubevirt load balancer is created.
+ * Signifies that a new kubernetex external load balancer is created.
*/
KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED,
/**
- * Signifies that a kubevirt load balancer is removed.
+ * Signifies that a kubernetex external load balancer is removed.
*/
KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED,
/**
- * Signifies that a kubevirt load balancer is updated.
+ * Signifies that a kubernetex external load balancer is updated.
*/
KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED,
+
+ /**
+ * Signifies that a kubernetes external load balancer gateway node is updated.
+ */
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_GATEWAY_CHANGED,
+
+ /**
+ * Signifies that a kubernetes external load balancer worker node is updated.
+ */
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_WORKER_CHANGED,
+ }
+
+ /**
+ * Returns the old gateway of the router event.
+ *
+ * @return gateway node hostname
+ */
+ public String oldGateway() {
+ return oldGateway;
+ }
+
+ /**
+ * Returns the old worker of the router event.
+ *
+ * @return worker node hostname
+ */
+ public String oldWorker() {
+ return oldWorker;
}
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbListener.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbListener.java
index 1a48764..5cbeb30 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbListener.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbListener.java
@@ -18,7 +18,7 @@
import org.onosproject.event.EventListener;
/**
- * Listener for kubernetes external lb event.
+ * Listener for kubernetes external load balancer event.
*/
public interface KubernetesExternalLbListener extends EventListener<KubernetesExternalLbEvent> {
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbService.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbService.java
index eeac4f2..dee7ae9 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbService.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbService.java
@@ -23,7 +23,7 @@
extends ListenerService<KubernetesExternalLbEvent, KubernetesExternalLbListener> {
/**
- * Returns the kubernetes external lb with the supplied service name.
+ * Returns the kubernetes external load balancer with the supplied service name.
*
* @param serviceName service name
* @return kubernetes external load balancer
@@ -31,9 +31,9 @@
KubernetesExternalLb loadBalancer(String serviceName);
/**
- * Returns all kubernetes external lb's registered in the service.
+ * Returns all kubernetes external load balancers registered in the service.
*
- * @return set of kubernetes external lb's
+ * @return set of kubernetes external load balancers
*/
Set<KubernetesExternalLb> loadBalancers();
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbStoreDelegate.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbStoreDelegate.java
index a7fba41..e7ff6ff 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbStoreDelegate.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLbStoreDelegate.java
@@ -18,7 +18,7 @@
import org.onosproject.store.StoreDelegate;
/**
- * Kubernetes external lb store delegate abstraction.
+ * Kubernetes external load balancer store delegate abstraction.
*/
public interface KubernetesExternalLbStoreDelegate extends StoreDelegate<KubernetesExternalLbEvent> {
}
diff --git a/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java b/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java
index a4a661f..228bb9f 100644
--- a/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java
+++ b/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2021-present Open Networking Foundation
+ * Copyright 2022-present Open Networking Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@
import com.google.common.testing.EqualsTester;
import org.junit.Before;
import org.junit.Test;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
import java.util.Set;
@@ -31,8 +33,8 @@
public class DefaultKubernetesExternalLbTest {
private static final String SERVICE_NAME_1 = "service_name_1";
private static final String SERVICE_NAME_2 = "service_name_2";
- private static final String LOADBALANCER_IP_1 = "1.1.1.1";
- private static final String LOADBALANCER_IP_2 = "2.2.2.2";
+ private static final IpAddress LOADBALANCER_IP_1 = IpAddress.valueOf("1.1.1.2");
+ private static final IpAddress LOADBALANCER_IP_2 = IpAddress.valueOf("2.2.2.2");
private static final Set<Integer> NODE_PORT_SET_1 = Sets.newHashSet(Integer.valueOf(32080));
private static final Set<Integer> NODE_PORT_SET_2 = Sets.newHashSet(Integer.valueOf(33080));
private static final Set<Integer> PORT_SET_1 = Sets.newHashSet(Integer.valueOf(8080));
@@ -41,6 +43,13 @@
private static final Set<String> ENDPOINT_SET_2 = Sets.newHashSet(String.valueOf("1.1.2.2"));
private static final String ELECTED_GATEWAY_1 = "gateway1";
private static final String ELECTED_GATEWAY_2 = "gateway2";
+ private static final String ELECTED_WORKER_1 = "worker1";
+ private static final String ELECTED_WORKER_2 = "worker2";
+
+ private static final IpAddress LOADBALANCER_GW_IP_1 = IpAddress.valueOf("1.1.1.1");
+ private static final IpAddress LOADBALANCER_GW_IP_2 = IpAddress.valueOf("2.2.2.1");
+ private static final MacAddress LOADBALANCER_GW_MAC_1 = MacAddress.valueOf("aa:bb:cc:dd:ee:ff");
+ private static final MacAddress LOADBALANCER_GW_MAC_2 = MacAddress.valueOf("ff:ee:dd:cc:bb:aa");
private KubernetesExternalLb lb1;
private KubernetesExternalLb sameAsLb1;
@@ -66,6 +75,9 @@
.portSet(PORT_SET_1)
.endpointSet(ENDPOINT_SET_1)
.electedGateway(ELECTED_GATEWAY_1)
+ .electedWorker(ELECTED_WORKER_1)
+ .loadBalancerGwIp(LOADBALANCER_GW_IP_1)
+ .loadBalancerGwMac(LOADBALANCER_GW_MAC_1)
.build();
sameAsLb1 = DefaultKubernetesExternalLb.builder()
@@ -75,6 +87,9 @@
.portSet(PORT_SET_1)
.endpointSet(ENDPOINT_SET_1)
.electedGateway(ELECTED_GATEWAY_1)
+ .electedWorker(ELECTED_WORKER_1)
+ .loadBalancerGwIp(LOADBALANCER_GW_IP_1)
+ .loadBalancerGwMac(LOADBALANCER_GW_MAC_1)
.build();
lb2 = DefaultKubernetesExternalLb.builder()
@@ -84,6 +99,9 @@
.portSet(PORT_SET_2)
.endpointSet(ENDPOINT_SET_2)
.electedGateway(ELECTED_GATEWAY_2)
+ .electedWorker(ELECTED_WORKER_2)
+ .loadBalancerGwIp(LOADBALANCER_GW_IP_2)
+ .loadBalancerGwMac(LOADBALANCER_GW_MAC_2)
.build();
}
@@ -104,10 +122,10 @@
public void testConstruction() {
KubernetesExternalLb lb = lb1;
- assertEquals(SERVICE_NAME_1, lb1.serviceName());
- assertEquals(LOADBALANCER_IP_1, lb1.loadBalancerIp());
- assertEquals(NODE_PORT_SET_1, lb1.nodePortSet());
- assertEquals(PORT_SET_1, lb1.portSet());
- assertEquals(ENDPOINT_SET_1, lb1.endpointSet());
+ assertEquals(SERVICE_NAME_1, lb.serviceName());
+ assertEquals(LOADBALANCER_IP_1, lb.loadBalancerIp());
+ assertEquals(NODE_PORT_SET_1, lb.nodePortSet());
+ assertEquals(PORT_SET_1, lb.portSet());
+ assertEquals(ENDPOINT_SET_1, lb.endpointSet());
}
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java
new file mode 100644
index 0000000..47d4a71
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2021-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.cli;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
+import org.apache.karaf.shell.api.action.Command;
+import org.apache.karaf.shell.api.action.lifecycle.Service;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbService;
+
+import java.util.List;
+
+import static org.onosproject.kubevirtnetworking.api.Constants.CLI_IP_ADDRESS_LENGTH;
+import static org.onosproject.kubevirtnetworking.api.Constants.CLI_MAC_ADDRESS_LENGTH;
+import static org.onosproject.kubevirtnetworking.api.Constants.CLI_MARGIN_LENGTH;
+import static org.onosproject.kubevirtnetworking.api.Constants.CLI_NAME_LENGTH;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.genFormatString;
+
+/**
+ * Lists kubernetes services.
+ */
+@Service
+@Command(scope = "onos", name = "kubernetes-services",
+ description = "Lists all kubernetes services")
+public class KubernetesListServiceCommand extends AbstractShellCommand {
+
+ @Override
+ protected void doExecute() throws Exception {
+ KubernetesExternalLbService service = get(KubernetesExternalLbService.class);
+
+ List<KubernetesExternalLb> elbList = Lists.newArrayList(service.loadBalancers());
+
+ String format = genFormatString(ImmutableList.of(CLI_NAME_LENGTH, CLI_IP_ADDRESS_LENGTH,
+ CLI_NAME_LENGTH, CLI_NAME_LENGTH, CLI_IP_ADDRESS_LENGTH, CLI_MAC_ADDRESS_LENGTH));
+
+
+ print(format, "Service Name", "Loadbalancer IP", "Elected Gateway", "Elected Worker",
+ "Loadbalancer GW IP", "Loadbalancer GW MAC");
+
+ for (KubernetesExternalLb elb : elbList) {
+ String lbIp = elb.loadBalancerIp() == null ? "N/A" : elb.loadBalancerIp().toString();
+ String electedGw = elb.electedGateway() == null ? "N/A" : elb.electedGateway();
+ String electedWorker = elb.electedWorker() == null ? "N/A" : elb.electedWorker();
+ String lbGwIp = elb.loadBalancerGwIp() == null ? "N/A" : elb.loadBalancerGwIp().toString();
+ String lbGwMac = elb.loadBalancerGwMac() == null ? "N/A" : elb.loadBalancerGwMac().toString();
+
+ print(format, StringUtils.substring(elb.serviceName(), 0,
+ CLI_NAME_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(lbIp, 0,
+ CLI_IP_ADDRESS_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(electedGw, 0,
+ CLI_NAME_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(electedWorker, 0,
+ CLI_NAME_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(lbGwIp, 0,
+ CLI_IP_ADDRESS_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(lbGwMac, 0,
+ CLI_MAC_ADDRESS_LENGTH - CLI_MARGIN_LENGTH)
+ );
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
index fce69d7..1bd286e 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
@@ -16,6 +16,7 @@
package org.onosproject.kubevirtnetworking.impl;
import com.google.common.collect.ImmutableSet;
+import org.onlab.packet.IpAddress;
import org.onlab.util.KryoNamespace;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
@@ -47,12 +48,14 @@
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED;
+import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_GATEWAY_CHANGED;
import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED;
import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED;
+import static org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent.Type.KUBERNETES_EXTERNAL_LOAD_BALANCER_WORKER_CHANGED;
import static org.slf4j.LoggerFactory.getLogger;
/**
- * Implementation of kubernetes external lb store using consistent map.
+ * Implementation of kubernetes external load balancer store using consistent map.
*/
@Component(immediate = true, service = KubernetesExternalLbStore.class)
public class DistributedKubernetesExternalLbStore
@@ -69,6 +72,7 @@
.register(KryoNamespaces.API)
.register(KubernetesExternalLb.class)
.register(DefaultKubernetesExternalLb.class)
+ .register(IpAddress.class)
.register(Collection.class)
.build();
@@ -161,19 +165,44 @@
KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED, event.newValue().value())));
break;
case UPDATE:
- eventExecutor.execute(() ->
- notifyDelegate(new KubernetesExternalLbEvent(
- KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED, event.newValue().value())));
+ eventExecutor.execute(() -> processMapUpdate(event));
break;
case REMOVE:
eventExecutor.execute(() ->
notifyDelegate(new KubernetesExternalLbEvent(
- KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED, event.newValue().value())));
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED, event.oldValue().value())));
break;
default:
//do nothing
break;
}
}
+
+ private void processMapUpdate(MapEvent<String, KubernetesExternalLb> event) {
+ log.debug("Kubernetes External LB updated");
+
+ KubernetesExternalLb oldValue = event.oldValue().value();
+ KubernetesExternalLb newValue = event.newValue().value();
+
+ if (oldValue.electedGateway() != null && newValue.electedGateway() != null &&
+ !oldValue.electedGateway().equals(newValue.electedGateway())) {
+ notifyDelegate(new KubernetesExternalLbEvent(
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_GATEWAY_CHANGED,
+ newValue, oldValue.electedGateway(), oldValue.electedWorker())
+ );
+ }
+
+ if (oldValue.electedWorker() != null && newValue.electedWorker() != null &&
+ !oldValue.electedWorker().equals(newValue.electedWorker())) {
+ notifyDelegate(new KubernetesExternalLbEvent(
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_WORKER_CHANGED,
+ newValue, oldValue.electedWorker())
+ );
+ }
+
+ notifyDelegate(new KubernetesExternalLbEvent(
+ KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED, event.newValue().value()));
+
+ }
}
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
new file mode 100644
index 0000000..7b1c733
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
@@ -0,0 +1,460 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.impl;
+
+import org.onlab.packet.ARP;
+import org.onlab.packet.EthType;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.TpPort;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbAdminService;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbListener;
+import org.onosproject.kubevirtnetworking.api.KubevirtFlowRuleService;
+import org.onosproject.kubevirtnetworking.api.KubevirtGroupRuleService;
+import org.onosproject.kubevirtnetworking.util.RulePopulatorUtil;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.kubevirtnode.api.KubevirtNodeService;
+import org.onosproject.net.Device;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.onosproject.net.packet.PacketService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.GW_DROP_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.GW_ENTRY_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBERNETES_EXTERNAL_LB_FAKE_MAC;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ELB_DOWNSTREAM_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ELB_UPSTREAM_RULE;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.elbPatchPortNum;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.externalPatchPortNum;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.kubernetesElbMac;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.CT_NAT_SRC_FLAG;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Handles Kubernetes External load balancer.
+ */
+@Component(immediate = true)
+public class KubernetesExternalLbHandler {
+ protected final Logger log = getLogger(getClass());
+
+ private static final int TP_PORT_MINIMUM_NUM = 1025;
+ private static final int TP_PORT_MAXIMUM_NUM = 65535;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtApiConfigService configService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeService nodeService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtGroupRuleService groupRuleService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected PacketService packetService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtFlowRuleService flowService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbAdminService externalLbService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DriverService driverService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ private final InternalKubernetesExternalLbListener lbListener =
+ new InternalKubernetesExternalLbListener();
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ externalLbService.addListener(lbListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+ externalLbService.removeListener(lbListener);
+
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+
+ private class InternalKubernetesExternalLbListener implements KubernetesExternalLbListener {
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubernetesExternalLbEvent event) {
+ switch (event.type()) {
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_CREATED:
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_UPDATED:
+ eventExecutor.execute(() -> processKubernetesExternalLbCreatedOrUpdated(
+ event.subject()));
+ break;
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_GATEWAY_CHANGED:
+ eventExecutor.execute(() -> processKubernetesExternalLbGatewayChanged(
+ event.subject(), event.oldGateway()));
+ break;
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_WORKER_CHANGED:
+ eventExecutor.execute(() -> processKubernetesExternalLbWorkerChanged(
+ event.subject(), event.oldWorker()));
+ break;
+ case KUBERNETES_EXTERNAL_LOAD_BALANCER_REMOVED:
+ eventExecutor.execute(() -> processKubernetesExternalLbRemoved(
+ event.subject()));
+ break;
+ default:
+ //do nothing
+ break;
+ }
+ }
+
+ private void processKubernetesExternalLbCreatedOrUpdated(KubernetesExternalLb lb) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (lb.electedGateway() == null || lb.electedWorker() == null) {
+ log.warn("processKubernetesExternalLbCreatedOrUpdated called but electedGateway " +
+ "or electedWorker is null. Stop this task.");
+ return;
+ }
+
+ log.info("processKubernetesExternalLbCreatedOrUpdated and updated elb with elecedGateway: {}", lb);
+
+ setExternalLbRulesForService(lb, true);
+ }
+
+ private void processKubernetesExternalLbGatewayChanged(KubernetesExternalLb lb, String oldGatway) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (lb.electedWorker() == null || oldGatway == null) {
+ return;
+ }
+
+ log.info("processKubernetesExternalLbGatewayChanged with oldateway: {}", oldGatway);
+
+ setExternalLbRulesForService(lb.updateElectedGateway(oldGatway), false);
+
+ setExternalLbRulesForService(lb, true);
+ }
+
+ private void processKubernetesExternalLbWorkerChanged(KubernetesExternalLb lb, String oldWorker) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (lb.electedGateway() == null || oldWorker == null) {
+ return;
+ }
+
+ log.info("processKubernetesExternalLbWorkerChanged with oldworker: {}", oldWorker);
+
+ setExternalLbRulesForService(lb.updateElectedWorker(oldWorker), false);
+
+ setExternalLbRulesForService(lb, true);
+ }
+
+
+ private void processKubernetesExternalLbRemoved(KubernetesExternalLb lb) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (lb.electedGateway() == null) {
+ return;
+ }
+
+ setExternalLbRulesForService(lb, false);
+ }
+ }
+
+ private void setExternalLbRulesForService(KubernetesExternalLb lb, boolean install) {
+ if (lb.electedGateway() == null) {
+ return;
+ }
+
+ KubevirtNode gateway = nodeService.node(lb.electedGateway());
+
+ if (gateway == null) {
+ return;
+ }
+
+ setLoadbalanceIpArpResponseRules(lb, gateway, install);
+ setDownstreamRules(lb, gateway, install);
+ setUpstreamRules(lb, gateway, install);
+ }
+
+ private void setLoadbalanceIpArpResponseRules(KubernetesExternalLb lb, KubevirtNode gateway, boolean install) {
+
+ IpAddress loadBalancerIp = lb.loadBalancerIp();
+
+ if (loadBalancerIp == null) {
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(externalPatchPortNum(deviceService, gateway))
+ .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(loadBalancerIp.getIp4Address())
+ .build();
+
+ Device device = deviceService.getDevice(gateway.intgBridge());
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .extension(RulePopulatorUtil.buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpShaToThaExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpOp(ARP.OP_REPLY)
+ .setEthSrc(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .setArpSha(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .setArpSpa(loadBalancerIp.getIp4Address())
+ .setOutput(PortNumber.IN_PORT)
+ .build();
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_ARP_GATEWAY_RULE,
+ GW_ENTRY_TABLE,
+ install);
+ }
+
+ private void setDownstreamRules(KubernetesExternalLb lb, KubevirtNode gateway, boolean install) {
+
+ IpAddress loadBalancerIp = lb.loadBalancerIp();
+
+ if (loadBalancerIp == null) {
+ log.warn("setDownstreamRules called but loadBalancerIp is null. Stop this task.");
+ return;
+ }
+
+ MacAddress elbIntfMac = kubernetesElbMac(deviceService, gateway);
+ if (elbIntfMac == null) {
+ log.warn("setDownstreamRules called but elbIntfMac is null. Stop this task.");
+ return;
+ }
+
+ PortNumber elbBridgePortNum = elbPatchPortNum(deviceService, gateway);
+ if (elbBridgePortNum == null) {
+ log.warn("setDownstreamRules called but elbBridgePortNum is null. Stop this task.");
+ return;
+ }
+
+ KubernetesExternalLbInterface externalLbInterface = gateway.kubernetesExternalLbInterface();
+ if (externalLbInterface == null) {
+ log.warn("setDownstreamRules called but externalLbInterface is null. Stop this task.");
+ return;
+ }
+
+ KubevirtNode electedWorker = nodeService.node(lb.electedWorker());
+ if (electedWorker == null) {
+ log.warn("setDownstreamRules called but electedWorker is null. Stop this task.");
+ return;
+ }
+
+ lb.nodePortSet().forEach(nodeport -> {
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthDst(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .matchIPDst(loadBalancerIp.toIpPrefix())
+ .matchIPProtocol(IPv4.PROTOCOL_TCP)
+ .matchTcpDst(TpPort.tpPort(nodeport.intValue()))
+ .build();
+
+ ExtensionTreatment natTreatment = RulePopulatorUtil
+ .niciraConnTrackTreatmentBuilder(driverService, gateway.intgBridge())
+ .commit(true)
+ .natFlag(CT_NAT_SRC_FLAG)
+ .natAction(true)
+ .natIp(externalLbInterface.externalLbIp())
+ .natPortMin(TpPort.tpPort(TP_PORT_MINIMUM_NUM))
+ .natPortMax(TpPort.tpPort(TP_PORT_MAXIMUM_NUM))
+ .build();
+
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .extension(natTreatment, gateway.intgBridge())
+ .setEthSrc(elbIntfMac)
+ .setEthDst(externalLbInterface.externalLbGwMac())
+ .setIpDst(electedWorker.dataIp())
+ .setOutput(elbBridgePortNum)
+ .build();
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_ELB_DOWNSTREAM_RULE,
+ GW_ENTRY_TABLE,
+ install);
+ });
+ }
+
+ private void setUpstreamRules(KubernetesExternalLb lb, KubevirtNode gateway, boolean install) {
+ IpAddress loadBalancerIp = lb.loadBalancerIp();
+
+ if (loadBalancerIp == null) {
+ log.warn("setUpstreamRules called but loadBalancerIp is null. Stop this task.");
+ return;
+ }
+
+ MacAddress elbIntfMac = kubernetesElbMac(deviceService, gateway);
+ if (elbIntfMac == null) {
+ log.warn("setUpstreamRules called but elbIntfMac is null. Stop this task.");
+ return;
+ }
+
+ PortNumber elbBridgePortNum = elbPatchPortNum(deviceService, gateway);
+ if (elbBridgePortNum == null) {
+ log.warn("setUpstreamRules called but elbBridgePortNum is null. Stop this task.");
+ return;
+ }
+
+ PortNumber externalPatchPortNum = externalPatchPortNum(deviceService, gateway);
+ if (externalPatchPortNum == null) {
+ log.warn("setUpstreamRules called but externalPatchPortNum is null. Stop this task.");
+ return;
+ }
+
+ KubernetesExternalLbInterface externalLbInterface = gateway.kubernetesExternalLbInterface();
+ if (externalLbInterface == null) {
+ log.warn("setUpstreamRules called but externalLbInterface is null. Stop this task.");
+ return;
+ }
+
+
+ KubevirtNode electedWorker = nodeService.node(lb.electedWorker());
+ if (electedWorker == null) {
+ log.warn("setDownstreamRules called but electedWorker is null. Stop this task.");
+ return;
+ }
+
+ lb.nodePortSet().forEach(nodePort -> {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPSrc(electedWorker.dataIp().toIpPrefix())
+ .matchIPDst(externalLbInterface.externalLbIp().toIpPrefix())
+ .matchIPProtocol(IPv4.PROTOCOL_TCP)
+ .matchTcpSrc(TpPort.tpPort(nodePort.intValue()));
+
+ ExtensionTreatment natTreatment = RulePopulatorUtil
+ .niciraConnTrackTreatmentBuilder(driverService, gateway.intgBridge())
+ .commit(false)
+ .natAction(true)
+ .table((short) GW_DROP_TABLE)
+ .build();
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setEthSrc(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .setIpSrc(lb.loadBalancerIp())
+ .setEthDst(lb.loadBalancerGwMac())
+ .extension(natTreatment, gateway.intgBridge())
+ .transition(GW_DROP_TABLE);
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_ELB_UPSTREAM_RULE,
+ GW_ENTRY_TABLE,
+ install);
+
+ sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(IPv4.PROTOCOL_TCP)
+ .matchTcpSrc(TpPort.tpPort(nodePort.intValue()));
+
+
+ tBuilder = DefaultTrafficTreatment.builder()
+ .setEthSrc(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .setIpSrc(lb.loadBalancerIp())
+ .setEthDst(lb.loadBalancerGwMac())
+ .setOutput(externalPatchPortNum);
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_ELB_UPSTREAM_RULE,
+ GW_DROP_TABLE,
+ install);
+ });
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
new file mode 100644
index 0000000..2aec742
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
@@ -0,0 +1,529 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.impl;
+
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.api.model.LoadBalancerIngress;
+import io.fabric8.kubernetes.api.model.LoadBalancerStatus;
+import io.fabric8.kubernetes.api.model.Service;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.Watcher;
+import io.fabric8.kubernetes.client.WatcherException;
+import io.fabric8.kubernetes.client.dsl.Resource;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnetworking.api.DefaultKubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbAdminService;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfig;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigListener;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigService;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigEvent;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigListener;
+import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.kubevirtnode.api.KubevirtNodeEvent;
+import org.onosproject.kubevirtnode.api.KubevirtNodeListener;
+import org.onosproject.kubevirtnode.api.KubevirtNodeService;
+import org.onosproject.mastership.MastershipService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedService;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.k8sClient;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.workerNodeForSpecifiedService;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Kubernetes service watcher used for external loadbalancing among PODs.
+ */
+@Component(immediate = true)
+public class KubernetesServiceWatcher {
+ private final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtApiConfigService apiConfigService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbConfigService lbConfigService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbAdminService adminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeService nodeService;
+
+ private static final String KUBE_DASH_VIP = "kube-vip";
+ private static final String KUBE_VIP = "kubevip";
+ private static final String LOADBALANCER_IP = "loadBalancerIP";
+ private static final String TYPE_LOADBALANCER = "LoadBalancer";
+ private static final String KUBE_SYSTEM = "kube-system";
+ private static final String GATEWAY_IP = "gateway-ip";
+ private static final String GATEWAY_MAC = "gateway-mac";
+ private static final String DEFAULT = "default";
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+
+ private final InternalKubevirtApiConfigListener
+ apiConfigListener = new InternalKubevirtApiConfigListener();
+
+ private final InternalKubernetesServiceWatcher
+ serviceWatcher = new InternalKubernetesServiceWatcher();
+
+ private final InternalKubernetesExternalLbConfigListener
+ lbConfigListener = new InternalKubernetesExternalLbConfigListener();
+
+ private final InternalNodeEventListener
+ nodeEventListener = new InternalNodeEventListener();
+
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+
+ apiConfigService.addListener(apiConfigListener);
+ lbConfigService.addListener(lbConfigListener);
+ nodeService.addListener(nodeEventListener);
+
+ log.info("Started");
+ }
+
+
+ @Deactivate
+ protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+
+ apiConfigService.removeListener(apiConfigListener);
+ lbConfigService.removeListener(lbConfigListener);
+ nodeService.removeListener(nodeEventListener);
+
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private void instantiateWatcher() {
+ KubernetesClient client = k8sClient(apiConfigService);
+
+ if (client != null) {
+ client.services().inAnyNamespace().watch(serviceWatcher);
+ }
+ }
+
+ private class InternalKubernetesExternalLbConfigListener
+ implements KubernetesExternalLbConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubernetesExternalLbConfigEvent event) {
+ switch (event.type()) {
+ case KUBERNETES_EXTERNAL_LB_CONFIG_CREATED:
+ case KUBERNETES_EXTERNAL_LB_CONFIG_UPDATED:
+ eventExecutor.execute(this::processConfigUpdate);
+ break;
+ case KUBERNETES_EXTERNAL_LB_CONFIG_REMOVED:
+ default:
+ //do nothing
+ break;
+ }
+ }
+
+ private void processConfigUpdate() {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ addOrUpdateExternalLoadBalancers();
+ }
+ }
+
+ private class InternalKubevirtApiConfigListener implements KubevirtApiConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubevirtApiConfigEvent event) {
+
+ switch (event.type()) {
+ case KUBEVIRT_API_CONFIG_UPDATED:
+ eventExecutor.execute(this::processConfigUpdate);
+ break;
+ case KUBEVIRT_API_CONFIG_CREATED:
+ case KUBEVIRT_API_CONFIG_REMOVED:
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ private void processConfigUpdate() {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ instantiateWatcher();
+ addOrUpdateExternalLoadBalancers();
+ }
+ }
+
+ private class InternalKubernetesServiceWatcher implements Watcher<Service> {
+
+ @Override
+ public void eventReceived(Action action, Service service) {
+ switch (action) {
+ case ADDED:
+ log.info("Service event ADDED received ");
+ eventExecutor.execute(() -> processAddOrMod(service));
+ break;
+ case MODIFIED:
+ log.info("Service event MODIFIED received");
+ eventExecutor.execute(() -> processAddOrMod(service));
+ break;
+ case DELETED:
+ log.info("Service event DELETED received");
+ eventExecutor.execute(() -> processDeletion(service));
+ break;
+ case ERROR:
+ log.warn("Failures processing pod manipulation.");
+ break;
+ default:
+ break;
+ }
+ }
+
+ @Override
+ public void onClose(WatcherException e) {
+ // due to the bugs in fabric8, pod watcher might be closed,
+ // we will re-instantiate the pod watcher in this case
+ // FIXME: https://github.com/fabric8io/kubernetes-client/issues/2135
+ log.warn("Service watcher OnClose, re-instantiate the POD watcher...");
+ instantiateWatcher();
+ }
+
+ private void processAddOrMod(Service service) {
+ if (service == null || !isMaster()) {
+ return;
+ }
+
+ if (!configMapUpdated()) {
+ log.warn("Config map is not set yet. Stop this task");
+ return;
+ }
+
+ try {
+ if (addOrUpdateExternalLoadBalancer(service) &&
+ !isLoadBalancerStatusAlreadySet(service)) {
+ serviceStatusUpdate(service);
+ }
+ } catch (Exception e) {
+ log.error("Exception occurred because of {}", e.toString());
+ }
+ }
+
+ private void processDeletion(Service service) {
+ if (service == null || !isMaster()) {
+ return;
+ }
+ if (isLoadBalancerType(service) &&
+ isKubeVipCloudProviderLabelIsSet(service)) {
+ KubernetesExternalLb lb = adminService.loadBalancer(service.getMetadata().getName());
+
+ if (lb == null) {
+ return;
+ }
+
+ adminService.removeExternalLb(lb.serviceName());
+ }
+ }
+ private boolean isMaster() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+ }
+
+
+ //When api config or configmap updated, check every prerequisite and update all external load balancers
+ private void addOrUpdateExternalLoadBalancers() {
+ KubernetesClient client = k8sClient(apiConfigService);
+
+ client.services().inNamespace(DEFAULT).list()
+ .getItems().forEach(this::addOrUpdateExternalLoadBalancer);
+ }
+
+ private boolean addOrUpdateExternalLoadBalancer(Service service) {
+ if (isLoadBalancerType(service) &&
+ isKubeVipCloudProviderLabelIsSet(service)) {
+
+ KubernetesExternalLb lb = parseKubernetesExternalLb(service);
+ if (lb == null) {
+ log.warn("Failed to parse the kubernetes external lb");
+ return false;
+ }
+
+ KubevirtNode electedGatewayNode = gatewayNodeForSpecifiedService(nodeService, lb);
+ if (electedGatewayNode == null) {
+ log.warn("Service created but there's no gateway nodes ready. Stop this task.");
+ return false;
+ }
+
+ lb = lb.updateElectedGateway(electedGatewayNode.hostname());
+
+ KubevirtNode electedWorkerNode = workerNodeForSpecifiedService(nodeService, lb);
+ if (electedWorkerNode == null) {
+ log.warn("Service created but there's no worker nodes ready. Stop this task.");
+ return false;
+ }
+ lb = lb.updateElectedWorker(electedWorkerNode.hostname());
+
+ log.trace("processAddOrMod called and parsed lb is {}", lb);
+
+ if (adminService.loadBalancer(lb.serviceName()) == null) {
+ adminService.createExternalLb(lb);
+ } else {
+ adminService.updateExternalLb(lb);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private void serviceStatusUpdate(Service service) {
+ KubernetesClient client = k8sClient(apiConfigService);
+
+ String lbIp = service.getSpec().getLoadBalancerIP();
+ if (lbIp == null) {
+ return;
+ }
+
+ LoadBalancerIngress lbIngress = new LoadBalancerIngress(KUBE_VIP, lbIp, Lists.newArrayList());
+
+ service.getStatus().getLoadBalancer().setIngress(Lists.newArrayList(lbIngress));
+
+ //When a service is deleted, the event MODIFED is also along with DELETED event
+ //So filter out this MODIFIED events
+ if (client.services().withName(service.getMetadata().getName()) != null) {
+ client.services().patchStatus(service);
+ }
+ }
+
+ //Only process if the event when the kube-vip-cloud-provider label is set
+ // and loadbalancer status is not set.
+ private boolean isKubeVipCloudProviderLabelIsSet(Service service) {
+ log.trace("isKubeVipCloudProviderLabelIsSet called with labels {}", service.getMetadata().getLabels());
+ if (service.getMetadata().getLabels() == null) {
+ return false;
+ }
+
+ return service.getMetadata().getLabels().containsValue(KUBE_DASH_VIP);
+ }
+
+ private boolean isLoadBalancerStatusAlreadySet(Service service) {
+ log.trace("isLoadBalancerStatusAlreadySet called with status {}", service.getStatus());
+
+ LoadBalancerStatus lbStatus = service.getStatus().getLoadBalancer();
+ if (lbStatus.getIngress().isEmpty()) {
+ return false;
+ }
+
+ String lbIp = service.getSpec().getLoadBalancerIP();
+ if (lbIp == null) {
+ return false;
+ }
+
+ return lbStatus.getIngress().stream()
+ .filter(lbIngress -> Objects.equals(lbIngress.getIp(), lbIp))
+ .findAny().isPresent();
+ }
+
+ private boolean configMapUpdated() {
+ KubernetesExternalLbConfig config = lbConfigService.lbConfigs().stream().findAny().orElse(null);
+
+ if (config == null) {
+ return false;
+ }
+
+ return config.configName() != null && config.globalIpRange() != null &&
+ config.loadBalancerGwIp() != null && config.loadBalancerGwMac() != null;
+ }
+
+ //Only process if the event when the service type is LoadBalancer
+ private boolean isLoadBalancerType(Service service) {
+ return service.getSpec().getType().equals(TYPE_LOADBALANCER);
+ }
+
+ private KubernetesExternalLb parseKubernetesExternalLb(Service service) {
+ if (service.getMetadata() == null || service.getSpec() == null) {
+ return null;
+ }
+
+ String serviceName = service.getMetadata().getName();
+ if (serviceName == null) {
+ return null;
+ }
+
+ String lbIp = service.getSpec().getLoadBalancerIP();
+ if (lbIp == null) {
+ return null;
+ }
+
+ Set<Integer> nodePortSet = Sets.newHashSet();
+ Set<Integer> portSet = Sets.newHashSet();
+ Set<String> endpointSet = Sets.newHashSet();
+
+ service.getSpec().getPorts().forEach(servicePort -> {
+ nodePortSet.add(servicePort.getNodePort());
+ portSet.add(servicePort.getPort());
+ });
+
+ nodeService.completeNodes(WORKER).forEach(workerNode -> {
+ endpointSet.add(workerNode.dataIp().toString());
+ });
+
+ String loadbalancerGatewayIp = loadBalancerGatewayIp();
+
+ if (loadbalancerGatewayIp == null) {
+ log.error("Can't find the loadbalancer gateway ip in the kubevip configmap.." +
+ "Failed to parse kubernetes external lb and return null");
+ return null;
+ }
+
+ MacAddress loadBalancerGatewayMac = loadBalancerGatewayMac();
+
+ if (loadbalancerGatewayIp == null) {
+ log.error("Can't find the loadbalancer gateway mac in the kubevip configmap.." +
+ "Failed to parse kubernetes external lb and return null");
+ return null;
+ }
+
+ return DefaultKubernetesExternalLb.builder().serviceName(serviceName)
+ .loadBalancerIp(IpAddress.valueOf(lbIp))
+ .nodePortSet(nodePortSet)
+ .portSet(portSet)
+ .endpointSet(endpointSet)
+ .loadBalancerGwIp(IpAddress.valueOf(loadbalancerGatewayIp))
+ .loadBalancerGwMac(loadBalancerGatewayMac)
+ .build();
+ }
+
+ private String loadBalancerGatewayIp() {
+ KubernetesClient client = k8sClient(apiConfigService);
+
+ Resource<ConfigMap> kubeVipConfigMapResource =
+ client.configMaps().inNamespace(KUBE_SYSTEM).withName(KUBE_VIP);
+
+ if (kubeVipConfigMapResource == null) {
+ return null;
+ }
+
+ Map<String, String> kubeVipConfigMap = kubeVipConfigMapResource.get().getData();
+
+ if (!kubeVipConfigMap.containsKey(GATEWAY_IP)) {
+ return null;
+ }
+
+ return kubeVipConfigMap.get(GATEWAY_IP);
+ }
+
+ private MacAddress loadBalancerGatewayMac() {
+ KubernetesExternalLbConfig config = lbConfigService.lbConfigs().stream().findAny().orElse(null);
+
+ if (config == null) {
+ return null;
+ }
+
+ return config.loadBalancerGwMac();
+ }
+
+ private class InternalNodeEventListener implements KubevirtNodeListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubevirtNodeEvent event) {
+ switch (event.type()) {
+ case KUBEVIRT_NODE_COMPLETE:
+ eventExecutor.execute(() -> processNodeCompletion(event.subject()));
+ break;
+ case KUBEVIRT_NODE_INCOMPLETE:
+ case KUBEVIRT_NODE_REMOVED:
+ eventExecutor.execute(() -> processNodeDeletion(event.subject()));
+ break;
+ case KUBEVIRT_NODE_UPDATED:
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ private void processNodeCompletion(KubevirtNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ addOrUpdateExternalLoadBalancers();
+ }
+
+ private void processNodeDeletion(KubevirtNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ addOrUpdateExternalLoadBalancers();
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
index 5bc1548..1d0a2e9 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
@@ -45,6 +45,8 @@
import org.onosproject.kubevirtnetworking.api.KubevirtRouterAdminService;
import org.onosproject.kubevirtnetworking.api.KubevirtRouterEvent;
import org.onosproject.kubevirtnetworking.api.KubevirtRouterListener;
+import org.onosproject.kubevirtnetworking.util.RulePopulatorUtil;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtNodeEvent;
@@ -109,10 +111,12 @@
import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.FLAT;
import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.VLAN;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.elbPatchPortNum;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedRouter;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtNetwork;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtPort;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterMacAddress;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.kubernetesElbMac;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.portNumber;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.resolveHostname;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.segmentIdHex;
@@ -1324,6 +1328,12 @@
}
} else if (node.type().equals(GATEWAY)) {
updateGatewayNodeForRouter();
+
+ KubernetesExternalLbInterface externalLbInterface = node.kubernetesExternalLbInterface();
+
+ if (externalLbInterface != null) {
+ setElbInternalIpArpResponseRules(node, true);
+ }
}
}
@@ -1358,6 +1368,49 @@
kubevirtRouterService.updateRouter(router.updatedElectedGateway(newGwNode.hostname()));
});
}
+
+ private void setElbInternalIpArpResponseRules(KubevirtNode gateway, boolean install) {
+ KubernetesExternalLbInterface externalLbInterface = gateway.kubernetesExternalLbInterface();
+ if (externalLbInterface == null) {
+ return;
+ }
+
+ IpAddress eIp = externalLbInterface.externalLbIp();
+
+ MacAddress elbIntfMac = kubernetesElbMac(deviceService, gateway);
+ if (elbIntfMac == null) {
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(elbPatchPortNum(deviceService, gateway))
+ .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(eIp.getIp4Address())
+ .build();
+
+ Device device = deviceService.getDevice(gateway.intgBridge());
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .extension(RulePopulatorUtil.buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpShaToThaExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpOp(ARP.OP_REPLY)
+ .setEthSrc(elbIntfMac)
+ .setArpSha(elbIntfMac)
+ .setArpSpa(eIp.getIp4Address())
+ .setOutput(PortNumber.IN_PORT)
+ .build();
+
+ flowService.setRule(
+ appId,
+ gateway.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_ARP_GATEWAY_RULE,
+ GW_ENTRY_TABLE,
+ install);
+ }
}
private class InternalKubevirtPortListener implements KubevirtPortListener {
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
index 3496008..539323f 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
@@ -53,6 +53,7 @@
import org.onosproject.kubevirtnetworking.api.KubevirtRouterService;
import org.onosproject.kubevirtnode.api.DefaultKubevirtNode;
import org.onosproject.kubevirtnode.api.DefaultKubevirtPhyInterface;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtApiConfig;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
import org.onosproject.kubevirtnode.api.KubevirtNode;
@@ -89,6 +90,7 @@
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.MASTER;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.OTHER;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
+import static org.onosproject.net.AnnotationKeys.PORT_MAC;
import static org.onosproject.net.AnnotationKeys.PORT_NAME;
/**
@@ -545,7 +547,7 @@
/**
* Returns the gateway node for the specified kubernetes external lb.
* Among gateways, only one gateway would act as a gateway per external lb.
- * Currently gateway node is selected based on modulo operation with router hashcode.
+ * Currently gateway node is selected based on modulo operation with external lb hashcode.
*
* @param nodeService kubevirt node service
* @param externalLb kubernetes external lb
@@ -565,6 +567,28 @@
}
/**
+ * Returns the worker node for the specified kubernetes external lb.
+ * Among worker nodes, only one worker would serve the traffic from and to the gateway.
+ * Currently worker node is selected based on modulo operation with external lb hashcode.
+ *
+ * @param nodeService kubevirt node service
+ * @param externalLb kubernetes external lb
+ * @return elected worker node
+ */
+ public static KubevirtNode workerNodeForSpecifiedService(KubevirtNodeService nodeService,
+ KubernetesExternalLb externalLb) {
+ //TODO: enhance election logic for a better load balancing
+
+ int numOfWorkers = nodeService.completeNodes(WORKER).size();
+ if (numOfWorkers == 0) {
+ return null;
+ }
+
+ return (KubevirtNode) nodeService.completeNodes(WORKER)
+ .toArray()[externalLb.hashCode() % numOfWorkers];
+ }
+
+ /**
* Returns the mac address of the router.
*
* @param router kubevirt router
@@ -659,6 +683,73 @@
}
/**
+ * Returns the external lb patch port number with specified gateway.
+ *
+ * @param deviceService device service
+ * @param gateway gateway node
+ * @return external lb bridge patch port number
+ */
+ public static PortNumber elbPatchPortNum(DeviceService deviceService, KubevirtNode gateway) {
+ KubernetesExternalLbInterface kubernetesExternalLbInterface =
+ gateway.kubernetesExternalLbInterface();
+
+ if (kubernetesExternalLbInterface == null) {
+ log.warn("No elb interface is attached to gateway {}", gateway.hostname());
+ return null;
+ }
+
+ String elbBridgeName = kubernetesExternalLbInterface.externalLbBridgeName();
+
+ String patchPortName = "int-to-" + elbBridgeName;
+
+ Port port = deviceService.getPorts(gateway.intgBridge()).stream()
+ .filter(p -> p.isEnabled() &&
+ Objects.equals(p.annotations().value(PORT_NAME), patchPortName))
+ .findAny().orElse(null);
+
+ return port != null ? port.number() : null;
+ }
+
+ /**
+ * Returns the external lb patch port Mac with specified gateway.
+ *
+ * @param deviceService device service
+ * @param gateway gateway node
+ * @return external lb bridge patch Mac Address
+ */
+ public static MacAddress kubernetesElbMac(DeviceService deviceService, KubevirtNode gateway) {
+
+ KubernetesExternalLbInterface kubernetesExternalLbInterface =
+ gateway.kubernetesExternalLbInterface();
+
+ if (kubernetesExternalLbInterface == null) {
+ log.warn("No elb interface is attached to gateway {}", gateway.hostname());
+ return null;
+ }
+
+ String elbBridgeName = kubernetesExternalLbInterface.externalLbBridgeName();
+
+ String patchPortName = "int-to-" + elbBridgeName;
+
+ Port port = deviceService.getPorts(gateway.intgBridge()).stream()
+ .filter(p -> p.isEnabled() &&
+ Objects.equals(p.annotations().value(PORT_NAME), patchPortName))
+ .findAny().orElse(null);
+
+ if (port == null) {
+ return null;
+ }
+
+ String portMacStr = port.annotations().value(PORT_MAC);
+
+ if (portMacStr == null) {
+ return null;
+ }
+
+ return MacAddress.valueOf(portMacStr);
+ }
+
+ /**
* Returns the kubevirt external network with specified router.
*
* @param networkService kubevirt network service