Retrieve MAC address when kubernetes external lb config or interface has only IP address.
Change-Id: I79538f262a1a5d9bceb20d461743212ec6717d5a
(cherry picked from commit 6a83ee6e704e035f8349f93c5bbd8d3b251fe2d5)
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
index 6866537..88ba57c 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
@@ -89,6 +89,7 @@
public static final int CLI_ID_LENGTH = 30;
public static final int CLI_NAME_LENGTH = 30;
public static final int CLI_LONG_NAME_LENGTH = 50;
+ public static final int CLI_LONG_SERVICE_PORT_LENGTH = 100;
public static final int CLI_IP_ADDRESSES_LENGTH = 50;
public static final int CLI_IP_ADDRESS_LENGTH = 25;
public static final int CLI_IP_ADDRESS_AVAILABILITY = 15;
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
index 10e7cba..c652d60 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLb.java
@@ -34,8 +34,7 @@
private final String serviceName;
private final IpAddress loadbalancerIp;
- private final Set<Integer> nodePortSet;
- private final Set<Integer> portSet;
+ private final Set<KubernetesServicePort> servicePorts;
private final Set<String> endpointSet;
private final String electedGateway;
private final IpAddress loadbalancerGwIp;
@@ -43,14 +42,13 @@
private final String electedWorker;
public DefaultKubernetesExternalLb(String serviceName, IpAddress loadbalancerIp,
- Set<Integer> nodePortSet, Set<Integer> portSet,
+ Set<KubernetesServicePort> servicePorts,
Set<String> endpointSet, String electedGateway,
String electedWorker,
IpAddress loadbalancerGwIp, MacAddress loadbalancerGwMac) {
this.serviceName = serviceName;
this.loadbalancerIp = loadbalancerIp;
- this.nodePortSet = nodePortSet;
- this.portSet = portSet;
+ this.servicePorts = servicePorts;
this.endpointSet = endpointSet;
this.electedGateway = electedGateway;
this.electedWorker = electedWorker;
@@ -69,13 +67,8 @@
}
@Override
- public Set<Integer> nodePortSet() {
- return ImmutableSet.copyOf(nodePortSet);
- }
-
- @Override
- public Set<Integer> portSet() {
- return ImmutableSet.copyOf(portSet);
+ public Set<KubernetesServicePort> servicePorts() {
+ return ImmutableSet.copyOf(servicePorts);
}
@Override
@@ -114,8 +107,7 @@
DefaultKubernetesExternalLb that = (DefaultKubernetesExternalLb) o;
return serviceName.equals(that.serviceName) && loadbalancerIp.equals(that.loadbalancerIp) &&
- Objects.equals(nodePortSet, that.nodePortSet) &&
- Objects.equals(portSet, that.portSet) &&
+ Objects.equals(servicePorts, that.servicePorts) &&
Objects.equals(endpointSet, that.endpointSet) &&
Objects.equals(electedGateway, that.electedGateway) &&
Objects.equals(electedWorker, that.electedWorker) &&
@@ -128,8 +120,7 @@
return DefaultKubernetesExternalLb.builder()
.serviceName(serviceName)
.loadBalancerIp(loadbalancerIp)
- .nodePortSet(nodePortSet)
- .portSet(portSet)
+ .servicePorts(servicePorts)
.endpointSet(endpointSet)
.electedGateway(electedGateway)
.electedWorker(electedWorker)
@@ -143,8 +134,7 @@
return DefaultKubernetesExternalLb.builder()
.serviceName(serviceName)
.loadBalancerIp(loadbalancerIp)
- .nodePortSet(nodePortSet)
- .portSet(portSet)
+ .servicePorts(servicePorts)
.endpointSet(endpointSet)
.electedGateway(electedGateway)
.electedWorker(electedWorker)
@@ -163,8 +153,7 @@
return MoreObjects.toStringHelper(this)
.add("serviceName", serviceName)
.add("loadbalancerIp", loadbalancerIp)
- .add("nodePort", nodePortSet)
- .add("port", portSet)
+ .add("servucePorts", servicePorts)
.add("endpointSet", endpointSet)
.add("electedGateway", electedGateway)
.add("electedWorker", electedWorker)
@@ -180,8 +169,7 @@
public static final class Builder implements KubernetesExternalLb.Builder {
private String serviceName;
private IpAddress loadbalancerIp;
- private Set<Integer> nodePortSet;
- private Set<Integer> portSet;
+ private Set<KubernetesServicePort> servicePorts;
private Set<String> endpointSet;
private String electedGateway;
private String electedWorker;
@@ -195,11 +183,10 @@
public KubernetesExternalLb build() {
checkArgument(serviceName != null, NOT_NULL_MSG, "serviceName");
checkArgument(loadbalancerIp != null, NOT_NULL_MSG, "loadbalancerIp");
- checkArgument(!nodePortSet.isEmpty(), NOT_NULL_MSG, "nodePortSet");
- checkArgument(!portSet.isEmpty(), NOT_NULL_MSG, "portSet");
+ checkArgument(!servicePorts.isEmpty(), NOT_NULL_MSG, "servicePorts");
return new DefaultKubernetesExternalLb(serviceName, loadbalancerIp,
- nodePortSet, portSet, endpointSet, electedGateway, electedWorker,
+ servicePorts, endpointSet, electedGateway, electedWorker,
loadbalancerGwip, loadbalancerGwMac);
}
@@ -216,14 +203,8 @@
}
@Override
- public Builder nodePortSet(Set<Integer> nodePortSet) {
- this.nodePortSet = nodePortSet;
- return this;
- }
-
- @Override
- public Builder portSet(Set<Integer> portSet) {
- this.portSet = portSet;
+ public Builder servicePorts(Set<KubernetesServicePort> servicePorts) {
+ this.servicePorts = servicePorts;
return this;
}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesServicePort.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesServicePort.java
new file mode 100644
index 0000000..c5f2306
--- /dev/null
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesServicePort.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.api;
+
+import com.google.common.base.MoreObjects;
+
+import java.util.Objects;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+public class DefaultKubernetesServicePort implements KubernetesServicePort {
+ private static final String NOT_EMPTY_MSG = "KubernetesServicePort % cannot be empty";
+
+ private Integer port;
+ private Integer nodePort;
+
+ public DefaultKubernetesServicePort(Integer port, Integer nodePort) {
+ this.port = port;
+ this.nodePort = nodePort;
+ }
+
+ @Override
+ public Integer port() {
+ return port;
+ }
+
+ @Override
+ public Integer nodePort() {
+ return nodePort;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ DefaultKubernetesServicePort that = (DefaultKubernetesServicePort) o;
+
+ return Objects.equals(port, that.port) &&
+ Objects.equals(nodePort, that.nodePort);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("port", port.toString())
+ .add("nodePort", nodePort.toString())
+ .toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(port, nodePort);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static final class Builder implements KubernetesServicePort.Builder {
+
+ private Integer port;
+ private Integer nodePort;
+
+ private Builder() {
+ }
+
+ @Override
+ public KubernetesServicePort build() {
+ checkArgument(port != null, NOT_EMPTY_MSG, "port");
+ checkArgument(nodePort != null, NOT_EMPTY_MSG, "nodePort");
+
+ return new DefaultKubernetesServicePort(port, nodePort);
+ }
+
+ @Override
+ public KubernetesServicePort.Builder port(Integer port) {
+ this.port = port;
+ return this;
+ }
+
+ @Override
+ public KubernetesServicePort.Builder nodePort(Integer nodePort) {
+ this.nodePort = nodePort;
+ return this;
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java
index 4e902b0..6b2eaaa 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesExternalLb.java
@@ -39,18 +39,11 @@
IpAddress loadBalancerIp();
/**
- * Returns the set of node port.
+ * Returns the set of service ports.
*
* @return node port
*/
- Set<Integer> nodePortSet();
-
- /**
- * Returns the set of port.
- *
- * @return port number
- */
- Set<Integer> portSet();
+ Set<KubernetesServicePort> servicePorts();
/**
* Returns the set of endpoint.
@@ -130,21 +123,14 @@
*/
Builder loadBalancerIp(IpAddress loadBalancerIp);
- /**
- * Returns kubernetes external load balancer builder with supplied node port set.
- *
- * @param nodePortSet node port set
- * @return external load balancer builder
- */
- Builder nodePortSet(Set<Integer> nodePortSet);
/**
- * Returns kubernetes external load balancer builder with supplied port set.
+ * Returns kubernetes external load balancer builder with supplied service port set.
*
- * @param portSet port set
+ * @param servicePorts service port set
* @return external load balancer builder
*/
- Builder portSet(Set<Integer> portSet);
+ Builder servicePorts(Set<KubernetesServicePort> servicePorts);
/**
* Returns kubernetes external load balancer builder with supplied endpoint set.
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesServicePort.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesServicePort.java
new file mode 100644
index 0000000..7afae49
--- /dev/null
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubernetesServicePort.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.api;
+
+/**
+ * Representation of a Kubernetes service port for kubernetes external load balancer.
+ */
+public interface KubernetesServicePort {
+
+ /**
+ * Returns the port.
+ *
+ * @return port
+ */
+ Integer port();
+
+ /**
+ * Returns the node port.
+ *
+ * @return node port
+ */
+ Integer nodePort();
+
+ interface Builder {
+
+ /**
+ * Builds immutable kubernetes service port instance.
+ *
+ * @return kubernetes service port instance
+ */
+ KubernetesServicePort build();
+
+ /**
+ * Returns kubernetes service port builder with supplied port.
+ *
+ * @param port port
+ * @return kubernetes service port builder
+ */
+ Builder port(Integer port);
+
+ /**
+ * Returns kubernetes service port builder with supplied node port.
+ *
+ * @param nodePort node port
+ * @return kubernetes service port builder
+ */
+ Builder nodePort(Integer nodePort);
+ }
+}
diff --git a/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java b/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java
index 228bb9f..ea50fef 100644
--- a/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java
+++ b/apps/kubevirt-networking/api/src/test/java/org/onosproject/kubevirtnetworking/api/DefaultKubernetesExternalLbTest.java
@@ -35,10 +35,16 @@
private static final String SERVICE_NAME_2 = "service_name_2";
private static final IpAddress LOADBALANCER_IP_1 = IpAddress.valueOf("1.1.1.2");
private static final IpAddress LOADBALANCER_IP_2 = IpAddress.valueOf("2.2.2.2");
- private static final Set<Integer> NODE_PORT_SET_1 = Sets.newHashSet(Integer.valueOf(32080));
- private static final Set<Integer> NODE_PORT_SET_2 = Sets.newHashSet(Integer.valueOf(33080));
- private static final Set<Integer> PORT_SET_1 = Sets.newHashSet(Integer.valueOf(8080));
- private static final Set<Integer> PORT_SET_2 = Sets.newHashSet(Integer.valueOf(9090));
+ private static final KubernetesServicePort SERVICE_PORT_1 = DefaultKubernetesServicePort.builder()
+ .port(Integer.valueOf(8080))
+ .nodePort(Integer.valueOf(31080))
+ .build();
+ private static final KubernetesServicePort SERVICE_PORT_2 = DefaultKubernetesServicePort.builder()
+ .port(Integer.valueOf(8081))
+ .nodePort(Integer.valueOf(31081))
+ .build();
+ private static final Set<KubernetesServicePort> SERVICE_PORT_SET_1 = Sets.newHashSet(SERVICE_PORT_1);
+ private static final Set<KubernetesServicePort> SERVICE_PORT_SET_2 = Sets.newHashSet(SERVICE_PORT_2);
private static final Set<String> ENDPOINT_SET_1 = Sets.newHashSet(String.valueOf("1.1.2.1"));
private static final Set<String> ENDPOINT_SET_2 = Sets.newHashSet(String.valueOf("1.1.2.2"));
private static final String ELECTED_GATEWAY_1 = "gateway1";
@@ -71,8 +77,7 @@
lb1 = DefaultKubernetesExternalLb.builder()
.serviceName(SERVICE_NAME_1)
.loadBalancerIp(LOADBALANCER_IP_1)
- .nodePortSet(NODE_PORT_SET_1)
- .portSet(PORT_SET_1)
+ .servicePorts(SERVICE_PORT_SET_1)
.endpointSet(ENDPOINT_SET_1)
.electedGateway(ELECTED_GATEWAY_1)
.electedWorker(ELECTED_WORKER_1)
@@ -83,8 +88,7 @@
sameAsLb1 = DefaultKubernetesExternalLb.builder()
.serviceName(SERVICE_NAME_1)
.loadBalancerIp(LOADBALANCER_IP_1)
- .nodePortSet(NODE_PORT_SET_1)
- .portSet(PORT_SET_1)
+ .servicePorts(SERVICE_PORT_SET_1)
.endpointSet(ENDPOINT_SET_1)
.electedGateway(ELECTED_GATEWAY_1)
.electedWorker(ELECTED_WORKER_1)
@@ -95,8 +99,7 @@
lb2 = DefaultKubernetesExternalLb.builder()
.serviceName(SERVICE_NAME_2)
.loadBalancerIp(LOADBALANCER_IP_2)
- .nodePortSet(NODE_PORT_SET_2)
- .portSet(PORT_SET_2)
+ .servicePorts(SERVICE_PORT_SET_2)
.endpointSet(ENDPOINT_SET_2)
.electedGateway(ELECTED_GATEWAY_2)
.electedWorker(ELECTED_WORKER_2)
@@ -124,8 +127,7 @@
assertEquals(SERVICE_NAME_1, lb.serviceName());
assertEquals(LOADBALANCER_IP_1, lb.loadBalancerIp());
- assertEquals(NODE_PORT_SET_1, lb.nodePortSet());
- assertEquals(PORT_SET_1, lb.portSet());
+ assertEquals(SERVICE_PORT_SET_1, lb.servicePorts());
assertEquals(ENDPOINT_SET_1, lb.endpointSet());
}
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java
index 47d4a71..fcb7188 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubernetesListServiceCommand.java
@@ -27,6 +27,7 @@
import java.util.List;
import static org.onosproject.kubevirtnetworking.api.Constants.CLI_IP_ADDRESS_LENGTH;
+import static org.onosproject.kubevirtnetworking.api.Constants.CLI_LONG_SERVICE_PORT_LENGTH;
import static org.onosproject.kubevirtnetworking.api.Constants.CLI_MAC_ADDRESS_LENGTH;
import static org.onosproject.kubevirtnetworking.api.Constants.CLI_MARGIN_LENGTH;
import static org.onosproject.kubevirtnetworking.api.Constants.CLI_NAME_LENGTH;
@@ -47,11 +48,12 @@
List<KubernetesExternalLb> elbList = Lists.newArrayList(service.loadBalancers());
String format = genFormatString(ImmutableList.of(CLI_NAME_LENGTH, CLI_IP_ADDRESS_LENGTH,
- CLI_NAME_LENGTH, CLI_NAME_LENGTH, CLI_IP_ADDRESS_LENGTH, CLI_MAC_ADDRESS_LENGTH));
+ CLI_NAME_LENGTH, CLI_NAME_LENGTH, CLI_IP_ADDRESS_LENGTH, CLI_MAC_ADDRESS_LENGTH,
+ CLI_LONG_SERVICE_PORT_LENGTH));
print(format, "Service Name", "Loadbalancer IP", "Elected Gateway", "Elected Worker",
- "Loadbalancer GW IP", "Loadbalancer GW MAC");
+ "Loadbalancer GW IP", "Loadbalancer GW MAC", "Service Port");
for (KubernetesExternalLb elb : elbList) {
String lbIp = elb.loadBalancerIp() == null ? "N/A" : elb.loadBalancerIp().toString();
@@ -59,6 +61,7 @@
String electedWorker = elb.electedWorker() == null ? "N/A" : elb.electedWorker();
String lbGwIp = elb.loadBalancerGwIp() == null ? "N/A" : elb.loadBalancerGwIp().toString();
String lbGwMac = elb.loadBalancerGwMac() == null ? "N/A" : elb.loadBalancerGwMac().toString();
+ String lbServicePort = elb.servicePorts().isEmpty() ? "N/A" : elb.servicePorts().toString();
print(format, StringUtils.substring(elb.serviceName(), 0,
CLI_NAME_LENGTH - CLI_MARGIN_LENGTH),
@@ -71,7 +74,9 @@
StringUtils.substring(lbGwIp, 0,
CLI_IP_ADDRESS_LENGTH - CLI_MARGIN_LENGTH),
StringUtils.substring(lbGwMac, 0,
- CLI_MAC_ADDRESS_LENGTH - CLI_MARGIN_LENGTH)
+ CLI_MAC_ADDRESS_LENGTH - CLI_MARGIN_LENGTH),
+ StringUtils.substring(lbServicePort, 0,
+ CLI_LONG_SERVICE_PORT_LENGTH - CLI_MARGIN_LENGTH)
);
}
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
index 1bd286e..ce78884 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubernetesExternalLbStore.java
@@ -21,10 +21,12 @@
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.kubevirtnetworking.api.DefaultKubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.DefaultKubernetesServicePort;
import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbEvent;
import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbStore;
import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbStoreDelegate;
+import org.onosproject.kubevirtnetworking.api.KubernetesServicePort;
import org.onosproject.store.AbstractStore;
import org.onosproject.store.serializers.KryoNamespaces;
import org.onosproject.store.service.ConsistentMap;
@@ -72,6 +74,8 @@
.register(KryoNamespaces.API)
.register(KubernetesExternalLb.class)
.register(DefaultKubernetesExternalLb.class)
+ .register(KubernetesServicePort.class)
+ .register(DefaultKubernetesServicePort.class)
.register(IpAddress.class)
.register(Collection.class)
.build();
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbArpHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbArpHandler.java
new file mode 100644
index 0000000..5af9834
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbArpHandler.java
@@ -0,0 +1,501 @@
+/*
+ * Copyright 2022-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.impl;
+
+import org.onlab.packet.ARP;
+import org.onlab.packet.EthType;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.VlanId;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnetworking.api.KubevirtFlowRuleService;
+import org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfig;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigAdminService;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigListener;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.kubevirtnode.api.KubevirtNodeAdminService;
+import org.onosproject.kubevirtnode.api.KubevirtNodeEvent;
+import org.onosproject.kubevirtnode.api.KubevirtNodeListener;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.InboundPacket;
+import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketProcessor;
+import org.onosproject.net.packet.PacketService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.nio.ByteBuffer;
+import java.util.Objects;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.GW_ENTRY_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBERNETES_EXTERNAL_LB_FAKE_MAC;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.kubernetesElbMac;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.GATEWAY;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Handles arp packets related to the kubernetes external loadbalancer handler.
+ */
+@Component(immediate = true)
+public class KubernetesExternalLbArpHandler {
+ protected final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected PacketService packetService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubernetesExternalLbConfigAdminService externalLbConfigAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeAdminService nodeAdminService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtFlowRuleService kubevirtFlowRuleService;
+
+ private static final IpAddress NON_ROUTABLE_META_ADDRESS = IpAddress.valueOf("0.0.0.0");
+ private final Timer externalLbGwTimer = new Timer("kubernetes-external-lb-gateway");
+ private final Timer externalLbIntfGwTimer = new Timer("kubernetes-external-lb-intf-gateway");
+ private static final long SECONDS = 1000L;
+ private static final long INITIAL_DELAY = 5 * SECONDS;
+ private static final long TASK_PERIOD = 60 * SECONDS;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+
+ private final PacketProcessor packetProcessor = new InternalPacketProcessor();
+ private final InternalKubernetesExternalLbConfigListener
+ lbConfigListener = new InternalKubernetesExternalLbConfigListener();
+
+ private final InternalNodeEventListener
+ nodeEventListener = new InternalNodeEventListener();
+
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+
+ packetService.addProcessor(packetProcessor, PacketProcessor.director(1));
+ externalLbConfigAdminService.addListener(lbConfigListener);
+ nodeAdminService.addListener(nodeEventListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+ packetService.removeProcessor(packetProcessor);
+ externalLbConfigAdminService.removeListener(lbConfigListener);
+ nodeAdminService.removeListener(nodeEventListener);
+
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private class InternalKubernetesExternalLbConfigListener
+ implements KubernetesExternalLbConfigListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubernetesExternalLbConfigEvent event) {
+ switch (event.type()) {
+ case KUBERNETES_EXTERNAL_LB_CONFIG_CREATED:
+ case KUBERNETES_EXTERNAL_LB_CONFIG_UPDATED:
+ eventExecutor.execute(() -> processConfigCreatedOrUpdated(event.subject()));
+ case KUBERNETES_EXTERNAL_LB_CONFIG_REMOVED:
+ default:
+ //do nothing
+ break;
+ }
+ }
+ private void processConfigCreatedOrUpdated(KubernetesExternalLbConfig config) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ if (config.loadBalancerGwMac() != null) {
+ return;
+ }
+
+ processKubernetesExternalLbConfigMacLearning(config);
+ }
+ }
+
+ private class InternalNodeEventListener implements KubevirtNodeListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+
+ @Override
+ public void event(KubevirtNodeEvent event) {
+ switch (event.type()) {
+ case KUBEVIRT_NODE_COMPLETE:
+ eventExecutor.execute(() -> processNodeCompletion(event.subject()));
+ break;
+ case KUBEVIRT_NODE_INCOMPLETE:
+ case KUBEVIRT_NODE_UPDATED:
+ case KUBEVIRT_NODE_REMOVED:
+ break;
+
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ private void processNodeCompletion(KubevirtNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ if (node.type().equals(GATEWAY)) {
+ KubernetesExternalLbInterface externalLbInterface =
+ node.kubernetesExternalLbInterface();
+
+ if (externalLbInterface != null && externalLbInterface.externalLbGwMac() == null) {
+ processKubernetesExternalLbIntfGwMacLearningForGatewayNode(node);
+ }
+
+ KubernetesExternalLbConfig config =
+ externalLbConfigAdminService.lbConfigs().stream().findAny().orElse(null);
+
+ if (config != null && config.loadBalancerGwMac() == null) {
+ processKubernetesExternalLbConfigMacLearning(config);
+ }
+ }
+ }
+ }
+
+ private void processKubernetesExternalLbConfigMacLearning(KubernetesExternalLbConfig config) {
+ nodeAdminService.completeExternalLbGatewayNodes().forEach(gateway -> {
+ setRuleArpRequestToController(config.loadBalancerGwIp(),
+ KUBERNETES_EXTERNAL_LB_FAKE_MAC, gateway, true);
+ });
+
+ KubevirtNode gateway = nodeAdminService.completeExternalLbGatewayNodes()
+ .stream().findAny().orElse(null);
+ if (gateway == null) {
+ return;
+ }
+ PortNumber externalPatchPortNum = KubevirtNetworkingUtil.externalPatchPortNum(deviceService, gateway);
+
+ if (externalPatchPortNum == null) {
+ log.warn("processKubernetesExternalLbConfigMacLearning" +
+ " called but there's no external patchPort for {}. Stop this task.",
+ gateway);
+ return;
+ }
+
+ retrievePeerMac(NON_ROUTABLE_META_ADDRESS, KUBERNETES_EXTERNAL_LB_FAKE_MAC,
+ config.loadBalancerGwIp(), gateway, externalPatchPortNum);
+ checkKubernetesExternalLbConfigMacRetrieved(config, gateway);
+ }
+
+
+ private void processKubernetesExternalLbIntfGwMacLearningForGatewayNode(KubevirtNode gatewayNode) {
+
+ MacAddress elbIntfMac = kubernetesElbMac(deviceService, gatewayNode);
+ if (elbIntfMac == null) {
+ log.warn("processKubernetesExternalLbGwMacLearningForNode called but elbIntfMac is null. Stop this task.");
+ return;
+ }
+
+ KubernetesExternalLbInterface externalLbInterface = gatewayNode.kubernetesExternalLbInterface();
+
+ setRuleArpRequestToController(externalLbInterface.externalLbGwIp(), elbIntfMac,
+ gatewayNode, true);
+
+ PortNumber elbPatchPortNum = KubevirtNetworkingUtil.elbPatchPortNum(deviceService, gatewayNode);
+
+ if (elbPatchPortNum == null) {
+ log.warn("processKubernetesExternalLbIntfGwMacLearningForGatewayNode" +
+ " called but there's no elb patchPort for {}. Stop this task.",
+ gatewayNode);
+ return;
+ }
+
+ retrievePeerMac(externalLbInterface.externalLbIp(), elbIntfMac,
+ externalLbInterface.externalLbGwIp(), gatewayNode, elbPatchPortNum);
+
+ checkKubernetesExternalLbIntfGwMacRetrieved(gatewayNode);
+ }
+
+ private void checkKubernetesExternalLbIntfGwMacRetrieved(KubevirtNode gateway) {
+ KubernetesExternalLbInterface externalLbInterface = gateway.kubernetesExternalLbInterface();
+
+ MacAddress elbIntfMac = kubernetesElbMac(deviceService, gateway);
+ if (elbIntfMac == null) {
+ log.warn("setDownstreamRules called but elbIntfMac is null. Stop this task.");
+ return;
+ }
+
+ KubernetexExternalLbIntfTimerTask task = new KubernetexExternalLbIntfTimerTask(
+ externalLbInterface.externalLbIp(), elbIntfMac,
+ externalLbInterface.externalLbGwIp(), gateway);
+
+ externalLbIntfGwTimer.schedule(task, INITIAL_DELAY, TASK_PERIOD);
+ }
+
+ private void checkKubernetesExternalLbConfigMacRetrieved(KubernetesExternalLbConfig config,
+ KubevirtNode gateway) {
+ KubernetesExternalLbConfigTimerTask task = new KubernetesExternalLbConfigTimerTask(
+ IpAddress.valueOf("0.0.0.0"), KUBERNETES_EXTERNAL_LB_FAKE_MAC,
+ config.loadBalancerGwIp(), gateway);
+
+ externalLbGwTimer.schedule(task, INITIAL_DELAY, TASK_PERIOD);
+ }
+
+
+ private void setRuleArpRequestToController(IpAddress targetIpAddress,
+ MacAddress dstMac,
+ KubevirtNode gatewayNode,
+ boolean install) {
+ if (targetIpAddress == null || dstMac == null || gatewayNode == null) {
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .matchArpOp(ARP.OP_REPLY)
+ .matchArpSpa(targetIpAddress.getIp4Address())
+ .matchArpTha(dstMac)
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .punt()
+ .build();
+
+ kubevirtFlowRuleService.setRule(
+ appId,
+ gatewayNode.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_ARP_GATEWAY_RULE,
+ GW_ENTRY_TABLE,
+ install
+ );
+ }
+
+
+ private void retrievePeerMac(IpAddress srcIp, MacAddress srcMac,
+ IpAddress peerIp, KubevirtNode gatewayNode,
+ PortNumber portNumber) {
+ log.trace("Sending ARP request to the peer {} to retrieve the MAC address.",
+ peerIp.getIp4Address().toString());
+
+ Ethernet ethRequest = ARP.buildArpRequest(srcMac.toBytes(),
+ srcIp.toOctets(),
+ peerIp.toOctets(), VlanId.NO_VID);
+
+ if (gatewayNode == null) {
+ log.warn("retrievePeerMac called but there's no gateway node for {}. Stop this task.",
+ gatewayNode);
+ return;
+ }
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(portNumber)
+ .build();
+
+ packetService.emit(new DefaultOutboundPacket(
+ gatewayNode.intgBridge(),
+ treatment,
+ ByteBuffer.wrap(ethRequest.serialize())));
+
+ }
+
+ private class KubernetesExternalLbConfigTimerTask extends TimerTask {
+ private final IpAddress srcIp;
+ private final MacAddress srcMac;
+ private final IpAddress peerIp;
+ private final KubevirtNode gatewayNode;
+
+ public KubernetesExternalLbConfigTimerTask(IpAddress srcIp, MacAddress srcMac,
+ IpAddress peerIp, KubevirtNode gatewayNode) {
+ this.srcIp = srcIp;
+ this.srcMac = srcMac;
+ this.peerIp = peerIp;
+ this.gatewayNode = gatewayNode;
+ }
+
+ @Override
+ public void run() {
+ KubernetesExternalLbConfig config =
+ externalLbConfigAdminService.lbConfigs().stream().findAny().orElse(null);
+
+ if (config == null) {
+ return;
+ }
+
+ if (config.loadBalancerGwMac() != null) {
+ log.info("Peer Mac {} for KubernetesExternalLbGateway is retrieved. Stop this task.",
+ config.loadBalancerGwMac());
+ this.cancel();
+ return;
+ }
+
+ PortNumber externalPatchPortNum = KubevirtNetworkingUtil.externalPatchPortNum(deviceService, gatewayNode);
+
+ if (externalPatchPortNum == null) {
+ log.warn("processKubernetesExternalLbConfigMacLearning" +
+ " called but there's no external patchPort for {}. Stop this task.",
+ gatewayNode);
+ return;
+ }
+
+ retrievePeerMac(srcIp, srcMac, peerIp, gatewayNode, externalPatchPortNum);
+ }
+ }
+
+
+ private class KubernetexExternalLbIntfTimerTask extends TimerTask {
+ private final IpAddress srcIp;
+ private final MacAddress srcMac;
+ private final IpAddress peerIp;
+ private final KubevirtNode gatewayNode;
+
+ public KubernetexExternalLbIntfTimerTask(IpAddress srcIp, MacAddress srcMac,
+ IpAddress peerIp, KubevirtNode gatewayNode) {
+ this.srcIp = srcIp;
+ this.srcMac = srcMac;
+ this.peerIp = peerIp;
+ this.gatewayNode = gatewayNode;
+ }
+
+ @Override
+ public void run() {
+
+ KubernetesExternalLbInterface externalLbInterface = gatewayNode.kubernetesExternalLbInterface();
+
+ if (externalLbInterface.externalLbGwMac() != null) {
+ log.info("Peer Mac {} for KubernetesExternalLbIntfGw for node {} is retrieved. Stop this task.",
+ externalLbInterface.externalLbGwMac(), gatewayNode.hostname());
+ this.cancel();
+ return;
+ }
+
+ PortNumber elbPatchPortNum = KubevirtNetworkingUtil.elbPatchPortNum(deviceService, gatewayNode);
+
+ if (elbPatchPortNum == null) {
+ log.warn("processKubernetesExternalLbIntfGwMacLearningForGatewayNode" +
+ " called but there's no elb patchPort for {}. Stop this task.",
+ gatewayNode);
+ return;
+ }
+
+ retrievePeerMac(srcIp, srcMac, peerIp, gatewayNode, elbPatchPortNum);
+ }
+ }
+
+ private class InternalPacketProcessor implements PacketProcessor {
+ @Override
+ public void process(PacketContext context) {
+ if (context.isHandled()) {
+ return;
+ }
+
+ InboundPacket pkt = context.inPacket();
+ Ethernet ethernet = pkt.parsed();
+
+ if (ethernet != null && ethernet.getEtherType() == Ethernet.TYPE_ARP) {
+ processArpPacket(ethernet);
+ }
+ }
+
+ private void processArpPacket(Ethernet ethernet) {
+ ARP arp = (ARP) ethernet.getPayload();
+
+ if (arp.getOpCode() == ARP.OP_REQUEST) {
+ return;
+ }
+ log.trace("ARP request {}", arp);
+
+ KubernetesExternalLbConfig config =
+ externalLbConfigAdminService.lbConfigs().stream().findAny().orElse(null);
+
+ IpAddress spa = Ip4Address.valueOf(arp.getSenderProtocolAddress());
+ MacAddress sha = MacAddress.valueOf(arp.getSenderHardwareAddress());
+
+ if (config != null && config.loadBalancerGwIp().equals(spa)) {
+ externalLbConfigAdminService.updateKubernetesExternalLbConfig(config.updateLbGatewayMac(sha));
+ }
+
+ nodeAdminService.completeExternalLbGatewayNodes().forEach(gateway -> {
+ KubernetesExternalLbInterface externalLbInterface =
+ gateway.kubernetesExternalLbInterface();
+ if (externalLbInterface == null) {
+ return;
+ }
+
+ if (externalLbInterface.externalLbGwIp().equals(spa)) {
+ if (externalLbInterface.externalLbGwMac() == null ||
+ !externalLbInterface.externalLbGwMac().equals(sha)) {
+ nodeAdminService.updateNode(gateway.updateKubernetesElbIntfGwMac(sha));
+ }
+ }
+ });
+ }
+ }
+}
+
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
index 7b1c733..5953c8a 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesExternalLbHandler.java
@@ -80,7 +80,7 @@
public class KubernetesExternalLbHandler {
protected final Logger log = getLogger(getClass());
- private static final int TP_PORT_MINIMUM_NUM = 1025;
+ private static final int TP_PORT_MINIMUM_NUM = 10000;
private static final int TP_PORT_MAXIMUM_NUM = 65535;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
@@ -182,9 +182,9 @@
return;
}
- if (lb.electedGateway() == null || lb.electedWorker() == null) {
+ if (lb.electedGateway() == null || lb.electedWorker() == null || lb.loadBalancerGwMac() == null) {
log.warn("processKubernetesExternalLbCreatedOrUpdated called but electedGateway " +
- "or electedWorker is null. Stop this task.");
+ "or electedWorker or loadBalancerGwMacis null. Stop this task.");
return;
}
@@ -198,7 +198,9 @@
return;
}
- if (lb.electedWorker() == null || oldGatway == null) {
+ if (lb.electedWorker() == null || oldGatway == null || lb.loadBalancerGwMac() == null) {
+ log.warn("processKubernetesExternalLbGatewayChanged called but old electedWorker " +
+ "or electedWorker or loadBalancerGwMacis null. Stop this task.");
return;
}
@@ -326,13 +328,13 @@
return;
}
- lb.nodePortSet().forEach(nodeport -> {
+ lb.servicePorts().forEach(servicePort -> {
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
.matchEthDst(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
.matchIPDst(loadBalancerIp.toIpPrefix())
.matchIPProtocol(IPv4.PROTOCOL_TCP)
- .matchTcpDst(TpPort.tpPort(nodeport.intValue()))
+ .matchTcpDst(TpPort.tpPort(servicePort.port().intValue()))
.build();
ExtensionTreatment natTreatment = RulePopulatorUtil
@@ -351,6 +353,7 @@
.setEthSrc(elbIntfMac)
.setEthDst(externalLbInterface.externalLbGwMac())
.setIpDst(electedWorker.dataIp())
+ .setTcpDst(TpPort.tpPort(servicePort.nodePort().intValue()))
.setOutput(elbBridgePortNum)
.build();
@@ -404,13 +407,13 @@
return;
}
- lb.nodePortSet().forEach(nodePort -> {
+ lb.servicePorts().forEach(servicePort -> {
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
.matchIPSrc(electedWorker.dataIp().toIpPrefix())
.matchIPDst(externalLbInterface.externalLbIp().toIpPrefix())
.matchIPProtocol(IPv4.PROTOCOL_TCP)
- .matchTcpSrc(TpPort.tpPort(nodePort.intValue()));
+ .matchTcpSrc(TpPort.tpPort(servicePort.nodePort().intValue()));
ExtensionTreatment natTreatment = RulePopulatorUtil
.niciraConnTrackTreatmentBuilder(driverService, gateway.intgBridge())
@@ -423,6 +426,7 @@
.setEthSrc(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
.setIpSrc(lb.loadBalancerIp())
.setEthDst(lb.loadBalancerGwMac())
+ .setTcpSrc(TpPort.tpPort(servicePort.port().intValue()))
.extension(natTreatment, gateway.intgBridge())
.transition(GW_DROP_TABLE);
@@ -438,13 +442,12 @@
sBuilder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
.matchIPProtocol(IPv4.PROTOCOL_TCP)
- .matchTcpSrc(TpPort.tpPort(nodePort.intValue()));
-
+ .matchEthSrc(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
+ .matchIPSrc(lb.loadBalancerIp().toIpPrefix())
+ .matchEthDst(lb.loadBalancerGwMac())
+ .matchTcpSrc(TpPort.tpPort(servicePort.port().intValue()));
tBuilder = DefaultTrafficTreatment.builder()
- .setEthSrc(KUBERNETES_EXTERNAL_LB_FAKE_MAC)
- .setIpSrc(lb.loadBalancerIp())
- .setEthDst(lb.loadBalancerGwMac())
.setOutput(externalPatchPortNum);
flowService.setRule(
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
index 2aec742..d80ee25 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubernetesServiceWatcher.java
@@ -34,8 +34,10 @@
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.kubevirtnetworking.api.DefaultKubernetesExternalLb;
+import org.onosproject.kubevirtnetworking.api.DefaultKubernetesServicePort;
import org.onosproject.kubevirtnetworking.api.KubernetesExternalLb;
import org.onosproject.kubevirtnetworking.api.KubernetesExternalLbAdminService;
+import org.onosproject.kubevirtnetworking.api.KubernetesServicePort;
import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfig;
import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent;
import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigListener;
@@ -63,6 +65,7 @@
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.configMapUpdated;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedService;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.k8sClient;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.workerNodeForSpecifiedService;
@@ -175,7 +178,7 @@
switch (event.type()) {
case KUBERNETES_EXTERNAL_LB_CONFIG_CREATED:
case KUBERNETES_EXTERNAL_LB_CONFIG_UPDATED:
- eventExecutor.execute(this::processConfigUpdate);
+ eventExecutor.execute(() -> processConfigUpdate(event.subject()));
break;
case KUBERNETES_EXTERNAL_LB_CONFIG_REMOVED:
default:
@@ -184,11 +187,13 @@
}
}
- private void processConfigUpdate() {
+ private void processConfigUpdate(KubernetesExternalLbConfig externalLbConfig) {
if (!isRelevantHelper()) {
return;
}
- addOrUpdateExternalLoadBalancers();
+ if (configMapUpdated(externalLbConfig)) {
+ addOrUpdateExternalLoadBalancers();
+ }
}
}
@@ -261,7 +266,9 @@
return;
}
- if (!configMapUpdated()) {
+ KubernetesExternalLbConfig config = lbConfigService.lbConfigs().stream().findAny().orElse(null);
+
+ if (!configMapUpdated(config)) {
log.warn("Config map is not set yet. Stop this task");
return;
}
@@ -390,17 +397,6 @@
.findAny().isPresent();
}
- private boolean configMapUpdated() {
- KubernetesExternalLbConfig config = lbConfigService.lbConfigs().stream().findAny().orElse(null);
-
- if (config == null) {
- return false;
- }
-
- return config.configName() != null && config.globalIpRange() != null &&
- config.loadBalancerGwIp() != null && config.loadBalancerGwMac() != null;
- }
-
//Only process if the event when the service type is LoadBalancer
private boolean isLoadBalancerType(Service service) {
return service.getSpec().getType().equals(TYPE_LOADBALANCER);
@@ -421,13 +417,15 @@
return null;
}
- Set<Integer> nodePortSet = Sets.newHashSet();
- Set<Integer> portSet = Sets.newHashSet();
+ Set<KubernetesServicePort> servicePorts = Sets.newHashSet();
Set<String> endpointSet = Sets.newHashSet();
service.getSpec().getPorts().forEach(servicePort -> {
- nodePortSet.add(servicePort.getNodePort());
- portSet.add(servicePort.getPort());
+ if (servicePort.getPort() != null && servicePort.getNodePort() != null) {
+ servicePorts.add(DefaultKubernetesServicePort.builder()
+ .nodePort(servicePort.getNodePort())
+ .port(servicePort.getPort()).build());
+ }
});
nodeService.completeNodes(WORKER).forEach(workerNode -> {
@@ -452,8 +450,7 @@
return DefaultKubernetesExternalLb.builder().serviceName(serviceName)
.loadBalancerIp(IpAddress.valueOf(lbIp))
- .nodePortSet(nodePortSet)
- .portSet(portSet)
+ .servicePorts(servicePorts)
.endpointSet(endpointSet)
.loadBalancerGwIp(IpAddress.valueOf(loadbalancerGatewayIp))
.loadBalancerGwMac(loadBalancerGatewayMac)
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
index 539323f..345e670 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
@@ -53,6 +53,7 @@
import org.onosproject.kubevirtnetworking.api.KubevirtRouterService;
import org.onosproject.kubevirtnode.api.DefaultKubevirtNode;
import org.onosproject.kubevirtnode.api.DefaultKubevirtPhyInterface;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbConfig;
import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtApiConfig;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
@@ -557,16 +558,32 @@
KubernetesExternalLb externalLb) {
//TODO: enhance election logic for a better load balancing
- int numOfGateways = nodeService.completeNodes(GATEWAY).size();
+ int numOfGateways = nodeService.completeExternalLbGatewayNodes().size();
if (numOfGateways == 0) {
return null;
}
- return (KubevirtNode) nodeService.completeNodes(GATEWAY)
+ return (KubevirtNode) nodeService.completeExternalLbGatewayNodes()
.toArray()[externalLb.hashCode() % numOfGateways];
}
/**
+ * Returns whether a mac address in kubernetes external lb config is updated.
+ *
+ * @param externalLbConfig kubernetes external lb config
+ * @return true if a mac address is added
+ */
+ public static boolean configMapUpdated(KubernetesExternalLbConfig externalLbConfig) {
+ if (externalLbConfig == null) {
+ return false;
+ }
+
+ return externalLbConfig.configName() != null && externalLbConfig.globalIpRange() != null &&
+ externalLbConfig.loadBalancerGwIp() != null && externalLbConfig.loadBalancerGwMac() != null;
+ }
+
+
+ /**
* Returns the worker node for the specified kubernetes external lb.
* Among worker nodes, only one worker would serve the traffic from and to the gateway.
* Currently worker node is selected based on modulo operation with external lb hashcode.
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbConfig.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbConfig.java
index 1147ac7..082c1be 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbConfig.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbConfig.java
@@ -103,7 +103,7 @@
return DefaultKubernetesExternalLbConfig.builder()
.configName(configName)
.loadBalancerGwIp(loadBalancerGwIp)
- .loadBalancerGwMac(loadBalancerGwMac)
+ .loadBalancerGwMac(gatewayMac)
.globalIpRange(globalIpRange)
.build();
}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbInterface.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbInterface.java
index fe53b53..676191e 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbInterface.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubernetesExternalLbInterface.java
@@ -109,7 +109,6 @@
checkArgument(elbBridgeName != null, NOT_NULL_MSG, "externalLbBridgeName");
checkArgument(elbIp != null, NOT_NULL_MSG, "externalLbIp");
checkArgument(elbGwIp != null, NOT_NULL_MSG, "externalLbGwIp");
- checkArgument(elbGwMac != null, NOT_NULL_MSG, "externalLbGwMac");
return new DefaultKubernetesExternalLbInterface(elbBridgeName, elbIp, elbGwIp, elbGwMac);
}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java
index 2c3d0be..cc5e43a 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java
@@ -193,6 +193,31 @@
}
@Override
+ public KubevirtNode updateKubernetesElbIntfGwMac(MacAddress macAddress) {
+
+ KubernetesExternalLbInterface externalLbInterface = DefaultKubernetesExternalLbInterface.builder()
+ .externalLbIp(this.kubernetesExternalLbIntf.externalLbIp())
+ .externalLbBridgeName(this.kubernetesExternalLbIntf.externalLbBridgeName())
+ .externallbGwIp(this.kubernetesExternalLbIntf.externalLbGwIp())
+ .externalLbGwMac(macAddress)
+ .build();
+
+ return new Builder()
+ .hostname(hostname)
+ .clusterName(clusterName)
+ .type(type)
+ .intgBridge(intgBridge)
+ .tunBridge(tunBridge)
+ .managementIp(managementIp)
+ .dataIp(dataIp)
+ .state(state)
+ .phyIntfs(phyIntfs)
+ .gatewayBridgeName(gatewayBridgeName)
+ .kubernetesExternalLbInterface(externalLbInterface)
+ .build();
+ }
+
+ @Override
public Collection<KubevirtPhyInterface> phyIntfs() {
if (phyIntfs == null) {
return new ArrayList<>();
@@ -366,10 +391,6 @@
private KubevirtNodeState state;
private Collection<KubevirtPhyInterface> phyIntfs;
private String gatewayBridgeName;
- private String elbBridgeName;
- private IpAddress elbIp;
- private IpAddress elbGwIp;
- private MacAddress elbGwMac;
private KubernetesExternalLbInterface kubernetesExternalLbInterface;
// private constructor not intended to use from external
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java
index 502784d..4e02997 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java
@@ -16,6 +16,7 @@
package org.onosproject.kubevirtnode.api;
import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
@@ -140,6 +141,13 @@
KubevirtNode updateTunBridge(DeviceId deviceId);
/**
+ * Returns new kubevirt node instance with given kubernetes external lb intf gw mac.
+ * @param macAddress kubernetes external lb intf gw mac
+ * @return updated kubevirt node
+ */
+ KubevirtNode updateKubernetesElbIntfGwMac(MacAddress macAddress);
+
+ /**
* Returns a collection of physical interfaces.
*
* @return physical interfaces
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNodeService.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNodeService.java
index e9ed65b..118c97b 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNodeService.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNodeService.java
@@ -59,6 +59,12 @@
Set<KubevirtNode> completeNodes(KubevirtNode.Type type);
/**
+ * Returns all gateway nodes with complete state and external lb interface added.
+ * @return set of kubevirt nodes
+ */
+ Set<KubevirtNode> completeExternalLbGatewayNodes();
+
+ /**
* Returns the node with the specified hostname.
*
* @param hostname hostname
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubernetesExternalLbConfigStore.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubernetesExternalLbConfigStore.java
index 16a816e..bee1d7b 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubernetesExternalLbConfigStore.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DistributedKubernetesExternalLbConfigStore.java
@@ -48,12 +48,9 @@
import static com.google.common.base.Preconditions.checkArgument;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static org.onlab.util.Tools.groupedThreads;
-import static org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent.Type
- .KUBERNETES_EXTERNAL_LB_CONFIG_CREATED;
-import static org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent.Type
- .KUBERNETES_EXTERNAL_LB_CONFIG_REMOVED;
-import static org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent.Type
- .KUBERNETES_EXTERNAL_LB_CONFIG_UPDATED;
+import static org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent.Type.KUBERNETES_EXTERNAL_LB_CONFIG_CREATED;
+import static org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent.Type.KUBERNETES_EXTERNAL_LB_CONFIG_REMOVED;
+import static org.onosproject.kubevirtnode.api.KubernetesExternalLbConfigEvent.Type.KUBERNETES_EXTERNAL_LB_CONFIG_UPDATED;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -128,7 +125,14 @@
lbConfigStore.compute(lbConfig.configName(), (configName, existing) -> {
final String error = lbConfig.configName() + ERR_NOT_FOUND;
checkArgument(existing != null, error);
- return lbConfig;
+
+ if (lbConfig.equals(existing) && lbConfig.loadBalancerGwMac() == null &&
+ existing.loadBalancerGwMac() != null) {
+ return existing;
+ } else {
+ return lbConfig;
+ }
+
});
}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubevirtNodeManager.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubevirtNodeManager.java
index c2b9aad..4a148c8 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubevirtNodeManager.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/KubevirtNodeManager.java
@@ -24,6 +24,7 @@
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.event.ListenerRegistry;
+import org.onosproject.kubevirtnode.api.KubernetesExternalLbInterface;
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtNodeAdminService;
import org.onosproject.kubevirtnode.api.KubevirtNodeEvent;
@@ -59,6 +60,7 @@
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.kubevirtnode.api.Constants.INTEGRATION_BRIDGE;
import static org.onosproject.kubevirtnode.api.Constants.TUNNEL_BRIDGE;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.GATEWAY;
import static org.onosproject.kubevirtnode.impl.OsgiPropertyConstants.OVSDB_PORT;
import static org.onosproject.kubevirtnode.impl.OsgiPropertyConstants.OVSDB_PORT_NUM_DEFAULT;
import static org.onosproject.kubevirtnode.util.KubevirtNodeUtil.genDpidFromName;
@@ -275,6 +277,23 @@
}
@Override
+ public Set<KubevirtNode> completeExternalLbGatewayNodes() {
+ Set<KubevirtNode> nodes = nodeStore.nodes().stream()
+ .filter(node -> node.type() == GATEWAY &&
+ node.state() == KubevirtNodeState.COMPLETE)
+ .filter(node -> {
+ KubernetesExternalLbInterface externalLbInterface = node.kubernetesExternalLbInterface();
+
+ if (externalLbInterface != null) {
+ return true;
+ }
+ return false;
+ })
+ .collect(Collectors.toSet());
+ return ImmutableSet.copyOf(nodes);
+ }
+
+ @Override
public KubevirtNode node(String hostname) {
return nodeStore.node(hostname);
}