Support tenant overlay network mode at kubevirt networking
Change-Id: Ife40e40e3ee5e342ac8b90ddea6eb81744ace18a
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
index 599a615..85b520c 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
@@ -31,6 +31,9 @@
public static final MacAddress DEFAULT_GATEWAY_MAC =
MacAddress.valueOf(DEFAULT_GATEWAY_MAC_STR);
+ public static final String TENANT_TO_TUNNEL_PREFIX = "i-to-t-";
+ public static final String TUNNEL_TO_TENANT_PREFIX = "t-to-i-";
+
// flow table index
public static final int STAT_INBOUND_TABLE = 0;
public static final int VTAP_INBOUND_TABLE = 1;
@@ -54,11 +57,19 @@
// tenant integration bridge flow table index
public static final int TENANT_INBOUND_TABLE = 0;
public static final int TENANT_DHCP_TABLE = 5;
+ public static final int TENANT_ARP_TABLE = 30;
+ public static final int TENANT_ICMP_TABLE = 35;
public static final int TENANT_FORWARDING_TABLE = 80;
+ // tunnel bridge flow table index
+ public static final int TUNNEL_DEFAULT_TABLE = 0;
+
// flow rule priority
- public static final int PRIORITY_SWITCHING_RULE = 30000;
+ public static final int PRIORITY_ICMP_RULE = 43000;
+ public static final int PRIORITY_FORWARDING_RULE = 30000;
public static final int PRIORITY_DHCP_RULE = 42000;
+ public static final int PRIORITY_ARP_GATEWAY_RULE = 41000;
+ public static final int PRIORITY_TUNNEL_RULE = 31000;
// CLI item length
public static final int CLI_ID_LENGTH = 30;
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubevirtNetwork.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubevirtNetwork.java
index 6c0bf8b..251ec0d 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubevirtNetwork.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/DefaultKubevirtNetwork.java
@@ -18,18 +18,24 @@
import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
+import org.onlab.osgi.DefaultServiceDirectory;
import org.onlab.packet.IpAddress;
import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import static com.google.common.base.Preconditions.checkArgument;
+import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.FLAT;
import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.GENEVE;
import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.GRE;
import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.VXLAN;
+import static org.onosproject.net.AnnotationKeys.PORT_NAME;
/**
* Default implementation class of kubevirt network.
@@ -159,6 +165,17 @@
}
@Override
+ public PortNumber tunnelToTenantPort(DeviceId deviceId) {
+ String portName = TUNNEL_TO_TENANT_PREFIX + segmentIdHex(segmentId);
+ Port port = port(deviceId, portName);
+ if (port == null) {
+ return null;
+ } else {
+ return port.number();
+ }
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) {
return true;
@@ -197,6 +214,14 @@
.toString();
}
+ private Port port(DeviceId deviceId, String portName) {
+ DeviceService deviceService = DefaultServiceDirectory.getService(DeviceService.class);
+ return deviceService.getPorts(deviceId).stream()
+ .filter(p -> p.isEnabled() &&
+ Objects.equals(p.annotations().value(PORT_NAME), portName))
+ .findAny().orElse(null);
+ }
+
private String segmentIdHex(String segIdStr) {
int segId = Integer.parseInt(segIdStr);
return String.format("%06x", segId).toLowerCase();
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubevirtNetwork.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubevirtNetwork.java
index 3c99544..90c39df 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubevirtNetwork.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/KubevirtNetwork.java
@@ -17,6 +17,7 @@
import org.onlab.packet.IpAddress;
import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
import java.util.Set;
@@ -138,6 +139,14 @@
DeviceId tenantDeviceId(String hostname);
/**
+ * Returns the tunnel to tenant port number.
+ *
+ * @param deviceId device identifier
+ * @return port number
+ */
+ PortNumber tunnelToTenantPort(DeviceId deviceId);
+
+ /**
* Builder of new network.
*/
interface Builder {
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubevirtSyncRulesCommand.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubevirtSyncRulesCommand.java
new file mode 100644
index 0000000..2c0f57d
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubevirtSyncRulesCommand.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2021-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.cli;
+
+import org.apache.karaf.shell.api.action.Command;
+import org.apache.karaf.shell.api.action.lifecycle.Service;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.kubevirtnode.api.KubevirtNodeAdminService;
+
+import static java.lang.Thread.sleep;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
+import static org.onosproject.kubevirtnode.api.KubevirtNodeState.COMPLETE;
+import static org.onosproject.kubevirtnode.api.KubevirtNodeState.INIT;
+
+/**
+ * Re-installs flow rules for KubeVirt networking.
+ */
+@Service
+@Command(scope = "onos", name = "kubevirt-sync-rules",
+ description = "Re-installs flow rules for KubeVirt networking")
+public class KubevirtSyncRulesCommand extends AbstractShellCommand {
+
+ private static final long SLEEP_MS = 5000; // we wait 5s for init each node
+ private static final long TIMEOUT_MS = 10000; // we wait 10s
+
+ private static final String SUCCESS_MSG = "Successfully synchronize flow rules for node %s!";
+ private static final String FAIL_MSG = "Failed to synchronize flow rules for node %s.";
+
+ @Override
+ protected void doExecute() throws Exception {
+ // All handlers in this application reacts the node complete event and
+ // tries to re-configure flow rules for the complete node.
+ KubevirtNodeAdminService nodeAdminService = get(KubevirtNodeAdminService.class);
+ if (nodeAdminService == null) {
+ error("Failed to re-install flow rules for OpenStack networking.");
+ return;
+ }
+
+ nodeAdminService.completeNodes(WORKER).forEach(node ->
+ syncRulesBaseForNode(nodeAdminService, node));
+
+ print("Successfully requested re-installing flow rules.");
+ }
+
+ private void syncRulesBaseForNode(KubevirtNodeAdminService service, KubevirtNode node) {
+ KubevirtNode updated = node.updateState(INIT);
+ service.updateNode(updated);
+
+ boolean result = true;
+ long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
+
+ while (service.node(node.hostname()).state() != COMPLETE) {
+ long waitMs = timeoutExpiredMs - System.currentTimeMillis();
+
+ try {
+ sleep(SLEEP_MS);
+ } catch (InterruptedException e) {
+ error("Exception caused during node synchronization...");
+ }
+
+ if (service.node(node.hostname()).state() == COMPLETE) {
+ break;
+ } else {
+ service.updateNode(updated);
+ print("Failed to synchronize flow rules, retrying...");
+ }
+
+ if (waitMs <= 0) {
+ result = false;
+ break;
+ }
+ }
+
+ if (result) {
+ print(SUCCESS_MSG, node.hostname());
+ } else {
+ error(FAIL_MSG, node.hostname());
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
index e538ef7..6572033 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
@@ -16,9 +16,13 @@
package org.onosproject.kubevirtnetworking.impl;
import com.google.common.collect.Lists;
+import org.onlab.packet.ARP;
+import org.onlab.packet.EthType;
import org.onlab.packet.Ethernet;
import org.onlab.packet.IPv4;
+import org.onlab.packet.Ip4Address;
import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
import org.onlab.packet.TpPort;
import org.onlab.packet.UDP;
import org.onosproject.cluster.ClusterService;
@@ -65,14 +69,31 @@
import static java.lang.Thread.sleep;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.packet.ICMP.CODE_ECHO_REQEUST;
+import static org.onlab.packet.ICMP.TYPE_ECHO_REPLY;
+import static org.onlab.packet.ICMP.TYPE_ECHO_REQUEST;
import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.DEFAULT_GATEWAY_MAC;
import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_DHCP_RULE;
-import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_SWITCHING_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_FORWARDING_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ICMP_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_ARP_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_DHCP_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_FORWARDING_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_ICMP_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_INBOUND_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_TO_TUNNEL_PREFIX;
+import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.segmentIdHex;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.NXM_NX_IP_TTL;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.NXM_OF_ICMP_TYPE;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildLoadExtension;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveArpSpaToTpaExtension;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveIpSrcToDstExtension;
import static org.onosproject.kubevirtnode.api.Constants.TUNNEL_BRIDGE;
import static org.slf4j.LoggerFactory.getLogger;
@@ -86,9 +107,7 @@
private static final int DEFAULT_OFPORT = 6653;
private static final int DPID_BEGIN = 3;
private static final long SLEEP_MS = 3000; // we wait 3s for init each node
-
- public static final String INTEGRATION_TO_TUNNEL_PREFIX = "i-to-t-";
- public static final String TUNNEL_TO_INTEGRATION_PREFIX = "t-to-i-";
+ private static final int DEFAULT_TTL = 0xff;
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected CoreService coreService;
@@ -195,29 +214,29 @@
InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
- String intToTunIntf =
- INTEGRATION_TO_TUNNEL_PREFIX + segmentIdHex(network.segmentId());
- String tunToIntIntf =
- TUNNEL_TO_INTEGRATION_PREFIX + segmentIdHex(network.segmentId());
+ String tenantToTunIntf =
+ TENANT_TO_TUNNEL_PREFIX + segmentIdHex(network.segmentId());
+ String tunToTenantIntf =
+ TUNNEL_TO_TENANT_PREFIX + segmentIdHex(network.segmentId());
- // integration bridge -> tunnel bridge
- PatchDescription brIntTunPatchDesc =
+ // tenant bridge -> tunnel bridge
+ PatchDescription brTenantTunPatchDesc =
DefaultPatchDescription.builder()
.deviceId(network.tenantBridgeName())
- .ifaceName(intToTunIntf)
- .peer(tunToIntIntf)
+ .ifaceName(tenantToTunIntf)
+ .peer(tunToTenantIntf)
.build();
- ifaceConfig.addPatchMode(intToTunIntf, brIntTunPatchDesc);
+ ifaceConfig.addPatchMode(tenantToTunIntf, brTenantTunPatchDesc);
- // tunnel bridge -> integration bridge
- PatchDescription brTunIntPatchDesc =
+ // tunnel bridge -> tenant bridge
+ PatchDescription brTunTenantPatchDesc =
DefaultPatchDescription.builder()
.deviceId(TUNNEL_BRIDGE)
- .ifaceName(tunToIntIntf)
- .peer(intToTunIntf)
+ .ifaceName(tunToTenantIntf)
+ .peer(tenantToTunIntf)
.build();
- ifaceConfig.addPatchMode(tunToIntIntf, brTunIntPatchDesc);
+ ifaceConfig.addPatchMode(tunToTenantIntf, brTunTenantPatchDesc);
}
private void removePatchInterface(KubevirtNode node, KubevirtNetwork network) {
@@ -230,7 +249,7 @@
InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
- String tunToIntIntf = TUNNEL_TO_INTEGRATION_PREFIX + segmentIdHex(network.segmentId());
+ String tunToIntIntf = TUNNEL_TO_TENANT_PREFIX + segmentIdHex(network.segmentId());
ifaceConfig.removePatchMode(tunToIntIntf);
}
@@ -249,10 +268,14 @@
}
flowService.connectTables(deviceId, TENANT_INBOUND_TABLE, TENANT_DHCP_TABLE);
- flowService.connectTables(deviceId, TENANT_DHCP_TABLE, TENANT_FORWARDING_TABLE);
+ flowService.connectTables(deviceId, TENANT_DHCP_TABLE, TENANT_ARP_TABLE);
+ flowService.connectTables(deviceId, TENANT_ARP_TABLE, TENANT_ICMP_TABLE);
+ flowService.connectTables(deviceId, TENANT_ICMP_TABLE, TENANT_FORWARDING_TABLE);
setDhcpRule(deviceId, true);
setForwardingRule(deviceId, true);
+ setGatewayArpRule(node, network, true);
+ setGatewayIcmpRule(node, network, true);
log.info("Install default flow rules for tenant bridge {}", network.tenantBridgeName());
}
@@ -279,7 +302,7 @@
install);
}
- public void setForwardingRule(DeviceId deviceId, boolean install) {
+ private void setForwardingRule(DeviceId deviceId, boolean install) {
TrafficSelector selector = DefaultTrafficSelector.builder().build();
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
.setOutput(PortNumber.NORMAL)
@@ -290,11 +313,71 @@
deviceId,
selector,
treatment,
- PRIORITY_SWITCHING_RULE,
+ PRIORITY_FORWARDING_RULE,
TENANT_FORWARDING_TABLE,
install);
}
+ private void setGatewayArpRule(KubevirtNode node, KubevirtNetwork network, boolean install) {
+ Device device = deviceService.getDevice(network.tenantDeviceId(node.hostname()));
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(Ip4Address.valueOf(network.gatewayIp().toString()));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+ tBuilder.extension(buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(buildMoveArpShaToThaExtension(device), device.id())
+ .extension(buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpOp(ARP.OP_REPLY)
+ .setArpSha(DEFAULT_GATEWAY_MAC)
+ .setArpSpa(Ip4Address.valueOf(network.gatewayIp().toString()))
+ .setEthSrc(DEFAULT_GATEWAY_MAC)
+ .setOutput(PortNumber.IN_PORT);
+
+ flowService.setRule(
+ appId,
+ device.id(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_ARP_GATEWAY_RULE,
+ TENANT_ARP_TABLE,
+ install
+ );
+ }
+
+ private void setGatewayIcmpRule(KubevirtNode node, KubevirtNetwork network, boolean install) {
+ DeviceId deviceId = network.tenantDeviceId(node.hostname());
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(IPv4.PROTOCOL_ICMP)
+ .matchIcmpType(TYPE_ECHO_REQUEST)
+ .matchIcmpCode(CODE_ECHO_REQEUST)
+ .matchIPDst(IpPrefix.valueOf(network.gatewayIp(), 32));
+
+ Device device = deviceService.getDevice(deviceId);
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .extension(buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(buildMoveIpSrcToDstExtension(device), device.id())
+ .extension(buildLoadExtension(device,
+ NXM_NX_IP_TTL, DEFAULT_TTL), device.id())
+ .extension(buildLoadExtension(device,
+ NXM_OF_ICMP_TYPE, TYPE_ECHO_REPLY), device.id())
+ .setIpSrc(network.gatewayIp())
+ .setEthSrc(DEFAULT_GATEWAY_MAC)
+ .setOutput(PortNumber.IN_PORT);
+
+ flowService.setRule(
+ appId,
+ deviceId,
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_ICMP_RULE,
+ TENANT_ICMP_TABLE,
+ install);
+ }
+
private class InternalNetworkEventListener implements KubevirtNetworkListener {
private boolean isRelevantHelper() {
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtPodPortMapper.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtPodPortMapper.java
index d9076da..a5379e8 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtPodPortMapper.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtPodPortMapper.java
@@ -155,12 +155,6 @@
return;
}
- KubernetesClient client = k8sClient(kubevirtApiConfigService);
-
- if (client == null) {
- return;
- }
-
Map<String, String> annots = pod.getMetadata().getAnnotations();
if (annots == null) {
return;
@@ -206,12 +200,6 @@
return;
}
- KubernetesClient client = k8sClient(kubevirtApiConfigService);
-
- if (client == null) {
- return;
- }
-
KubevirtPort port = getPort(kubevirtNetworkAdminService.networks(), pod);
if (port == null) {
return;
@@ -244,6 +232,12 @@
Map<String, String> annots = pod.getMetadata().getAnnotations();
annots.put(NETWORK_STATUS_KEY, networkStatus.toString(4));
+ KubernetesClient client = k8sClient(kubevirtApiConfigService);
+
+ if (client == null) {
+ return;
+ }
+
client.pods().inNamespace(pod.getMetadata().getNamespace())
.withName(pod.getMetadata().getName())
.edit(r -> new PodBuilder(r)
@@ -263,12 +257,6 @@
return;
}
- KubernetesClient client = k8sClient(kubevirtApiConfigService);
-
- if (client == null) {
- return;
- }
-
KubevirtPort port = getPort(kubevirtNetworkAdminService.networks(), pod);
if (port == null) {
return;
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingPhysicalHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingPhysicalHandler.java
index 96a31da..c8089fb 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingPhysicalHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingPhysicalHandler.java
@@ -49,7 +49,7 @@
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
import static org.onosproject.kubevirtnetworking.api.Constants.PRE_FLAT_TABLE;
-import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_SWITCHING_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_FORWARDING_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.VTAG_TABLE;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.structurePortName;
import static org.onosproject.kubevirtnode.api.Constants.INTEGRATION_TO_PHYSICAL_PREFIX;
@@ -57,7 +57,7 @@
import static org.slf4j.LoggerFactory.getLogger;
/**
- * Populates switching flow rules on OVS for the physical interfaces.
+ * Populates switching flow rules on OVS for the provider network (underlay).
*/
@Component(immediate = true)
public class KubevirtSwitchingPhysicalHandler {
@@ -121,7 +121,7 @@
deviceId,
selector.build(),
treatment.build(),
- PRIORITY_SWITCHING_RULE,
+ PRIORITY_FORWARDING_RULE,
VTAG_TABLE,
install);
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java
new file mode 100644
index 0000000..bc124c6
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java
@@ -0,0 +1,359 @@
+/*
+ * Copyright 2021-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.impl;
+
+import io.fabric8.kubernetes.api.model.Pod;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpPrefix;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnetworking.api.KubevirtFlowRuleService;
+import org.onosproject.kubevirtnetworking.api.KubevirtNetwork;
+import org.onosproject.kubevirtnetworking.api.KubevirtNetworkService;
+import org.onosproject.kubevirtnetworking.api.KubevirtPodEvent;
+import org.onosproject.kubevirtnetworking.api.KubevirtPodListener;
+import org.onosproject.kubevirtnetworking.api.KubevirtPodService;
+import org.onosproject.kubevirtnetworking.api.KubevirtPort;
+import org.onosproject.kubevirtnetworking.api.KubevirtPortService;
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.kubevirtnode.api.KubevirtNodeEvent;
+import org.onosproject.kubevirtnode.api.KubevirtNodeListener;
+import org.onosproject.kubevirtnode.api.KubevirtNodeService;
+import org.onosproject.mastership.MastershipService;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_TUNNEL_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_DEFAULT_TABLE;
+import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.FLAT;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getPort;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.tunnelPort;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.tunnelToTenantPort;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildExtension;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.MASTER;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Populates switching flow rules on OVS for the tenant network (overlay).
+ */
+@Component(immediate = true)
+public class KubevirtSwitchingTenantHandler {
+ private final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected MastershipService mastershipService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DeviceService deviceService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DriverService driverService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtFlowRuleService flowRuleService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeService kubevirtNodeService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNetworkService kubevirtNetworkService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtPortService kubevirtPortService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtPodService kubevirtPodService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+
+ private final InternalKubevirtPodListener kubevirtPodListener =
+ new InternalKubevirtPodListener();
+ private final InternalKubevirtNodeListener kubevirtNodeListener =
+ new InternalKubevirtNodeListener();
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ kubevirtPodService.addListener(kubevirtPodListener);
+ kubevirtNodeService.addListener(kubevirtNodeListener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ kubevirtPodService.removeListener(kubevirtPodListener);
+ kubevirtNodeService.removeListener(kubevirtNodeListener);
+ leadershipService.withdraw(appId.name());
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private KubevirtPort getPortByPod(Pod pod) {
+ return getPort(kubevirtNetworkService.networks(), pod);
+ }
+
+ private void setIngressRules(Pod pod, boolean install) {
+ KubevirtPort port = getPortByPod(pod);
+
+ if (port == null) {
+ return;
+ }
+
+ if (port.ipAddress() == null) {
+ return;
+ }
+
+ KubevirtNetwork network = kubevirtNetworkService.network(port.networkId());
+
+ if (network == null) {
+ return;
+ }
+
+ // TODO: need to handle VLAN case
+ if (network.type() == FLAT) {
+ return;
+ }
+
+ if (network.segmentId() == null) {
+ return;
+ }
+
+ KubevirtNode localNode = kubevirtNodeService.node(pod.getSpec().getNodeName());
+ if (localNode == null || localNode.type() == MASTER) {
+ return;
+ }
+
+ PortNumber patchPortNumber = tunnelToTenantPort(localNode, network);
+ if (patchPortNumber == null) {
+ return;
+ }
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchTunnelId(Long.parseLong(network.segmentId()));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setOutput(patchPortNumber);
+
+ flowRuleService.setRule(
+ appId,
+ localNode.tunBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_TUNNEL_RULE,
+ TUNNEL_DEFAULT_TABLE,
+ install);
+
+ log.debug("Install ingress rules for instance {}, segment ID {}",
+ port.ipAddress(), network.segmentId());
+ }
+
+ private void setEgressRules(Pod pod, boolean install) {
+ KubevirtNode localNode = kubevirtNodeService.node(pod.getSpec().getNodeName());
+
+ if (localNode == null) {
+ return;
+ }
+
+ if (localNode.type() == MASTER) {
+ return;
+ }
+
+ KubevirtPort port = getPortByPod(pod);
+
+ if (port == null) {
+ return;
+ }
+
+ if (port.ipAddress() == null) {
+ return;
+ }
+
+ KubevirtNetwork network = kubevirtNetworkService.network(port.networkId());
+
+ if (network == null) {
+ return;
+ }
+
+ // TODO: need to handle VLAN case
+ if (network.type() == FLAT) {
+ return;
+ }
+
+ if (network.segmentId() == null) {
+ return;
+ }
+
+ for (KubevirtNode remoteNode : kubevirtNodeService.completeNodes(WORKER)) {
+ if (remoteNode.hostname().equals(localNode.hostname())) {
+ continue;
+ }
+
+ PortNumber patchPortNumber = tunnelToTenantPort(remoteNode, network);
+ if (patchPortNumber == null) {
+ return;
+ }
+
+ PortNumber tunnelPortNumber = tunnelPort(remoteNode, network);
+ if (tunnelPortNumber == null) {
+ return;
+ }
+
+ TrafficSelector.Builder sIpBuilder = DefaultTrafficSelector.builder()
+ .matchInPort(patchPortNumber)
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(port.ipAddress(), 32));
+
+ TrafficSelector.Builder sArpBuilder = DefaultTrafficSelector.builder()
+ .matchInPort(patchPortNumber)
+ .matchEthType(Ethernet.TYPE_ARP)
+ .matchArpTpa(Ip4Address.valueOf(port.ipAddress().toString()));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setTunnelId(Long.parseLong(network.segmentId()))
+ .extension(buildExtension(
+ deviceService,
+ remoteNode.tunBridge(),
+ localNode.dataIp().getIp4Address()),
+ remoteNode.tunBridge())
+ .setOutput(tunnelPortNumber);
+
+ flowRuleService.setRule(
+ appId,
+ remoteNode.tunBridge(),
+ sIpBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_TUNNEL_RULE,
+ TUNNEL_DEFAULT_TABLE,
+ install);
+
+ flowRuleService.setRule(
+ appId,
+ remoteNode.tunBridge(),
+ sArpBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_TUNNEL_RULE,
+ TUNNEL_DEFAULT_TABLE,
+ install);
+ }
+
+ log.debug("Install egress rules for instance {}, segment ID {}",
+ port.ipAddress(), network.segmentId());
+ }
+
+ private class InternalKubevirtNodeListener implements KubevirtNodeListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubevirtNodeEvent event) {
+
+ switch (event.type()) {
+ case KUBEVIRT_NODE_COMPLETE:
+ eventExecutor.execute(() -> processNodeCompletion(event.subject()));
+ break;
+ case KUBEVIRT_NODE_INCOMPLETE:
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ private void processNodeCompletion(KubevirtNode node) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ kubevirtPodService.pods().stream()
+ .filter(pod -> node.hostname().equals(pod.getSpec().getNodeName()))
+ .forEach(pod -> {
+ setIngressRules(pod, true);
+ setEgressRules(pod, true);
+ });
+ }
+ }
+
+ private class InternalKubevirtPodListener implements KubevirtPodListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubevirtPodEvent event) {
+
+ switch (event.type()) {
+ case KUBEVIRT_POD_UPDATED:
+ eventExecutor.execute(() -> processPodUpdate(event.subject()));
+ break;
+ case KUBEVIRT_POD_REMOVED:
+ eventExecutor.execute(() -> processPodRemoval(event.subject()));
+ break;
+ default:
+ // do nothing
+ break;
+ }
+ }
+
+ private void processPodUpdate(Pod pod) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setIngressRules(pod, true);
+ setEgressRules(pod, true);
+ }
+
+ private void processPodRemoval(Pod pod) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ setIngressRules(pod, false);
+ setEgressRules(pod, false);
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
index 48d9bc0..d759db0 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
@@ -26,6 +26,7 @@
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
+import org.onlab.osgi.DefaultServiceDirectory;
import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
import org.onosproject.cfg.ConfigProperty;
@@ -34,6 +35,11 @@
import org.onosproject.kubevirtnetworking.api.KubevirtPort;
import org.onosproject.kubevirtnode.api.KubevirtApiConfig;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,10 +48,14 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
+import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
+import static org.onosproject.net.AnnotationKeys.PORT_NAME;
+
/**
* An utility that used in KubeVirt networking app.
*/
@@ -233,7 +243,6 @@
if (config.scheme() == KubevirtApiConfig.Scheme.HTTPS) {
configBuilder.withTrustCerts(true)
- .withOauthToken(config.token())
.withCaCertData(config.caCertData())
.withClientCertData(config.clientCertData())
.withClientKeyData(config.clientKeyData());
@@ -277,6 +286,27 @@
}
/**
+ * Obtains the tunnel port number with the given network and node.
+ *
+ * @param network kubevirt network
+ * @param node kubevirt node
+ * @return tunnel port number
+ */
+ public static PortNumber tunnelPort(KubevirtNetwork network, KubevirtNode node) {
+ switch (network.type()) {
+ case VXLAN:
+ return node.vxlanPort();
+ case GRE:
+ return node.grePort();
+ case GENEVE:
+ return node.genevePort();
+ default:
+ break;
+ }
+ return null;
+ }
+
+ /**
* Obtains the kubevirt port from kubevirt POD.
*
* @param networks set of existing kubevirt networks
@@ -331,4 +361,64 @@
return null;
}
+
+ /**
+ * Obtains the tunnel bridge to tenant bridge patch port number.
+ *
+ * @param node kubevirt node
+ * @param network kubevirt network
+ * @return patch port number
+ */
+ public static PortNumber tunnelToTenantPort(KubevirtNode node, KubevirtNetwork network) {
+ if (network.segmentId() == null) {
+ return null;
+ }
+
+ if (node.tunBridge() == null) {
+ return null;
+ }
+
+ String tunToTenantPortName = TUNNEL_TO_TENANT_PREFIX + segmentIdHex(network.segmentId());
+ return portNumber(node.tunBridge(), tunToTenantPortName);
+ }
+
+ /**
+ * Obtains the tunnel port number of the given node.
+ *
+ * @param node kubevirt node
+ * @param network kubevirt network
+ * @return tunnel port number
+ */
+ public static PortNumber tunnelPort(KubevirtNode node, KubevirtNetwork network) {
+ if (network.segmentId() == null) {
+ return null;
+ }
+
+ if (node.tunBridge() == null) {
+ return null;
+ }
+
+ switch (network.type()) {
+ case VXLAN:
+ return node.vxlanPort();
+ case GRE:
+ return node.grePort();
+ case GENEVE:
+ return node.genevePort();
+ case FLAT:
+ default:
+ // do nothing
+ return null;
+ }
+ }
+
+ private static PortNumber portNumber(DeviceId deviceId, String portName) {
+ DeviceService deviceService = DefaultServiceDirectory.getService(DeviceService.class);
+ Port port = deviceService.getPorts(deviceId).stream()
+ .filter(p -> p.isEnabled() &&
+ Objects.equals(p.annotations().value(PORT_NAME), portName))
+ .findAny().orElse(null);
+ return port != null ? port.number() : null;
+ }
+
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/RulePopulatorUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/RulePopulatorUtil.java
new file mode 100644
index 0000000..f005c5f
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/RulePopulatorUtil.java
@@ -0,0 +1,210 @@
+/*
+ * Copyright 2021-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.util;
+
+import org.onlab.packet.Ip4Address;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.instructions.ExtensionPropertyException;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.slf4j.Logger;
+
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_LOAD;
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_MOV_ARP_SHA_TO_THA;
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_MOV_ARP_SPA_TO_TPA;
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_MOV_ETH_SRC_TO_DST;
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_MOV_IP_SRC_TO_DST;
+import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Provides common methods to help populating flow rules for SONA applications.
+ */
+public final class RulePopulatorUtil {
+
+ private static final Logger log = getLogger(RulePopulatorUtil.class);
+
+ private static final int OFF_SET_BIT = 0;
+ private static final int REMAINDER_BIT = 8;
+
+ private static final String OFF_SET_N_BITS = "ofsNbits";
+ private static final String DESTINATION = "dst";
+ private static final String VALUE = "value";
+ private static final String TUNNEL_DST = "tunnelDst";
+
+ // layer 3 nicira fields
+ private static final int NXM_OF_IP_SRC = 0x00000e04;
+ private static final int NXM_OF_IP_DST = 0x00001004;
+ private static final int NXM_OF_IP_PROT = 0x00000c01;
+
+ public static final int NXM_NX_IP_TTL = 0x00013a01;
+ public static final int NXM_NX_IP_FRAG = 0x00013401;
+ public static final int NXM_OF_ARP_OP = 0x00001e02;
+ public static final int NXM_OF_ARP_SPA = 0x00002004;
+ public static final int NXM_OF_ARP_TPA = 0x00002204;
+ public static final int NXM_NX_ARP_SHA = 0x00012206;
+ public static final int NXM_NX_ARP_THA = 0x00012406;
+
+ // layer 4 nicira fields
+ public static final int NXM_OF_TCP_SRC = 0x00001202;
+ public static final int NXM_OF_TCP_DST = 0x00001402;
+ public static final int NXM_NX_TCP_FLAGS = 0x00014402;
+ public static final int NXM_OF_UDP_SRC = 0x00001602;
+ public static final int NXM_OF_UDP_DST = 0x00001802;
+
+ public static final int NXM_OF_ICMP_TYPE = 0x00001a01;
+ public static final int NXM_OF_ICMP_CODE = 0x00001c01;
+
+ private RulePopulatorUtil() {
+ }
+
+ /**
+ * Returns the nicira load extension treatment.
+ *
+ * @param device device instance
+ * @param field field code
+ * @param value value to load
+ * @return load extension treatment
+ */
+ public static ExtensionTreatment buildLoadExtension(Device device,
+ long field,
+ long value) {
+ if (!checkTreatmentResolver(device)) {
+ return null;
+ }
+
+ ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+ ExtensionTreatment treatment =
+ resolver.getExtensionInstruction(NICIRA_LOAD.type());
+
+ int ofsNbits = OFF_SET_BIT << 6 | (REMAINDER_BIT - 1);
+
+ try {
+ treatment.setPropertyValue(OFF_SET_N_BITS, ofsNbits);
+ treatment.setPropertyValue(DESTINATION, field);
+ treatment.setPropertyValue(VALUE, value);
+ return treatment;
+ } catch (ExtensionPropertyException e) {
+ log.error("Failed to set nicira load extension treatment for {}",
+ device.id());
+ return null;
+ }
+ }
+
+ /**
+ * Returns the nicira move source MAC to destination MAC extension treatment.
+ *
+ * @param device device instance
+ * @return move extension treatment
+ */
+ public static ExtensionTreatment buildMoveEthSrcToDstExtension(Device device) {
+ if (!checkTreatmentResolver(device)) {
+ return null;
+ }
+
+ ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+ return resolver.getExtensionInstruction(NICIRA_MOV_ETH_SRC_TO_DST.type());
+ }
+
+ /**
+ * Returns the nicira move source IP to destination IP extension treatment.
+ *
+ * @param device device instance
+ * @return move extension treatment
+ */
+ public static ExtensionTreatment buildMoveIpSrcToDstExtension(Device device) {
+ if (!checkTreatmentResolver(device)) {
+ return null;
+ }
+
+ ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+ return resolver.getExtensionInstruction(NICIRA_MOV_IP_SRC_TO_DST.type());
+ }
+
+ /**
+ * Returns the nicira move ARP SHA to THA extension treatment.
+ *
+ * @param device device instance
+ * @return move extension treatment
+ */
+ public static ExtensionTreatment buildMoveArpShaToThaExtension(Device device) {
+ if (!checkTreatmentResolver(device)) {
+ return null;
+ }
+
+ ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+ return resolver.getExtensionInstruction(NICIRA_MOV_ARP_SHA_TO_THA.type());
+ }
+
+ /**
+ * Returns the nicira move ARP SPA to TPA extension treatment.
+ *
+ * @param device device instance
+ * @return move extension treatment
+ */
+ public static ExtensionTreatment buildMoveArpSpaToTpaExtension(Device device) {
+ if (!checkTreatmentResolver(device)) {
+ return null;
+ }
+
+ ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+ return resolver.getExtensionInstruction(NICIRA_MOV_ARP_SPA_TO_TPA.type());
+ }
+
+ private static boolean checkTreatmentResolver(Device device) {
+ if (device == null || !device.is(ExtensionTreatmentResolver.class)) {
+ log.warn("Nicira extension treatment is not supported");
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Returns tunnel destination extension treatment object.
+ *
+ * @param deviceService driver service
+ * @param deviceId device id to apply this treatment
+ * @param remoteIp tunnel destination ip address
+ * @return extension treatment
+ */
+ public static ExtensionTreatment buildExtension(DeviceService deviceService,
+ DeviceId deviceId,
+ Ip4Address remoteIp) {
+ Device device = deviceService.getDevice(deviceId);
+ if (!checkTreatmentResolver(device)) {
+ return null;
+ }
+
+ if (device == null) {
+ return null;
+ }
+
+ ExtensionTreatmentResolver resolver = device.as(ExtensionTreatmentResolver.class);
+ ExtensionTreatment treatment =
+ resolver.getExtensionInstruction(NICIRA_SET_TUNNEL_DST.type());
+ try {
+ treatment.setPropertyValue(TUNNEL_DST, remoteIp);
+ return treatment;
+ } catch (ExtensionPropertyException e) {
+ log.warn("Failed to get tunnelDst extension treatment for {} " +
+ "because of {}", deviceId, e);
+ return null;
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/web/KubevirtManagementWebResource.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/web/KubevirtManagementWebResource.java
new file mode 100644
index 0000000..a1587a5
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/web/KubevirtManagementWebResource.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2021-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.web;
+
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.kubevirtnode.api.KubevirtNodeAdminService;
+import org.onosproject.rest.AbstractWebResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import static java.lang.Thread.sleep;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
+import static org.onosproject.kubevirtnode.api.KubevirtNodeState.COMPLETE;
+import static org.onosproject.kubevirtnode.api.KubevirtNodeState.INIT;
+
+/**
+ * REST interface for synchronizing kubevirt network states and rules.
+ */
+@Path("management")
+public class KubevirtManagementWebResource extends AbstractWebResource {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ private static final long SLEEP_MS = 5000; // we wait 5s for init each node
+ private static final long TIMEOUT_MS = 10000; // we wait 10s
+
+ private final KubevirtNodeAdminService nodeAdminService =
+ get(KubevirtNodeAdminService.class);
+
+ /**
+ * Synchronizes the flow rules.
+ *
+ * @return 200 OK with sync result, 404 not found
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("sync/rules")
+ public Response syncRules() {
+
+ nodeAdminService.completeNodes(WORKER).forEach(this::syncRulesBase);
+ return ok(mapper().createObjectNode()).build();
+ }
+
+ private void syncRulesBase(KubevirtNode node) {
+ KubevirtNode updated = node.updateState(INIT);
+ nodeAdminService.updateNode(updated);
+
+ boolean result = true;
+ long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
+
+ while (nodeAdminService.node(node.hostname()).state() != COMPLETE) {
+
+ long waitMs = timeoutExpiredMs - System.currentTimeMillis();
+
+ try {
+ sleep(SLEEP_MS);
+ } catch (InterruptedException e) {
+ log.error("Exception caused during node synchronization...");
+ }
+
+ if (nodeAdminService.node(node.hostname()).state() == COMPLETE) {
+ break;
+ } else {
+ nodeAdminService.updateNode(updated);
+ log.info("Failed to synchronize flow rules, retrying...");
+ }
+
+ if (waitMs <= 0) {
+ result = false;
+ break;
+ }
+ }
+
+ if (result) {
+ log.info("Successfully synchronize flow rules for node {}!", node.hostname());
+ } else {
+ log.warn("Failed to synchronize flow rules for node {}.", node.hostname());
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/web/KubevirtNetworkingWebApplication.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/web/KubevirtNetworkingWebApplication.java
index 71f3eb4..29fe7bc 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/web/KubevirtNetworkingWebApplication.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/web/KubevirtNetworkingWebApplication.java
@@ -25,6 +25,9 @@
public class KubevirtNetworkingWebApplication extends AbstractWebApplication {
@Override
public Set<Class<?>> getClasses() {
- return getClasses(KubevirtNetworkWebResource.class);
+ return getClasses(
+ KubevirtNetworkWebResource.class,
+ KubevirtManagementWebResource.class
+ );
}
}
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java
index 246d6c4..1d3512a 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/DefaultKubevirtNode.java
@@ -17,8 +17,12 @@
import com.google.common.base.MoreObjects;
import org.apache.commons.lang.StringUtils;
+import org.onlab.osgi.DefaultServiceDirectory;
import org.onlab.packet.IpAddress;
import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
import java.util.ArrayList;
import java.util.Collection;
@@ -26,6 +30,10 @@
import static com.google.common.base.Preconditions.checkArgument;
import static org.onosproject.kubevirtnode.api.Constants.DEFAULT_CLUSTER_NAME;
+import static org.onosproject.kubevirtnode.api.Constants.GENEVE;
+import static org.onosproject.kubevirtnode.api.Constants.GRE;
+import static org.onosproject.kubevirtnode.api.Constants.VXLAN;
+import static org.onosproject.net.AnnotationKeys.PORT_NAME;
/**
* Representation of a KubeVirt node.
@@ -173,6 +181,33 @@
return phyIntfs;
}
+ @Override
+ public PortNumber vxlanPort() {
+ return tunnelPort(VXLAN);
+ }
+
+ @Override
+ public PortNumber grePort() {
+ return tunnelPort(GRE);
+ }
+
+ @Override
+ public PortNumber genevePort() {
+ return tunnelPort(GENEVE);
+ }
+
+ private PortNumber tunnelPort(String tunnelType) {
+ if (dataIp == null) {
+ return null;
+ }
+ DeviceService deviceService = DefaultServiceDirectory.getService(DeviceService.class);
+ Port port = deviceService.getPorts(tunBridge).stream()
+ .filter(p -> p.isEnabled() &&
+ Objects.equals(p.annotations().value(PORT_NAME), tunnelType))
+ .findAny().orElse(null);
+ return port != null ? port.number() : null;
+ }
+
/**
* Returns new builder instance.
*
diff --git a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java
index 2cbb471..382c662 100644
--- a/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java
+++ b/apps/kubevirt-node/api/src/main/java/org/onosproject/kubevirtnode/api/KubevirtNode.java
@@ -17,6 +17,7 @@
import org.onlab.packet.IpAddress;
import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
import java.util.Collection;
@@ -135,6 +136,27 @@
Collection<KubevirtPhyInterface> phyIntfs();
/**
+ * Returns the VXLAN tunnel port.
+ *
+ * @return VXLAN port number; null if tunnel port does not exist
+ */
+ PortNumber vxlanPort();
+
+ /**
+ * Returns the GRE tunnel port.
+ *
+ * @return GRE port number; null if the GRE tunnel port does not exist
+ */
+ PortNumber grePort();
+
+ /**
+ * Returns the GENEVE tunnel port number.
+ *
+ * @return GENEVE port number; null if the GRE tunnel port does not exist
+ */
+ PortNumber genevePort();
+
+ /**
* Builder of new node entity.
*/
interface Builder {
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java
index 9031666..72d1ec0 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java
@@ -222,7 +222,9 @@
}
// create patch ports between integration to other bridges
- createPatchInterfaces(node);
+ // for now, we do not directly connect br-int with br-tun,
+ // as br-int only deals with FLAT and VLAN network
+ // createPatchInterfaces(node);
if (node.dataIp() != null && !isIntfEnabled(node, VXLAN)) {
createVxlanTunnelInterface(node);