Implement floating Ip capability in kubevirt networking app.
Change-Id: Iad9358b321dd0b46a2418c8c2d78ffd91c605e28
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
index 012fb6c..a2099e1 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
@@ -92,5 +92,6 @@
public static final int CLI_MARGIN_LENGTH = 2;
public static final int PRIORITY_STATEFUL_SNAT_RULE = 40500;
+ public static final int PRIORITY_FLOATING_IP_RULE = 40800;
public static final int PRIORITY_INTERNAL_ROUTING_RULE = 41000;
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubevirtListRouterCommand.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubevirtListRouterCommand.java
index 5c59e0e..dd08419 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubevirtListRouterCommand.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/cli/KubevirtListRouterCommand.java
@@ -61,11 +61,12 @@
print(format, "Name", "SNAT", "Internal", "External", "GatewayNode");
for (KubevirtRouter router : routers) {
- Set<String> internalCidrs = router.internal();
- Set<String> externalIps = router.external().keySet();
+ Set<String> internalNetworks = router.internal();
+ String externalNetwork = router.external().values().stream().findAny().orElse(null);
- String internal = internalCidrs.size() == 0 ? "[]" : internalCidrs.toString();
- String external = externalIps.size() == 0 ? "[]" : externalIps.toString();
+
+ String internal = internalNetworks.size() == 0 ? "[]" : internalNetworks.toString();
+ String external = externalNetwork == null ? "[]" : externalNetwork;
String gwNode = router.electedGateway() == null ? "N/A" : router.electedGateway();
print(format, StringUtils.substring(router.name(), 0,
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubevirtRouterStore.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubevirtRouterStore.java
index 93d7cc3..8f3fdbd 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubevirtRouterStore.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/DistributedKubevirtRouterStore.java
@@ -399,7 +399,7 @@
notifyDelegate(new KubevirtRouterEvent(
KUBEVIRT_FLOATING_IP_DISASSOCIATED,
router,
- event.newValue().value(), oldPodName));
+ event.oldValue().value(), oldPodName));
log.info(String.format(MSG_FLOATING_IP,
event.newValue().value().floatingIp(), MSG_DISASSOCIATED, oldPodName));
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtFloatingIpHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtFloatingIpHandler.java
new file mode 100644
index 0000000..726abf2
--- /dev/null
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtFloatingIpHandler.java
@@ -0,0 +1,434 @@
+/*
+ * Copyright 2021-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.kubevirtnetworking.impl;
+
+import org.onlab.packet.ARP;
+import org.onlab.packet.EthType;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.LeadershipService;
+import org.onosproject.cluster.NodeId;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.kubevirtnetworking.api.KubevirtFloatingIp;
+import org.onosproject.kubevirtnetworking.api.KubevirtFlowRuleService;
+import org.onosproject.kubevirtnetworking.api.KubevirtNetwork;
+import org.onosproject.kubevirtnetworking.api.KubevirtNetworkService;
+import org.onosproject.kubevirtnetworking.api.KubevirtPort;
+import org.onosproject.kubevirtnetworking.api.KubevirtPortService;
+import org.onosproject.kubevirtnetworking.api.KubevirtRouter;
+import org.onosproject.kubevirtnetworking.api.KubevirtRouterEvent;
+import org.onosproject.kubevirtnetworking.api.KubevirtRouterListener;
+import org.onosproject.kubevirtnetworking.api.KubevirtRouterService;
+import org.onosproject.kubevirtnetworking.util.RulePopulatorUtil;
+import org.onosproject.kubevirtnode.api.KubevirtNode;
+import org.onosproject.kubevirtnode.api.KubevirtNodeService;
+import org.onosproject.net.Device;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceAdminService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.FORWARDING_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRE_FLAT_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_FLOATING_IP_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_FORWARDING_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_DEFAULT_TABLE;
+import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.GENEVE;
+import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.GRE;
+import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.VXLAN;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.externalPatchPortNum;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedRouter;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterMacAddress;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.tunnelPort;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildExtension;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Handles kubevirt floating ip.
+ */
+@Component(immediate = true)
+public class KubevirtFloatingIpHandler {
+ protected final Logger log = getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected ClusterService clusterService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected LeadershipService leadershipService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DeviceAdminService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtPortService kubevirtPortService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeService kubevirtNodeService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNetworkService kubevirtNetworkService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtFlowRuleService flowService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected DriverService driverService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtRouterService kubevirtRouterService;
+
+ private final ExecutorService eventExecutor = newSingleThreadExecutor(
+ groupedThreads(this.getClass().getSimpleName(), "event-handler"));
+
+ private ApplicationId appId;
+ private NodeId localNodeId;
+
+ private final InternalRouterEventListener kubevirtRouterlistener =
+ new InternalRouterEventListener();
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
+ localNodeId = clusterService.getLocalNode().id();
+ leadershipService.runForLeadership(appId.name());
+ kubevirtRouterService.addListener(kubevirtRouterlistener);
+
+ log.info("Started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ leadershipService.withdraw(appId.name());
+ kubevirtRouterService.removeListener(kubevirtRouterlistener);
+
+ eventExecutor.shutdown();
+
+ log.info("Stopped");
+ }
+
+ private void setFloatingIpRules(KubevirtRouter router,
+ KubevirtFloatingIp floatingIp,
+ boolean install) {
+
+ KubevirtPort kubevirtPort = getKubevirtPort(floatingIp);
+ if (kubevirtPort == null) {
+ log.warn("Failed to install floating Ip rules for floating ip {} " +
+ "because there's no kubevirt port associated to it", floatingIp.floatingIp());
+ return;
+ }
+
+ KubevirtNetwork kubevirtNetwork = kubevirtNetworkService.network(kubevirtPort.networkId());
+ if (kubevirtNetwork.type() == VXLAN || kubevirtNetwork.type() == GENEVE || kubevirtNetwork.type() == GRE) {
+ setFloatingIpDownstreamRulesToGatewayTunBridge(router, floatingIp, kubevirtNetwork, kubevirtPort, install);
+ }
+
+ setFloatingIpArpResponseRules(router, floatingIp, kubevirtPort, install);
+ setFloatingIpUpstreamRules(router, floatingIp, kubevirtPort, install);
+ setFloatingIpDownstreamRules(router, floatingIp, kubevirtPort, install);
+ }
+
+ private void setFloatingIpArpResponseRules(KubevirtRouter router,
+ KubevirtFloatingIp floatingIp,
+ KubevirtPort port,
+ boolean install) {
+
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+ if (electedGw == null) {
+ log.warn("Failed to install floating Ip rules for floating ip {} " +
+ "because there's no gateway assigned to it", floatingIp.floatingIp());
+ return;
+ }
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchInPort(externalPatchPortNum(deviceService, electedGw))
+ .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(floatingIp.floatingIp().getIp4Address())
+ .build();
+
+ Device device = deviceService.getDevice(electedGw.intgBridge());
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .extension(RulePopulatorUtil.buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpShaToThaExtension(device), device.id())
+ .extension(RulePopulatorUtil.buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpOp(ARP.OP_REPLY)
+ .setEthSrc(port.macAddress())
+ .setArpSha(port.macAddress())
+ .setArpSpa(floatingIp.floatingIp().getIp4Address())
+ .setOutput(PortNumber.IN_PORT)
+ .build();
+
+ flowService.setRule(
+ appId,
+ electedGw.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_ARP_GATEWAY_RULE,
+ PRE_FLAT_TABLE,
+ install);
+ }
+ private KubevirtPort getKubevirtPort(KubevirtFloatingIp floatingIp) {
+
+ return kubevirtPortService.ports().stream()
+ .filter(port -> port.ipAddress().equals(floatingIp.fixedIp()))
+ .findAny().orElse(null);
+ }
+
+ private void setFloatingIpUpstreamRules(KubevirtRouter router,
+ KubevirtFloatingIp floatingIp,
+ KubevirtPort port,
+ boolean install) {
+
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+ if (electedGw == null) {
+ log.warn("Failed to install floating Ip rules for floating ip {} " +
+ "because there's no gateway assigned to it", floatingIp.floatingIp());
+ return;
+ }
+
+ MacAddress peerMacAddress = router.peerRouter().macAddress();
+
+ if (peerMacAddress == null) {
+ log.warn("Failed to install floating Ip rules for floating ip {} and router {}" +
+ "because there's no peer router mac address", floatingIp.floatingIp(),
+ router.name());
+ return;
+ }
+
+ MacAddress routerMacAddress = getRouterMacAddress(router);
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthSrc(port.macAddress())
+ .matchEthDst(routerMacAddress)
+ .matchIPSrc(IpPrefix.valueOf(floatingIp.fixedIp(), 32))
+ .build();
+
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setEthDst(peerMacAddress)
+ .setEthSrc(port.macAddress())
+ .setIpSrc(floatingIp.floatingIp())
+ .setOutput(externalPatchPortNum(deviceService, electedGw))
+ .build();
+
+ flowService.setRule(
+ appId,
+ electedGw.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_FLOATING_IP_RULE,
+ PRE_FLAT_TABLE,
+ install);
+ }
+
+ private void setFloatingIpDownstreamRules(KubevirtRouter router,
+ KubevirtFloatingIp floatingIp,
+ KubevirtPort port,
+ boolean install) {
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+ if (electedGw == null) {
+ log.warn("Failed to install floating Ip rules for floating ip {} " +
+ "because there's no gateway assigned to it", floatingIp.floatingIp());
+ return;
+ }
+
+ MacAddress routerMacAddress = getRouterMacAddress(router);
+
+ TrafficSelector selector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthDst(port.macAddress())
+ .matchIPDst(IpPrefix.valueOf(floatingIp.floatingIp(), 32))
+ .build();
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setEthSrc(routerMacAddress)
+ .setEthDst(port.macAddress())
+ .setIpDst(floatingIp.fixedIp())
+ .transition(FORWARDING_TABLE)
+ .build();
+
+ flowService.setRule(
+ appId,
+ electedGw.intgBridge(),
+ selector,
+ treatment,
+ PRIORITY_FLOATING_IP_RULE,
+ PRE_FLAT_TABLE,
+ install);
+ }
+
+
+ private void setFloatingIpDownstreamRulesToGatewayTunBridge(KubevirtRouter router,
+ KubevirtFloatingIp floatingIp,
+ KubevirtNetwork network,
+ KubevirtPort port,
+ boolean install) {
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+ if (electedGw == null) {
+ log.warn("Failed to install floating Ip rules for floating ip {} " +
+ "because there's no gateway assigned to it", floatingIp.floatingIp());
+ return;
+ }
+
+ KubevirtNode workerNode = kubevirtNodeService.node(port.deviceId());
+ if (workerNode == null) {
+ log.warn("Failed to install floating Ip rules for floating ip {} " +
+ "because fail to fine the worker node that the associated port is running on",
+ floatingIp.floatingIp());
+ }
+
+ PortNumber tunnelPortNumber = tunnelPort(electedGw, network);
+ if (tunnelPortNumber == null) {
+ return;
+ }
+
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(port.ipAddress(), 32))
+ .matchEthDst(port.macAddress());
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setTunnelId(Long.parseLong(network.segmentId()))
+ .extension(buildExtension(
+ deviceService,
+ electedGw.tunBridge(),
+ workerNode.dataIp().getIp4Address()),
+ electedGw.tunBridge())
+ .setOutput(tunnelPortNumber);
+
+ flowService.setRule(
+ appId,
+ electedGw.tunBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_FORWARDING_RULE,
+ TUNNEL_DEFAULT_TABLE,
+ install);
+ }
+
+ private class InternalRouterEventListener implements KubevirtRouterListener {
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+
+ @Override
+ public void event(KubevirtRouterEvent event) {
+ switch (event.type()) {
+ case KUBEVIRT_ROUTER_CREATED:
+ eventExecutor.execute(() -> processRouterCreation(event.subject()));
+ break;
+ case KUBEVIRT_ROUTER_UPDATED:
+ eventExecutor.execute(() -> processRouterUpdate(event.subject()));
+ break;
+ case KUBEVIRT_ROUTER_REMOVED:
+ eventExecutor.execute(() -> processRouterDeletion(event.subject()));
+ break;
+ case KUBEVIRT_FLOATING_IP_ASSOCIATED:
+ eventExecutor.execute(() -> processFloatingIpAssociation(event.subject(),
+ event.floatingIp()));
+ break;
+ case KUBEVIRT_FLOATING_IP_DISASSOCIATED:
+ eventExecutor.execute(() -> processFloatingIpDisassociation(event.subject(),
+ event.floatingIp()));
+ break;
+
+ default:
+ //do nothing
+ break;
+ }
+ }
+
+ private void processRouterCreation(KubevirtRouter router) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ kubevirtRouterService.floatingIpsByRouter(router.name())
+ .stream()
+ .filter(fip -> fip.fixedIp() != null)
+ .forEach(fip -> {
+ processFloatingIpAssociation(router, fip);
+ });
+ }
+
+ private void processRouterDeletion(KubevirtRouter router) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ kubevirtRouterService.floatingIpsByRouter(router.name())
+ .stream()
+ .filter(fip -> fip.fixedIp() != null)
+ .forEach(fip -> {
+ processFloatingIpDisassociation(router, fip);
+ });
+ }
+
+ private void processRouterUpdate(KubevirtRouter router) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ kubevirtRouterService.floatingIpsByRouter(router.name())
+ .stream()
+ .filter(fip -> fip.fixedIp() != null)
+ .forEach(fip -> {
+ processFloatingIpAssociation(router, fip);
+ });
+ }
+
+ private void processFloatingIpAssociation(KubevirtRouter router, KubevirtFloatingIp floatingIp) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ setFloatingIpRules(router, floatingIp, true);
+ }
+
+ private void processFloatingIpDisassociation(KubevirtRouter router, KubevirtFloatingIp floatingIp) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ setFloatingIpRules(router, floatingIp, false);
+ }
+ }
+}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
index a8867f2..f1b41bc 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
@@ -93,26 +93,33 @@
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_FORWARDING_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ICMP_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_INTERNAL_ROUTING_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_TUNNEL_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_ARP_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_DHCP_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_FORWARDING_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_ICMP_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_INBOUND_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_TO_TUNNEL_PREFIX;
+import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_DEFAULT_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedRouter;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtNetwork;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtPort;
-import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getbrIntMacAddress;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterMacAddress;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.portNumber;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.segmentIdHex;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.tunnelPort;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.tunnelToTenantPort;
import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.NXM_NX_IP_TTL;
import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.NXM_OF_ICMP_TYPE;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildExtension;
import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildLoadExtension;
import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveArpShaToThaExtension;
import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveArpSpaToTpaExtension;
import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveEthSrcToDstExtension;
import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildMoveIpSrcToDstExtension;
import static org.onosproject.kubevirtnode.api.Constants.TUNNEL_BRIDGE;
+import static org.onosproject.kubevirtnode.api.Constants.TUNNEL_TO_INTEGRATION;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.GATEWAY;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.WORKER;
import static org.slf4j.LoggerFactory.getLogger;
@@ -213,7 +220,7 @@
if (tunBridge != null) {
log.warn("The tunnel bridge {} already exists at node {}",
network.tenantBridgeName(), node.hostname());
- setDefaultRules(node, network);
+ setDefaultRulesForTenantNetwork(node, network);
return;
}
@@ -248,7 +255,7 @@
deviceService.removeDevice(network.tenantDeviceId(node.hostname()));
}
- private void createPatchInterfaceInCaseOveray(KubevirtNode node, KubevirtNetwork network) {
+ private void createPatchTenantInterface(KubevirtNode node, KubevirtNetwork network) {
Device device = deviceService.getDevice(node.ovsdb());
if (device == null || !device.is(InterfaceConfig.class)) {
@@ -298,7 +305,7 @@
ifaceConfig.removePatchMode(tunToIntIntf);
}
- private void setArpRules(KubevirtNode node, KubevirtNetwork network) {
+ private void setArpRulesForTenantNetwork(KubevirtNode node, KubevirtNetwork network) {
KubevirtRouter router = getRouterForKubevirtNetwork(kubevirtRouterService, network);
if (router == null) {
@@ -310,11 +317,11 @@
return;
}
- setGatewayArpRuleForInternalNetworkInCaseOveray(network, TENANT_ARP_TABLE, electedGw.intgBridge(),
+ setGatewayArpRuleForTenantInternalNetwork(router, network, TENANT_ARP_TABLE, electedGw.intgBridge(),
network.tenantDeviceId(node.hostname()), true);
}
- private void setIcmpRules(KubevirtNode node, KubevirtNetwork network) {
+ private void setIcmpRulesForTenantNetwork(KubevirtNode node, KubevirtNetwork network) {
KubevirtRouter router = getRouterForKubevirtNetwork(kubevirtRouterService, network);
if (router == null) {
return;
@@ -325,12 +332,25 @@
return;
}
- setGatewayIcmpRuleForInternalNetworkInCaseOveray(network, TENANT_ICMP_TABLE, electedGw.intgBridge(),
+ setGatewayIcmpRuleForTenantInternalNetwork(router, network, TENANT_ICMP_TABLE, electedGw.intgBridge(),
network.tenantDeviceId(node.hostname()), true);
}
+ private void setDefaultGatewayRuleToWorkerNodeWhenNodeCreated(KubevirtNode node, KubevirtNetwork network) {
+ KubevirtRouter router = getRouterForKubevirtNetwork(kubevirtRouterService, network);
+ if (router == null) {
+ return;
+ }
- private void setDefaultRules(KubevirtNode node, KubevirtNetwork network) {
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(nodeService, router);
+ if (electedGw == null) {
+ return;
+ }
+
+ setDefaulGatewayRuleToWorkerNodeTunBridge(router, network, electedGw.intgBridge(), node, true);
+ }
+
+ private void setDefaultRulesForTenantNetwork(KubevirtNode node, KubevirtNetwork network) {
DeviceId deviceId = network.tenantDeviceId(node.hostname());
while (!deviceService.isAvailable(deviceId)) {
@@ -348,13 +368,13 @@
flowService.connectTables(deviceId, TENANT_ARP_TABLE, TENANT_ICMP_TABLE);
flowService.connectTables(deviceId, TENANT_ICMP_TABLE, TENANT_FORWARDING_TABLE);
- setDhcpRule(deviceId, true);
+ setDhcpRuleForTenantNetwork(deviceId, true);
setForwardingRule(deviceId, true);
log.info("Install default flow rules for tenant bridge {}", network.tenantBridgeName());
}
- private void setDhcpRule(DeviceId deviceId, boolean install) {
+ private void setDhcpRuleForTenantNetwork(DeviceId deviceId, boolean install) {
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
.matchIPProtocol(IPv4.PROTOCOL_UDP)
@@ -400,39 +420,136 @@
case VXLAN:
case GRE:
case GENEVE:
-
+ setDefaultEgressRuleToGatewayNode(router, network, electedGateway.intgBridge(), install);
kubevirtNodeService.completeNodes(WORKER).forEach(node -> {
- setGatewayArpRuleForInternalNetworkInCaseOveray(network, TENANT_ARP_TABLE,
+ setGatewayArpRuleForTenantInternalNetwork(router, network, TENANT_ARP_TABLE,
electedGateway.intgBridge(),
network.tenantDeviceId(node.hostname()), install);
- setGatewayIcmpRuleForInternalNetworkInCaseOveray(network, TENANT_ICMP_TABLE,
+ setGatewayIcmpRuleForTenantInternalNetwork(router, network, TENANT_ICMP_TABLE,
electedGateway.intgBridge(),
network.tenantDeviceId(node.hostname()), install);
+ setDefaulGatewayRuleToWorkerNodeTunBridge(router, network,
+ electedGateway.intgBridge(), node, install);
});
break;
case FLAT:
case VLAN:
- setGatewayArpRuleForInternalNetworkInCaseVlanFlat(network, PRE_FLAT_TABLE,
+ setGatewayArpRuleForProviderInternalNetwork(router, network, PRE_FLAT_TABLE,
electedGateway.intgBridge(), install);
- setGatewayIcmpRuleForInternalNetworkInCaseVlanFlat(network, PRE_FLAT_TABLE,
+ setGatewayIcmpRuleForProviderInternalNetwork(router, network, PRE_FLAT_TABLE,
electedGateway.intgBridge(), install);
- setGatewayInterNetworkRoutingWithinSameRouter(network, router, electedGateway, install);
+ setGatewayProviderInterNetworkRoutingWithinSameRouter(network, router, electedGateway, install);
break;
default:
// do nothing
break;
}
-
}
- private void setGatewayIcmpRuleForInternalNetworkInCaseOveray(KubevirtNetwork network,
- int tableNum,
- DeviceId gwDeviceId,
- DeviceId tenantDeviceId,
- boolean install) {
- MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gwDeviceId);
+ private void setDefaulGatewayRuleToWorkerNodeTunBridge(KubevirtRouter router,
+ KubevirtNetwork network,
+ DeviceId gwDeviceId,
+ KubevirtNode workerNode,
+ boolean install) {
+ MacAddress routerMacAddress = getRouterMacAddress(router);
- if (brIntMacAddress == null) {
+ if (routerMacAddress == null) {
+ log.warn("Setting gateway default eggress rule to gateway for tenant internal network because " +
+ "there's no br-int port for device {}", gwDeviceId);
+ return;
+ }
+
+ KubevirtNode gwNode = kubevirtNodeService.node(gwDeviceId);
+
+ if (gwNode == null) {
+ log.warn("Setting gateway default eggress rule to gateway for tenant internal network because " +
+ "there's no gateway node for device {}", gwDeviceId);
+ return;
+ }
+
+
+ PortNumber patchPortNumber = tunnelToTenantPort(workerNode, network);
+ if (patchPortNumber == null) {
+ return;
+ }
+
+ PortNumber tunnelPortNumber = tunnelPort(workerNode, network);
+ if (tunnelPortNumber == null) {
+ return;
+ }
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchInPort(patchPortNumber)
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthDst((routerMacAddress));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setTunnelId(Long.parseLong(network.segmentId()))
+ .extension(buildExtension(
+ deviceService,
+ workerNode.tunBridge(),
+ gwNode.dataIp().getIp4Address()),
+ workerNode.tunBridge())
+ .setOutput(tunnelPortNumber);
+
+ flowService.setRule(
+ appId,
+ workerNode.tunBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_FORWARDING_RULE,
+ TUNNEL_DEFAULT_TABLE,
+ install);
+ }
+
+ private void setDefaultEgressRuleToGatewayNode(KubevirtRouter router,
+ KubevirtNetwork network,
+ DeviceId gwDeviceId,
+ boolean install) {
+ MacAddress routerMacAddress = getRouterMacAddress(router);
+
+ if (routerMacAddress == null) {
+ log.warn("Setting gateway default eggress rule to gateway for tenant internal network because " +
+ "there's no br-int port for device {}", gwDeviceId);
+ return;
+ }
+
+ KubevirtNode gwNode = kubevirtNodeService.node(gwDeviceId);
+
+ if (gwNode == null) {
+ log.warn("Setting gateway default eggress rule to gateway for tenant internal network because " +
+ "there's no gateway node for device {}", gwDeviceId);
+ return;
+ }
+
+ PortNumber tunToIntPortNum = portNumber(gwNode.tunBridge(), TUNNEL_TO_INTEGRATION);
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchTunnelId(Long.parseLong(network.segmentId()));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setOutput(tunToIntPortNum);
+
+ flowService.setRule(
+ appId,
+ gwNode.tunBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_TUNNEL_RULE,
+ TUNNEL_DEFAULT_TABLE,
+ install);
+ }
+
+
+ private void setGatewayIcmpRuleForTenantInternalNetwork(KubevirtRouter router,
+ KubevirtNetwork network,
+ int tableNum,
+ DeviceId gwDeviceId,
+ DeviceId tenantDeviceId,
+ boolean install) {
+ MacAddress routerMacAddress = getRouterMacAddress(router);
+
+ if (routerMacAddress == null) {
log.warn("Setting gateway ICMP rule for internal network because " +
"there's no br-int port for device {}", gwDeviceId);
return;
@@ -463,7 +580,7 @@
.extension(buildLoadExtension(device,
NXM_OF_ICMP_TYPE, TYPE_ECHO_REPLY), device.id())
.setIpSrc(network.gatewayIp())
- .setEthSrc(brIntMacAddress)
+ .setEthSrc(routerMacAddress)
.setOutput(PortNumber.IN_PORT);
flowService.setRule(
@@ -476,15 +593,16 @@
install);
}
- private void setGatewayArpRuleForInternalNetworkInCaseOveray(KubevirtNetwork network,
- int tableNum,
- DeviceId gwDeviceId,
- DeviceId tenantDeviceId,
- boolean install) {
+ private void setGatewayArpRuleForTenantInternalNetwork(KubevirtRouter router,
+ KubevirtNetwork network,
+ int tableNum,
+ DeviceId gwDeviceId,
+ DeviceId tenantDeviceId,
+ boolean install) {
- MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gwDeviceId);
+ MacAddress routerMacAddress = getRouterMacAddress(router);
- if (brIntMacAddress == null) {
+ if (routerMacAddress == null) {
log.warn("Setting gateway arp rule for internal network because " +
"there's no br-int port for device {}", gwDeviceId);
return;
@@ -510,9 +628,9 @@
.extension(buildMoveArpShaToThaExtension(device), device.id())
.extension(buildMoveArpSpaToTpaExtension(device), device.id())
.setArpOp(ARP.OP_REPLY)
- .setArpSha(brIntMacAddress)
+ .setArpSha(routerMacAddress)
.setArpSpa(Ip4Address.valueOf(network.gatewayIp().toString()))
- .setEthSrc(brIntMacAddress)
+ .setEthSrc(routerMacAddress)
.setOutput(PortNumber.IN_PORT);
flowService.setRule(
@@ -526,7 +644,7 @@
);
}
- private void setGatewayInterNetworkRoutingWithinSameRouter(
+ private void setGatewayProviderInterNetworkRoutingWithinSameRouter(
KubevirtNetwork network, KubevirtRouter router, KubevirtNode gatewayNode, boolean install) {
router.internal().forEach(srcNetwork -> {
if (srcNetwork.equals(network.networkId())
@@ -535,14 +653,17 @@
}
kubevirtPortService.ports(network.networkId()).forEach(port -> {
- setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+ setGatewayInterNetworkRoutingFromNetworkToPort(router, kubevirtNetworkService.network(srcNetwork),
port, gatewayNode, install);
});
});
}
- private void setGatewayInterNetworkRoutingFromNetworkToPort(KubevirtNetwork srcNetwork, KubevirtPort dstPort,
- KubevirtNode gatewayNode, boolean install) {
+ private void setGatewayInterNetworkRoutingFromNetworkToPort(KubevirtRouter router,
+ KubevirtNetwork srcNetwork,
+ KubevirtPort dstPort,
+ KubevirtNode gatewayNode,
+ boolean install) {
Device gwDevice = deviceService.getDevice(gatewayNode.intgBridge());
@@ -552,9 +673,9 @@
return;
}
- MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gatewayNode.intgBridge());
+ MacAddress routerMacAddress = getRouterMacAddress(router);
- if (brIntMacAddress == null) {
+ if (routerMacAddress == null) {
log.warn("Failed to set internal network routing rule because " +
"there's no br-int port for device {}", gatewayNode.intgBridge());
return;
@@ -562,12 +683,12 @@
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
- .matchEthDst(brIntMacAddress)
+ .matchEthDst(routerMacAddress)
.matchIPSrc(IpPrefix.valueOf(srcNetwork.cidr()))
.matchIPDst(IpPrefix.valueOf(dstPort.ipAddress(), 32));
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
- .setEthSrc(brIntMacAddress)
+ .setEthSrc(routerMacAddress)
.setEthDst(dstPort.macAddress())
.transition(FORWARDING_TABLE)
.build();
@@ -583,14 +704,14 @@
);
}
- private void setGatewayArpRuleForInternalNetworkInCaseVlanFlat(KubevirtNetwork network,
- int tableNum, DeviceId gwDeviceId, boolean install) {
+ private void setGatewayArpRuleForProviderInternalNetwork(KubevirtRouter router, KubevirtNetwork network,
+ int tableNum, DeviceId gwDeviceId, boolean install) {
Device device = deviceService.getDevice(gwDeviceId);
- MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gwDeviceId);
+ MacAddress routerMacAddress = getRouterMacAddress(router);
- if (brIntMacAddress == null) {
+ if (routerMacAddress == null) {
log.warn("Setting gateway arp rule for internal network because " +
"there's no br-int port for device {}", gwDeviceId);
return;
@@ -606,9 +727,9 @@
.extension(buildMoveArpShaToThaExtension(device), device.id())
.extension(buildMoveArpSpaToTpaExtension(device), device.id())
.setArpOp(ARP.OP_REPLY)
- .setArpSha(brIntMacAddress)
+ .setArpSha(routerMacAddress)
.setArpSpa(Ip4Address.valueOf(network.gatewayIp().toString()))
- .setEthSrc(brIntMacAddress)
+ .setEthSrc(routerMacAddress)
.setOutput(PortNumber.IN_PORT);
flowService.setRule(
@@ -625,16 +746,17 @@
/**
* Sends ICMP echo reply for the ICMP echo request from the kubevirt VM.
*
+ * @param router kubevirt router
* @param network kubevirt network
* @param tableNum flow table number
* @param deviceId device id of the selected gateway for the network
* @param install install if true, remove otherwise
*/
- private void setGatewayIcmpRuleForInternalNetworkInCaseVlanFlat(KubevirtNetwork network,
- int tableNum, DeviceId deviceId, boolean install) {
- MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, deviceId);
+ private void setGatewayIcmpRuleForProviderInternalNetwork(KubevirtRouter router, KubevirtNetwork network,
+ int tableNum, DeviceId deviceId, boolean install) {
+ MacAddress routerMacAddress = getRouterMacAddress(router);
- if (brIntMacAddress == null) {
+ if (routerMacAddress == null) {
log.error("Setting gateway ICMP rule for internal network because " +
"there's no br-int port for device {}", deviceId);
return;
@@ -656,7 +778,7 @@
.extension(buildLoadExtension(device,
NXM_OF_ICMP_TYPE, TYPE_ECHO_REPLY), device.id())
.setIpSrc(network.gatewayIp())
- .setEthSrc(brIntMacAddress)
+ .setEthSrc(routerMacAddress)
.setOutput(PortNumber.IN_PORT);
flowService.setRule(
@@ -815,11 +937,12 @@
private void removeDetachedInternalNetworkRules(KubevirtNetwork removedNetwork, KubevirtRouter router,
KubevirtNode electedGw) {
- router.internal().forEach(networkName -> {
- kubevirtPortService.ports(networkName).forEach(kubevirtPort -> {
- setGatewayInterNetworkRoutingFromNetworkToPort(
- removedNetwork, kubevirtPort, electedGw, false);
- });
+ router.internal().stream().filter(networkId -> kubevirtNetworkService.network(networkId) != null)
+ .forEach(networkId -> {
+ kubevirtPortService.ports(networkId).forEach(kubevirtPort -> {
+ setGatewayInterNetworkRoutingFromNetworkToPort(
+ router, removedNetwork, kubevirtPort, electedGw, false);
+ });
});
}
@@ -966,8 +1089,8 @@
nodeService.completeNodes().forEach(n -> {
createBridge(n, network);
- createPatchInterfaceInCaseOveray(n, network);
- setDefaultRules(n, network);
+ createPatchTenantInterface(n, network);
+ setDefaultRulesForTenantNetwork(n, network);
});
}
@@ -1021,11 +1144,11 @@
continue;
}
createBridge(node, network);
- createPatchInterfaceInCaseOveray(node, network);
- setDefaultRules(node, network);
- setArpRules(node, network);
- setIcmpRules(node, network);
-
+ createPatchTenantInterface(node, network);
+ setDefaultRulesForTenantNetwork(node, network);
+ setArpRulesForTenantNetwork(node, network);
+ setIcmpRulesForTenantNetwork(node, network);
+ setDefaultGatewayRuleToWorkerNodeWhenNodeCreated(node, network);
break;
case FLAT:
case VLAN:
@@ -1130,7 +1253,7 @@
|| kubevirtNetworkService.network(srcNetwork) == null) {
return;
}
- setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+ setGatewayInterNetworkRoutingFromNetworkToPort(router, kubevirtNetworkService.network(srcNetwork),
kubevirtPort, gwNode, true);
});
}
@@ -1155,7 +1278,7 @@
|| kubevirtNetworkService.network(srcNetwork) == null) {
return;
}
- setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+ setGatewayInterNetworkRoutingFromNetworkToPort(router, kubevirtNetworkService.network(srcNetwork),
kubevirtPort, gwNode, true);
});
}
@@ -1180,7 +1303,7 @@
|| kubevirtNetworkService.network(srcNetwork) == null) {
return;
}
- setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+ setGatewayInterNetworkRoutingFromNetworkToPort(router, kubevirtNetworkService.network(srcNetwork),
kubevirtPort, gwNode, false);
});
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtPodPortMapper.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtPodPortMapper.java
index 9675ca2..8529b5c 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtPodPortMapper.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtPodPortMapper.java
@@ -34,6 +34,7 @@
import org.onosproject.kubevirtnetworking.api.KubevirtPort;
import org.onosproject.kubevirtnetworking.api.KubevirtPortAdminService;
import org.onosproject.kubevirtnode.api.KubevirtApiConfigService;
+import org.onosproject.kubevirtnode.api.KubevirtNodeService;
import org.onosproject.mastership.MastershipService;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.driver.DriverService;
@@ -99,6 +100,9 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected KubevirtApiConfigService kubevirtApiConfigService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeService kubevirtNodeService;
+
private final ExecutorService eventExecutor = newSingleThreadExecutor(
groupedThreads(this.getClass().getSimpleName(), "event-handler"));
@@ -186,7 +190,7 @@
log.error("Failed to reserve IP address", e);
}
- Set<KubevirtPort> ports = getPorts(kubevirtNetworkAdminService.networks(), pod);
+ Set<KubevirtPort> ports = getPorts(kubevirtNodeService, kubevirtNetworkAdminService.networks(), pod);
if (ports.size() == 0) {
return;
}
@@ -203,7 +207,7 @@
return;
}
- Set<KubevirtPort> ports = getPorts(kubevirtNetworkAdminService.networks(), pod);
+ Set<KubevirtPort> ports = getPorts(kubevirtNodeService, kubevirtNetworkAdminService.networks(), pod);
if (ports.size() == 0) {
return;
}
@@ -261,7 +265,7 @@
return;
}
- Set<KubevirtPort> ports = getPorts(kubevirtNetworkAdminService.networks(), pod);
+ Set<KubevirtPort> ports = getPorts(kubevirtNodeService, kubevirtNetworkAdminService.networks(), pod);
if (ports.size() == 0) {
return;
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtRoutingSnatHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtRoutingSnatHandler.java
index 30cca24..264017a 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtRoutingSnatHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtRoutingSnatHandler.java
@@ -23,6 +23,7 @@
import org.onlab.packet.IpPrefix;
import org.onlab.packet.MacAddress;
import org.onlab.packet.TpPort;
+import org.onlab.packet.VlanId;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
import org.onosproject.cluster.NodeId;
@@ -30,8 +31,6 @@
import org.onosproject.core.CoreService;
import org.onosproject.kubevirtnetworking.api.KubevirtFlowRuleService;
import org.onosproject.kubevirtnetworking.api.KubevirtNetwork;
-import org.onosproject.kubevirtnetworking.api.KubevirtNetworkEvent;
-import org.onosproject.kubevirtnetworking.api.KubevirtNetworkListener;
import org.onosproject.kubevirtnetworking.api.KubevirtNetworkService;
import org.onosproject.kubevirtnetworking.api.KubevirtPort;
import org.onosproject.kubevirtnetworking.api.KubevirtPortEvent;
@@ -45,7 +44,6 @@
import org.onosproject.kubevirtnode.api.KubevirtNode;
import org.onosproject.kubevirtnode.api.KubevirtNodeService;
import org.onosproject.net.Device;
-import org.onosproject.net.Port;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceAdminService;
import org.onosproject.net.driver.DriverService;
@@ -73,13 +71,22 @@
import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
import static org.onosproject.kubevirtnetworking.api.Constants.PRE_FLAT_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_FORWARDING_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_STATEFUL_SNAT_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_DEFAULT_TABLE;
+import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.GENEVE;
+import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.GRE;
+import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.VLAN;
+import static org.onosproject.kubevirtnetworking.api.KubevirtNetwork.Type.VXLAN;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.externalPatchPortNum;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedRouter;
-import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterSnatIpAddress;
-import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getbrIntMacAddress;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getExternalNetworkByRouter;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtPort;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterSnatIpAddress;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterMacAddress;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.tunnelPort;
import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.CT_NAT_SRC_FLAG;
-import static org.onosproject.net.AnnotationKeys.PORT_NAME;
+import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.buildExtension;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -133,9 +140,6 @@
private final InternalRouterEventListener kubevirtRouterlistener =
new InternalRouterEventListener();
- private final InternalNetworkEventListener kubevirtNetworkEventListener =
- new InternalNetworkEventListener();
-
private ApplicationId appId;
private NodeId localNodeId;
@@ -147,7 +151,6 @@
kubevirtPortService.addListener(kubevirtPortListener);
kubevirtRouterService.addListener(kubevirtRouterlistener);
- kubevirtNetworkService.addListener(kubevirtNetworkEventListener);
log.info("Started");
}
@@ -157,7 +160,6 @@
leadershipService.withdraw(appId.name());
kubevirtPortService.removeListener(kubevirtPortListener);
kubevirtRouterService.removeListener(kubevirtRouterlistener);
- kubevirtNetworkService.removeListener(kubevirtNetworkEventListener);
eventExecutor.shutdown();
@@ -183,13 +185,13 @@
setArpResponseToPeerRouter(electedGw, Ip4Address.valueOf(routerSnatIp), install);
setStatefulSnatUpstreamRules(electedGw, router, Ip4Address.valueOf(routerSnatIp), install);
- setStatefulSnatDownstreamRuleForRouter(router, electedGw, Ip4Address.valueOf(routerSnatIp), install);
+ setStatefulSnatDownstreamRuleForRouter(electedGw, router, Ip4Address.valueOf(routerSnatIp), install);
}
private void setArpResponseToPeerRouter(KubevirtNode gatewayNode, Ip4Address ip4Address, boolean install) {
TrafficSelector selector = DefaultTrafficSelector.builder()
- .matchInPort(externalPatchPortNum(gatewayNode))
+ .matchInPort(externalPatchPortNum(deviceService, gatewayNode))
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.matchArpOp(ARP.OP_REQUEST)
.matchArpTpa(ip4Address)
@@ -218,15 +220,22 @@
install);
}
- private void setStatefulSnatUpstreamRules(KubevirtNode gatewayNode, KubevirtRouter router,
- Ip4Address ip4Address, boolean install) {
+ private void setStatefulSnatUpstreamRules(KubevirtNode gatewayNode,
+ KubevirtRouter router,
+ Ip4Address ip4Address,
+ boolean install) {
+ MacAddress routerMacAddress = getRouterMacAddress(router);
+ if (routerMacAddress == null) {
+ return;
+ }
+ MacAddress peerRouterMacAddres = router.peerRouter().macAddress();
+ if (peerRouterMacAddres == null) {
+ return;
+ }
- MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gatewayNode.intgBridge());
-
- TrafficSelector selector = DefaultTrafficSelector.builder()
+ TrafficSelector.Builder selector = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
- .matchEthDst(brIntMacAddress)
- .build();
+ .matchEthDst(routerMacAddress);
TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
@@ -241,41 +250,27 @@
.build();
tBuilder.extension(natTreatment, gatewayNode.intgBridge())
- .setEthDst(router.peerRouter().macAddress())
+ .setEthDst(peerRouterMacAddres)
.setEthSrc(DEFAULT_GATEWAY_MAC)
- .setOutput(externalPatchPortNum(gatewayNode));
+ .setOutput(externalPatchPortNum(deviceService, gatewayNode));
flowService.setRule(
appId,
gatewayNode.intgBridge(),
- selector,
+ selector.build(),
tBuilder.build(),
PRIORITY_STATEFUL_SNAT_RULE,
- FLAT_TABLE,
+ PRE_FLAT_TABLE,
install);
}
- private void setStatefulSnatDownStreamRuleForNetwork(KubevirtNode gatewayNode,
- KubevirtRouter router,
- KubevirtNetwork network,
- boolean install) {
- kubevirtPortService.ports(network.networkId()).forEach(kubevirtPort -> {
- String routerSnatIp = router.external().keySet().stream().findAny().orElse(null);
- if (routerSnatIp == null) {
- return;
- }
- setStatefulSnatDownStreamRuleForKubevirtPort(gatewayNode, IpAddress.valueOf(routerSnatIp),
- kubevirtPort, install);
- });
- }
+ private void setStatefulSnatDownStreamRuleForKubevirtPort(KubevirtRouter router,
+ KubevirtNode gatewayNode,
+ KubevirtPort kubevirtPort,
+ boolean install) {
+ MacAddress routerMacAddress = getRouterMacAddress(router);
- private void setStatefulSnatDownStreamRuleForKubevirtPort(KubevirtNode gatewayNode,
- IpAddress gatewaySnatIp,
- KubevirtPort kubevirtPort,
- boolean install) {
- MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gatewayNode.intgBridge());
-
- if (brIntMacAddress == null) {
+ if (routerMacAddress == null) {
log.error("Failed to set stateful snat downstream rule because " +
"there's no br-int port for device {}", gatewayNode.intgBridge());
return;
@@ -283,39 +278,105 @@
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthSrc(routerMacAddress)
.matchIPDst(IpPrefix.valueOf(kubevirtPort.ipAddress(), 32));
- TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ KubevirtNetwork network = kubevirtNetworkService.network(kubevirtPort.networkId());
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
.setEthDst(kubevirtPort.macAddress())
- .transition(FORWARDING_TABLE)
- .build();
+ .transition(FORWARDING_TABLE);
flowService.setRule(
appId,
gatewayNode.intgBridge(),
sBuilder.build(),
- treatment,
+ tBuilder.build(),
PRIORITY_STATEFUL_SNAT_RULE,
FLAT_TABLE,
install);
+
+ if (network.type() == VXLAN || network.type() == GENEVE || network.type() == GRE) {
+ setDownStreamRulesToGatewayTunBridge(router, network, kubevirtPort, install);
+ }
}
- private void setStatefulSnatDownstreamRuleForRouter(KubevirtRouter router,
- KubevirtNode gatewayNode,
- IpAddress gatewaySnatIp,
- boolean install) {
+ private void setDownStreamRulesToGatewayTunBridge(KubevirtRouter router,
+ KubevirtNetwork network,
+ KubevirtPort port, boolean install) {
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
- MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gatewayNode.intgBridge());
+ if (electedGw == null) {
+ return;
+ }
- if (brIntMacAddress == null) {
- log.error("Failed to set stateful snat downstream rule because " +
- "there's no br-int port for device {}", gatewayNode.intgBridge());
+ KubevirtNode workerNode = kubevirtNodeService.node(port.deviceId());
+ if (workerNode == null) {
+ return;
+ }
+
+ PortNumber tunnelPortNumber = tunnelPort(electedGw, network);
+ if (tunnelPortNumber == null) {
return;
}
TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
.matchEthType(Ethernet.TYPE_IPV4)
- .matchIPDst(IpPrefix.valueOf(gatewaySnatIp, 32));
+ .matchIPDst(IpPrefix.valueOf(port.ipAddress(), 32))
+ .matchEthDst(port.macAddress());
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .setTunnelId(Long.parseLong(network.segmentId()))
+ .extension(buildExtension(
+ deviceService,
+ electedGw.tunBridge(),
+ workerNode.dataIp().getIp4Address()),
+ electedGw.tunBridge())
+ .setOutput(tunnelPortNumber);
+
+ flowService.setRule(
+ appId,
+ electedGw.tunBridge(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_FORWARDING_RULE,
+ TUNNEL_DEFAULT_TABLE,
+ install);
+ }
+
+ private void setStatefulSnatDownstreamRuleForRouter(KubevirtNode gatewayNode,
+ KubevirtRouter router,
+ IpAddress gatewaySnatIp,
+ boolean install) {
+
+ MacAddress routerMacAddress = getRouterMacAddress(router);
+
+ if (routerMacAddress == null) {
+ log.warn("Failed to set stateful snat downstream rule because " +
+ "there's no br-int port for device {}", gatewayNode.intgBridge());
+ return;
+ }
+
+ KubevirtNetwork externalNetwork = getExternalNetworkByRouter(kubevirtNetworkService, router);
+
+ if (externalNetwork == null) {
+ log.warn("Failed to set stateful snat downstream rule because " +
+ "there's no external network router {}", router.name());
+ return;
+ }
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ if (externalNetwork.type() == VLAN) {
+ sBuilder.matchEthType(Ethernet.TYPE_VLAN)
+ .matchVlanId(VlanId.vlanId(externalNetwork.segmentId()));
+ tBuilder.popVlan();
+ } else {
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4);
+ }
+
+ sBuilder.matchIPDst(IpPrefix.valueOf(gatewaySnatIp, 32));
ExtensionTreatment natTreatment = RulePopulatorUtil
.niciraConnTrackTreatmentBuilder(driverService, gatewayNode.intgBridge())
@@ -324,36 +385,17 @@
.table((short) FLAT_TABLE)
.build();
- TrafficTreatment treatment = DefaultTrafficTreatment.builder()
- .setEthSrc(brIntMacAddress)
- .extension(natTreatment, gatewayNode.intgBridge())
- .build();
+ tBuilder.setEthSrc(routerMacAddress)
+ .extension(natTreatment, gatewayNode.intgBridge());
flowService.setRule(
appId,
gatewayNode.intgBridge(),
sBuilder.build(),
- treatment,
+ tBuilder.build(),
PRIORITY_STATEFUL_SNAT_RULE,
- FLAT_TABLE,
+ PRE_FLAT_TABLE,
install);
-
- router.internal().forEach(networkName -> {
- KubevirtNetwork network = kubevirtNetworkService.network(networkName);
-
- if (network != null) {
- setStatefulSnatDownStreamRuleForNetwork(gatewayNode, router, network, install);
- }
- });
- }
-
- private PortNumber externalPatchPortNum(KubevirtNode gatewayNode) {
- Port port = deviceService.getPorts(gatewayNode.intgBridge()).stream()
- .filter(p -> p.isEnabled() &&
- Objects.equals(p.annotations().value(PORT_NAME), "int-to-gateway"))
- .findAny().orElse(null);
-
- return port != null ? port.number() : null;
}
private class InternalRouterEventListener implements KubevirtRouterListener {
@@ -381,11 +423,150 @@
eventExecutor.execute(() -> processRouterInternalNetworksDetached(event.subject(),
event.internal()));
break;
+ case KUBEVIRT_GATEWAY_NODE_ATTACHED:
+ eventExecutor.execute(() -> processRouterGatewayNodeAttached(event.subject(),
+ event.gateway()));
+ break;
+ case KUBEVIRT_GATEWAY_NODE_DETACHED:
+ eventExecutor.execute(() -> processRouterGatewayNodeDetached(event.subject(),
+ event.gateway()));
+ break;
+ case KUBEVIRT_ROUTER_EXTERNAL_NETWORK_ATTACHED:
+ eventExecutor.execute(() -> processRouterExternalNetAttached(event.subject(), event.externalIp()));
+ break;
+ case KUBEVIRT_ROUTER_EXTERNAL_NETWORK_DETACHED:
+ eventExecutor.execute(() -> processRouterExternalNetDetached(event.subject(), event.externalIp()));
+ break;
default:
//do nothing
break;
}
}
+
+ private void processRouterExternalNetAttached(KubevirtRouter router, String externalIp) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+ if (electedGw == null) {
+ log.warn("Fail to process router external network attached gateway node snat for router {} " +
+ "there's no gateway assigned to it", router.name());
+ return;
+ }
+
+ if (router.enableSnat() && router.peerRouter() != null && externalIp != null) {
+ setArpResponseToPeerRouter(electedGw, Ip4Address.valueOf(externalIp), true);
+ setStatefulSnatUpstreamRules(electedGw, router, Ip4Address.valueOf(externalIp), true);
+ setStatefulSnatDownstreamRuleForRouter(electedGw, router, Ip4Address.valueOf(externalIp), true);
+ }
+
+ router.internal()
+ .stream()
+ .filter(networkId -> kubevirtNetworkService.network(networkId) != null)
+ .map(kubevirtNetworkService::network)
+ .forEach(network -> {
+ String routerSnatIp = router.external().keySet().stream().findAny().orElse(null);
+ if (routerSnatIp == null) {
+ return;
+ }
+ kubevirtPortService.ports(network.networkId()).forEach(kubevirtPort -> {
+ setStatefulSnatDownStreamRuleForKubevirtPort(router,
+ electedGw, kubevirtPort, true);
+ });
+ });
+ }
+
+ private void processRouterExternalNetDetached(KubevirtRouter router, String externalIp) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+ if (!isRelevantHelper()) {
+ return;
+ }
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+ if (electedGw == null) {
+ log.warn("Fail to process router external network attached gateway node snat for router {} " +
+ "there's no gateway assigned to it", router.name());
+ return;
+ }
+
+ if (router.enableSnat() && router.peerRouter() != null && externalIp != null) {
+ setArpResponseToPeerRouter(electedGw, Ip4Address.valueOf(externalIp), false);
+ setStatefulSnatUpstreamRules(electedGw, router, Ip4Address.valueOf(externalIp), false);
+ setStatefulSnatDownstreamRuleForRouter(electedGw, router, Ip4Address.valueOf(externalIp), false);
+ }
+
+ router.internal()
+ .stream()
+ .filter(networkId -> kubevirtNetworkService.network(networkId) != null)
+ .map(kubevirtNetworkService::network)
+ .forEach(network -> {
+ String routerSnatIp = router.external().keySet().stream().findAny().orElse(null);
+ if (routerSnatIp == null) {
+ return;
+ }
+ kubevirtPortService.ports(network.networkId()).forEach(kubevirtPort -> {
+ setStatefulSnatDownStreamRuleForKubevirtPort(router,
+ electedGw, kubevirtPort, false);
+ });
+ });
+ }
+
+ private void processRouterGatewayNodeAttached(KubevirtRouter router, String attachedGatewayId) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ KubevirtNode attachedGateway = kubevirtNodeService.node(attachedGatewayId);
+ if (attachedGateway == null) {
+ return;
+ }
+
+ router.internal()
+ .stream()
+ .filter(networkId -> kubevirtNetworkService.network(networkId) != null)
+ .map(kubevirtNetworkService::network)
+ .forEach(network -> {
+ String routerSnatIp = router.external().keySet().stream().findAny().orElse(null);
+ if (routerSnatIp == null) {
+ return;
+ }
+ kubevirtPortService.ports(network.networkId()).forEach(kubevirtPort -> {
+ setStatefulSnatDownStreamRuleForKubevirtPort(router,
+ attachedGateway, kubevirtPort, true);
+ });
+ });
+ }
+
+ private void processRouterGatewayNodeDetached(KubevirtRouter router, String detachedGatewayId) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ KubevirtNode detachedGateway = kubevirtNodeService.node(detachedGatewayId);
+ if (detachedGateway == null) {
+ return;
+ }
+
+ router.internal()
+ .stream()
+ .filter(networkId -> kubevirtNetworkService.network(networkId) != null)
+ .map(kubevirtNetworkService::network)
+ .forEach(network -> {
+ String routerSnatIp = router.external().keySet().stream().findAny().orElse(null);
+ if (routerSnatIp == null) {
+ return;
+ }
+
+ kubevirtPortService.ports(network.networkId()).forEach(kubevirtPort -> {
+ setStatefulSnatDownStreamRuleForKubevirtPort(router,
+ detachedGateway, kubevirtPort, false);
+ });
+ });
+ }
+
private void processRouterInternalNetworksAttached(KubevirtRouter router,
Set<String> attachedInternalNetworks) {
if (!isRelevantHelper()) {
@@ -398,13 +579,13 @@
}
attachedInternalNetworks.forEach(networkId -> {
+ String routerSnatIp = router.external().keySet().stream().findAny().orElse(null);
+ if (routerSnatIp == null) {
+ return;
+ }
+
kubevirtPortService.ports(networkId).forEach(kubevirtPort -> {
- String routerSnatIp = router.external().keySet().stream().findAny().orElse(null);
- if (routerSnatIp == null) {
- return;
- }
- setStatefulSnatDownStreamRuleForKubevirtPort(gwNode, IpAddress.valueOf(routerSnatIp),
- kubevirtPort, true);
+ setStatefulSnatDownStreamRuleForKubevirtPort(router, gwNode, kubevirtPort, true);
});
});
}
@@ -421,14 +602,14 @@
}
detachedInternalNetworks.forEach(networkId -> {
+ String routerSnatIp = router.external().keySet().stream().findAny().orElse(null);
+ if (routerSnatIp == null) {
+ log.info("snatIp is null");
+ return;
+ }
+
kubevirtPortService.ports(networkId).forEach(kubevirtPort -> {
- String routerSnatIp = router.external().keySet().stream().findAny().orElse(null);
- if (routerSnatIp == null) {
- log.info("snatIp is null");
- return;
- }
- setStatefulSnatDownStreamRuleForKubevirtPort(gwNode, IpAddress.valueOf(routerSnatIp),
- kubevirtPort, false);
+ setStatefulSnatDownStreamRuleForKubevirtPort(router, gwNode, kubevirtPort, false);
});
});
}
@@ -460,67 +641,6 @@
}
}
- private class InternalNetworkEventListener implements KubevirtNetworkListener {
-
- private boolean isRelevantHelper() {
- return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
- }
-
- @Override
- public void event(KubevirtNetworkEvent event) {
- switch (event.type()) {
- case KUBEVIRT_NETWORK_CREATED:
- eventExecutor.execute(() -> processNetworkCreation(event.subject()));
- break;
- case KUBEVIRT_NETWORK_REMOVED:
- eventExecutor.execute(() -> processNetworkRemoval(event.subject()));
- break;
- case KUBEVIRT_NETWORK_UPDATED:
- default:
- // do nothing
- break;
- }
- }
-
- private void processNetworkCreation(KubevirtNetwork network) {
- if (!isRelevantHelper()) {
- return;
- }
-
- switch (network.type()) {
- case VXLAN:
- case GRE:
- case GENEVE:
- break;
- case FLAT:
- case VLAN:
- break;
- default:
- // do nothing
- break;
- }
- }
-
- private void processNetworkRemoval(KubevirtNetwork network) {
- if (!isRelevantHelper()) {
- return;
- }
-
- switch (network.type()) {
- case VXLAN:
- case GRE:
- case GENEVE:
- break;
- case FLAT:
- case VLAN:
- break;
- default:
- // do nothing
- break;
- }
- }
- }
-
private class InternalKubevirtPortListener implements KubevirtPortListener {
private boolean isRelevantHelper() {
@@ -562,7 +682,7 @@
if (gatewaySnatIp == null) {
return;
}
- setStatefulSnatDownStreamRuleForKubevirtPort(gwNode, gatewaySnatIp, kubevirtPort, true);
+ setStatefulSnatDownStreamRuleForKubevirtPort(router, gwNode, kubevirtPort, true);
}
}
@@ -579,11 +699,7 @@
KubevirtNode gwNode = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
if (gwNode != null) {
- IpAddress gatewaySnatIp = getRouterSnatIpAddress(kubevirtRouterService, kubevirtPort.networkId());
- if (gatewaySnatIp == null) {
- return;
- }
- setStatefulSnatDownStreamRuleForKubevirtPort(gwNode, gatewaySnatIp, kubevirtPort, true);
+ setStatefulSnatDownStreamRuleForKubevirtPort(router, gwNode, kubevirtPort, true);
}
}
@@ -600,11 +716,7 @@
KubevirtNode gwNode = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
if (gwNode != null) {
- IpAddress gatewaySnatIp = getRouterSnatIpAddress(kubevirtRouterService, kubevirtPort.networkId());
- if (gatewaySnatIp == null) {
- return;
- }
- setStatefulSnatDownStreamRuleForKubevirtPort(gwNode, gatewaySnatIp, kubevirtPort, false);
+ setStatefulSnatDownStreamRuleForKubevirtPort(router, gwNode, kubevirtPort, false);
}
}
}
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java
index 91984af..5eb7732 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java
@@ -133,7 +133,7 @@
}
private Set<KubevirtPort> getPortByPod(Pod pod) {
- return getPorts(kubevirtNetworkService.networks(), pod);
+ return getPorts(kubevirtNodeService, kubevirtNetworkService.networks(), pod);
}
private void setIngressRules(Pod pod, boolean install) {
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
index e1a26d7..e061495 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
@@ -34,6 +34,7 @@
import org.onosproject.cfg.ConfigProperty;
import org.onosproject.kubevirtnetworking.api.DefaultKubevirtPort;
import org.onosproject.kubevirtnetworking.api.KubevirtNetwork;
+import org.onosproject.kubevirtnetworking.api.KubevirtNetworkService;
import org.onosproject.kubevirtnetworking.api.KubevirtPort;
import org.onosproject.kubevirtnetworking.api.KubevirtRouter;
import org.onosproject.kubevirtnetworking.api.KubevirtRouterService;
@@ -60,7 +61,6 @@
import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.GATEWAY;
-import static org.onosproject.net.AnnotationKeys.PORT_MAC;
import static org.onosproject.net.AnnotationKeys.PORT_NAME;
/**
@@ -318,11 +318,12 @@
/**
* Obtains the kubevirt port from kubevirt POD.
*
+ * @param nodeService kubevirt node service
* @param networks set of existing kubevirt networks
* @param pod kubevirt POD
* @return kubevirt ports attached to the POD
*/
- public static Set<KubevirtPort> getPorts(Set<KubevirtNetwork> networks, Pod pod) {
+ public static Set<KubevirtPort> getPorts(KubevirtNodeService nodeService, Set<KubevirtNetwork> networks, Pod pod) {
try {
Map<String, String> annots = pod.getMetadata().getAnnotations();
if (annots == null) {
@@ -338,6 +339,12 @@
if (networkStatusStr == null) {
return ImmutableSet.of();
}
+ KubevirtPort.Builder builder = DefaultKubevirtPort.builder();
+
+ KubevirtNode node = nodeService.node(pod.getSpec().getNodeName());
+ if (node != null) {
+ builder.deviceId(node.intgBridge());
+ }
JSONArray networkStatus = new JSONArray(networkStatusStr);
Set<KubevirtPort> ports = new HashSet<>();
@@ -351,8 +358,7 @@
if (network != null) {
String mac = object.getString(MAC);
- KubevirtPort.Builder builder = DefaultKubevirtPort.builder()
- .macAddress(MacAddress.valueOf(mac))
+ builder.macAddress(MacAddress.valueOf(mac))
.networkId(network.networkId());
if (object.has(IPS)) {
@@ -435,7 +441,7 @@
return "";
}
- private static PortNumber portNumber(DeviceId deviceId, String portName) {
+ public static PortNumber portNumber(DeviceId deviceId, String portName) {
DeviceService deviceService = DefaultServiceDirectory.getService(DeviceService.class);
Port port = deviceService.getPorts(deviceId).stream()
.filter(p -> p.isEnabled() &&
@@ -471,12 +477,19 @@
* @param deviceId device Id
* @return mac address of the br-int port
*/
- public static MacAddress getbrIntMacAddress(DeviceService deviceService,
- DeviceId deviceId) {
- return MacAddress.valueOf(deviceService.getPorts(deviceId).stream()
- .filter(port -> Objects.equals(port.annotations().value(PORT_NAME), BR_INT))
- .map(port -> port.annotations().value(PORT_MAC))
- .findAny().orElse(null));
+
+ /**
+ * Returns the mac address of the router.
+ *
+ * @param router kubevirt router
+ * @return macc address of the router
+ */
+ public static MacAddress getRouterMacAddress(KubevirtRouter router) {
+ if (router.mac() == null) {
+ log.warn("Failed to get mac address of router {}", router.name());
+ }
+
+ return router.mac();
}
/**
@@ -535,4 +548,30 @@
.filter(router -> router.internal().contains(kubevirtNetwork.networkId()))
.findAny().orElse(null);
}
+
+ /**
+ * Returns the external patch port number with specified gateway.
+ *
+ * @param deviceService device service
+ * @param gatewayNode gateawy node
+ * @return external patch port number
+ */
+ public static PortNumber externalPatchPortNum(DeviceService deviceService, KubevirtNode gatewayNode) {
+ Port port = deviceService.getPorts(gatewayNode.intgBridge()).stream()
+ .filter(p -> p.isEnabled() &&
+ Objects.equals(p.annotations().value(PORT_NAME), "int-to-gateway"))
+ .findAny().orElse(null);
+
+ return port != null ? port.number() : null;
+ }
+
+ public static KubevirtNetwork getExternalNetworkByRouter(KubevirtNetworkService networkService,
+ KubevirtRouter router) {
+ String networkId = router.external().values().stream().findAny().orElse(null);
+ if (networkId == null) {
+ return null;
+ }
+
+ return networkService.network(networkId);
+ }
}
diff --git a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java
index 4756cde..f224dbc 100644
--- a/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java
+++ b/apps/kubevirt-node/app/src/main/java/org/onosproject/kubevirtnode/impl/DefaultKubevirtNodeHandler.java
@@ -79,10 +79,13 @@
import static org.onosproject.kubevirtnode.api.Constants.GRE;
import static org.onosproject.kubevirtnode.api.Constants.INTEGRATION_BRIDGE;
import static org.onosproject.kubevirtnode.api.Constants.INTEGRATION_TO_PHYSICAL_PREFIX;
+import static org.onosproject.kubevirtnode.api.Constants.INTEGRATION_TO_TUNNEL;
import static org.onosproject.kubevirtnode.api.Constants.PHYSICAL_TO_INTEGRATION_SUFFIX;
import static org.onosproject.kubevirtnode.api.Constants.TENANT_BRIDGE_PREFIX;
import static org.onosproject.kubevirtnode.api.Constants.TUNNEL_BRIDGE;
+import static org.onosproject.kubevirtnode.api.Constants.TUNNEL_TO_INTEGRATION;
import static org.onosproject.kubevirtnode.api.Constants.VXLAN;
+import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.GATEWAY;
import static org.onosproject.kubevirtnode.api.KubevirtNodeService.APP_ID;
import static org.onosproject.kubevirtnode.api.KubevirtNodeState.COMPLETE;
import static org.onosproject.kubevirtnode.api.KubevirtNodeState.DEVICE_CREATED;
@@ -498,6 +501,10 @@
// to physical bridge, adding patch ports to both physical bridge and br-int
provisionPhysicalInterfaces(node);
+ if (node.type() == GATEWAY) {
+ createPatchInterfaceBetweenBrIntBrTun(node);
+ }
+
return node.intgBridge() != null && node.tunBridge() != null &&
deviceService.isAvailable(node.intgBridge()) &&
deviceService.isAvailable(node.tunBridge());
@@ -554,6 +561,14 @@
}
}
+ if (node.type() == GATEWAY) {
+ if (!(hasPhyIntf(node, INTEGRATION_TO_TUNNEL) &&
+ hasPhyIntf(node, TUNNEL_TO_INTEGRATION))) {
+ log.warn("IntToTunPort {}", hasPhyIntf(node, INTEGRATION_TO_TUNNEL));
+ log.warn("TunToIntPort {}", hasPhyIntf(node, TUNNEL_TO_INTEGRATION));
+ return false;
+ }
+ }
return true;
}
@@ -627,6 +642,38 @@
}
}
+
+ private void createPatchInterfaceBetweenBrIntBrTun(KubevirtNode node) {
+ Device device = deviceService.getDevice(node.ovsdb());
+
+ if (device == null || !device.is(InterfaceConfig.class)) {
+ log.error("Failed to create patch interface on {}", node.ovsdb());
+ return;
+ }
+
+ InterfaceConfig ifaceConfig = device.as(InterfaceConfig.class);
+
+ // int bridge -> tunnel bridge
+ PatchDescription brIntTunPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(INTEGRATION_BRIDGE)
+ .ifaceName(INTEGRATION_TO_TUNNEL)
+ .peer(TUNNEL_TO_INTEGRATION)
+ .build();
+
+ ifaceConfig.addPatchMode(INTEGRATION_TO_TUNNEL, brIntTunPatchDesc);
+
+ // tunnel bridge -> int bridge
+ PatchDescription brTunIntPatchDesc =
+ DefaultPatchDescription.builder()
+ .deviceId(TUNNEL_BRIDGE)
+ .ifaceName(TUNNEL_TO_INTEGRATION)
+ .peer(INTEGRATION_TO_TUNNEL)
+ .build();
+
+ ifaceConfig.addPatchMode(TUNNEL_TO_INTEGRATION, brTunIntPatchDesc);
+ }
+
private void unprovisionPhysicalInterfaces(KubevirtNode node) {
node.phyIntfs().forEach(pi -> {
detachPhysicalPort(node, pi.network(), pi.intf());