Implement Internal Routing Capability.
Change-Id: I0e3017f730a046f099c71116ded023aa2f45d154
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
index b734226..a8867f2 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
@@ -36,6 +36,10 @@
import org.onosproject.kubevirtnetworking.api.KubevirtNetworkEvent;
import org.onosproject.kubevirtnetworking.api.KubevirtNetworkListener;
import org.onosproject.kubevirtnetworking.api.KubevirtNetworkService;
+import org.onosproject.kubevirtnetworking.api.KubevirtPort;
+import org.onosproject.kubevirtnetworking.api.KubevirtPortEvent;
+import org.onosproject.kubevirtnetworking.api.KubevirtPortListener;
+import org.onosproject.kubevirtnetworking.api.KubevirtPortService;
import org.onosproject.kubevirtnetworking.api.KubevirtRouter;
import org.onosproject.kubevirtnetworking.api.KubevirtRouterAdminService;
import org.onosproject.kubevirtnetworking.api.KubevirtRouterEvent;
@@ -80,12 +84,15 @@
import static org.onlab.packet.ICMP.TYPE_ECHO_REPLY;
import static org.onlab.packet.ICMP.TYPE_ECHO_REQUEST;
import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.FLAT_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.FORWARDING_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
import static org.onosproject.kubevirtnetworking.api.Constants.PRE_FLAT_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_DHCP_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_FORWARDING_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ICMP_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_INTERNAL_ROUTING_RULE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_ARP_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_DHCP_TABLE;
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_FORWARDING_TABLE;
@@ -94,6 +101,8 @@
import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_TO_TUNNEL_PREFIX;
import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedRouter;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtNetwork;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtPort;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getbrIntMacAddress;
import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.segmentIdHex;
import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.NXM_NX_IP_TTL;
@@ -150,8 +159,18 @@
@Reference(cardinality = ReferenceCardinality.MANDATORY)
protected KubevirtRouterAdminService kubevirtRouterService;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtPortService kubevirtPortService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNetworkService kubevirtNetworkService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ protected KubevirtNodeService kubevirtNodeService;
+
private final KubevirtNetworkListener networkListener = new InternalNetworkEventListener();
private final KubevirtNodeListener nodeListener = new InternalNodeEventListener();
+ private final KubevirtPortListener portListener = new InternalKubevirtPortListener();
private final InternalRouterEventListener kubevirtRouterlistener =
new InternalRouterEventListener();
@@ -166,10 +185,11 @@
protected void activate() {
appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
localNodeId = clusterService.getLocalNode().id();
- networkService.addListener(networkListener);
- nodeService.addListener(nodeListener);
leadershipService.runForLeadership(appId.name());
+ networkService.addListener(networkListener);
+ nodeService.addListener(nodeListener);
+ kubevirtPortService.addListener(portListener);
kubevirtRouterService.addListener(kubevirtRouterlistener);
log.info("Started");
@@ -179,9 +199,9 @@
protected void deactivate() {
networkService.removeListener(networkListener);
nodeService.removeListener(nodeListener);
- leadershipService.withdraw(appId.name());
-
+ kubevirtPortService.removeListener(portListener);
kubevirtRouterService.removeListener(kubevirtRouterlistener);
+ leadershipService.withdraw(appId.name());
eventExecutor.shutdown();
log.info("Stopped");
@@ -228,7 +248,7 @@
deviceService.removeDevice(network.tenantDeviceId(node.hostname()));
}
- private void createPatchInterface(KubevirtNode node, KubevirtNetwork network) {
+ private void createPatchInterfaceInCaseOveray(KubevirtNode node, KubevirtNetwork network) {
Device device = deviceService.getDevice(node.ovsdb());
if (device == null || !device.is(InterfaceConfig.class)) {
@@ -278,6 +298,38 @@
ifaceConfig.removePatchMode(tunToIntIntf);
}
+ private void setArpRules(KubevirtNode node, KubevirtNetwork network) {
+
+ KubevirtRouter router = getRouterForKubevirtNetwork(kubevirtRouterService, network);
+ if (router == null) {
+ return;
+ }
+
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(nodeService, router);
+ if (electedGw == null) {
+ return;
+ }
+
+ setGatewayArpRuleForInternalNetworkInCaseOveray(network, TENANT_ARP_TABLE, electedGw.intgBridge(),
+ network.tenantDeviceId(node.hostname()), true);
+ }
+
+ private void setIcmpRules(KubevirtNode node, KubevirtNetwork network) {
+ KubevirtRouter router = getRouterForKubevirtNetwork(kubevirtRouterService, network);
+ if (router == null) {
+ return;
+ }
+
+ KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(nodeService, router);
+ if (electedGw == null) {
+ return;
+ }
+
+ setGatewayIcmpRuleForInternalNetworkInCaseOveray(network, TENANT_ICMP_TABLE, electedGw.intgBridge(),
+ network.tenantDeviceId(node.hostname()), true);
+ }
+
+
private void setDefaultRules(KubevirtNode node, KubevirtNetwork network) {
DeviceId deviceId = network.tenantDeviceId(node.hostname());
@@ -298,10 +350,6 @@
setDhcpRule(deviceId, true);
setForwardingRule(deviceId, true);
- setGatewayArpRuleForInternalNetwork(network, TENANT_ARP_TABLE,
- network.tenantDeviceId(node.hostname()), true);
- setGatewayIcmpRuleForInternalNetwork(network, TENANT_ICMP_TABLE,
- network.tenantDeviceId(node.hostname()), true);
log.info("Install default flow rules for tenant bridge {}", network.tenantBridgeName());
}
@@ -348,29 +396,203 @@
KubevirtRouter router,
KubevirtNode electedGateway,
boolean install) {
- setGatewayArpRuleForInternalNetwork(network, PRE_FLAT_TABLE, electedGateway.intgBridge(), install);
- setGatewayIcmpRuleForInternalNetwork(network, PRE_FLAT_TABLE, electedGateway.intgBridge(), install);
+ switch (network.type()) {
+ case VXLAN:
+ case GRE:
+ case GENEVE:
+
+ kubevirtNodeService.completeNodes(WORKER).forEach(node -> {
+ setGatewayArpRuleForInternalNetworkInCaseOveray(network, TENANT_ARP_TABLE,
+ electedGateway.intgBridge(),
+ network.tenantDeviceId(node.hostname()), install);
+ setGatewayIcmpRuleForInternalNetworkInCaseOveray(network, TENANT_ICMP_TABLE,
+ electedGateway.intgBridge(),
+ network.tenantDeviceId(node.hostname()), install);
+ });
+ break;
+ case FLAT:
+ case VLAN:
+ setGatewayArpRuleForInternalNetworkInCaseVlanFlat(network, PRE_FLAT_TABLE,
+ electedGateway.intgBridge(), install);
+ setGatewayIcmpRuleForInternalNetworkInCaseVlanFlat(network, PRE_FLAT_TABLE,
+ electedGateway.intgBridge(), install);
+ setGatewayInterNetworkRoutingWithinSameRouter(network, router, electedGateway, install);
+ break;
+ default:
+ // do nothing
+ break;
+ }
+
}
+ private void setGatewayIcmpRuleForInternalNetworkInCaseOveray(KubevirtNetwork network,
+ int tableNum,
+ DeviceId gwDeviceId,
+ DeviceId tenantDeviceId,
+ boolean install) {
+ MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gwDeviceId);
- private void setGatewayInterNetworkRouting(KubevirtNetwork network, KubevirtRouter router, boolean install) {
+ if (brIntMacAddress == null) {
+ log.warn("Setting gateway ICMP rule for internal network because " +
+ "there's no br-int port for device {}", gwDeviceId);
+ return;
+ }
+
+ Device device = deviceService.getDevice(tenantDeviceId);
+
+ if (device == null) {
+ log.warn("Setting gateway icmp rule for internal network because " +
+ "there's no tenant device for {} to install gateway arp rule",
+ tenantDeviceId);
+ return;
+ }
+
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(IPv4.PROTOCOL_ICMP)
+ .matchIcmpType(TYPE_ECHO_REQUEST)
+ .matchIcmpCode(CODE_ECHO_REQEUST)
+ .matchIPDst(IpPrefix.valueOf(network.gatewayIp(), 32));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+ .extension(buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(buildMoveIpSrcToDstExtension(device), device.id())
+ .extension(buildLoadExtension(device,
+ NXM_NX_IP_TTL, DEFAULT_TTL), device.id())
+ .extension(buildLoadExtension(device,
+ NXM_OF_ICMP_TYPE, TYPE_ECHO_REPLY), device.id())
+ .setIpSrc(network.gatewayIp())
+ .setEthSrc(brIntMacAddress)
+ .setOutput(PortNumber.IN_PORT);
+
+ flowService.setRule(
+ appId,
+ tenantDeviceId,
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_ICMP_RULE,
+ tableNum,
+ install);
+ }
+
+ private void setGatewayArpRuleForInternalNetworkInCaseOveray(KubevirtNetwork network,
+ int tableNum,
+ DeviceId gwDeviceId,
+ DeviceId tenantDeviceId,
+ boolean install) {
+
+ MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gwDeviceId);
+
+ if (brIntMacAddress == null) {
+ log.warn("Setting gateway arp rule for internal network because " +
+ "there's no br-int port for device {}", gwDeviceId);
+ return;
+ }
+
+ Device device = deviceService.getDevice(tenantDeviceId);
+
+ if (device == null) {
+ log.warn("Setting gateway arp rule for internal network because " +
+ "there's no tenant device for {} to install gateway arp rule",
+ tenantDeviceId);
+ return;
+ }
+
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .matchArpOp(ARP.OP_REQUEST)
+ .matchArpTpa(Ip4Address.valueOf(network.gatewayIp().toString()));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+ tBuilder.extension(buildMoveEthSrcToDstExtension(device), device.id())
+ .extension(buildMoveArpShaToThaExtension(device), device.id())
+ .extension(buildMoveArpSpaToTpaExtension(device), device.id())
+ .setArpOp(ARP.OP_REPLY)
+ .setArpSha(brIntMacAddress)
+ .setArpSpa(Ip4Address.valueOf(network.gatewayIp().toString()))
+ .setEthSrc(brIntMacAddress)
+ .setOutput(PortNumber.IN_PORT);
+
+ flowService.setRule(
+ appId,
+ device.id(),
+ sBuilder.build(),
+ tBuilder.build(),
+ PRIORITY_ARP_GATEWAY_RULE,
+ tableNum,
+ install
+ );
+ }
+
+ private void setGatewayInterNetworkRoutingWithinSameRouter(
+ KubevirtNetwork network, KubevirtRouter router, KubevirtNode gatewayNode, boolean install) {
router.internal().forEach(srcNetwork -> {
- if (srcNetwork.equals(network.networkId())) {
+ if (srcNetwork.equals(network.networkId())
+ || kubevirtNetworkService.network(srcNetwork) == null) {
return;
}
+ kubevirtPortService.ports(network.networkId()).forEach(port -> {
+ setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+ port, gatewayNode, install);
+ });
});
}
- private void setGatewayArpRuleForInternalNetwork(KubevirtNetwork network,
- int tableNum, DeviceId deviceId, boolean install) {
- Device device = deviceService.getDevice(deviceId);
+ private void setGatewayInterNetworkRoutingFromNetworkToPort(KubevirtNetwork srcNetwork, KubevirtPort dstPort,
+ KubevirtNode gatewayNode, boolean install) {
- MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, deviceId);
+ Device gwDevice = deviceService.getDevice(gatewayNode.intgBridge());
+
+ if (gwDevice == null) {
+ log.warn("Failed to set internal network routing rule because " +
+ "there's no device Id for device {}", gatewayNode.intgBridge());
+ return;
+ }
+
+ MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gatewayNode.intgBridge());
if (brIntMacAddress == null) {
- log.error("Setting gateway arp rule for internal network because " +
- "there's no br-int port for device {}", deviceId);
+ log.warn("Failed to set internal network routing rule because " +
+ "there's no br-int port for device {}", gatewayNode.intgBridge());
+ return;
+ }
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchEthDst(brIntMacAddress)
+ .matchIPSrc(IpPrefix.valueOf(srcNetwork.cidr()))
+ .matchIPDst(IpPrefix.valueOf(dstPort.ipAddress(), 32));
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setEthSrc(brIntMacAddress)
+ .setEthDst(dstPort.macAddress())
+ .transition(FORWARDING_TABLE)
+ .build();
+
+ flowService.setRule(
+ appId,
+ gwDevice.id(),
+ sBuilder.build(),
+ treatment,
+ PRIORITY_INTERNAL_ROUTING_RULE,
+ FLAT_TABLE,
+ install
+ );
+ }
+
+ private void setGatewayArpRuleForInternalNetworkInCaseVlanFlat(KubevirtNetwork network,
+ int tableNum, DeviceId gwDeviceId, boolean install) {
+
+
+ Device device = deviceService.getDevice(gwDeviceId);
+ MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gwDeviceId);
+
+ if (brIntMacAddress == null) {
+ log.warn("Setting gateway arp rule for internal network because " +
+ "there's no br-int port for device {}", gwDeviceId);
return;
}
@@ -408,8 +630,8 @@
* @param deviceId device id of the selected gateway for the network
* @param install install if true, remove otherwise
*/
- private void setGatewayIcmpRuleForInternalNetwork(KubevirtNetwork network,
- int tableNum, DeviceId deviceId, boolean install) {
+ private void setGatewayIcmpRuleForInternalNetworkInCaseVlanFlat(KubevirtNetwork network,
+ int tableNum, DeviceId deviceId, boolean install) {
MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, deviceId);
if (brIntMacAddress == null) {
@@ -586,8 +808,19 @@
if (network != null) {
initGatewayNodeForInternalNetwork(network, router, electedGw, false);
}
- });
+ removeDetachedInternalNetworkRules(network, router, electedGw);
+ });
+ }
+
+ private void removeDetachedInternalNetworkRules(KubevirtNetwork removedNetwork, KubevirtRouter router,
+ KubevirtNode electedGw) {
+ router.internal().forEach(networkName -> {
+ kubevirtPortService.ports(networkName).forEach(kubevirtPort -> {
+ setGatewayInterNetworkRoutingFromNetworkToPort(
+ removedNetwork, kubevirtPort, electedGw, false);
+ });
+ });
}
private void processRouterGatewayNodeAttached(KubevirtRouter router,
@@ -733,7 +966,7 @@
nodeService.completeNodes().forEach(n -> {
createBridge(n, network);
- createPatchInterface(n, network);
+ createPatchInterfaceInCaseOveray(n, network);
setDefaultRules(n, network);
});
}
@@ -788,8 +1021,11 @@
continue;
}
createBridge(node, network);
- createPatchInterface(node, network);
+ createPatchInterfaceInCaseOveray(node, network);
setDefaultRules(node, network);
+ setArpRules(node, network);
+ setIcmpRules(node, network);
+
break;
case FLAT:
case VLAN:
@@ -850,4 +1086,105 @@
});
}
}
+
+ private class InternalKubevirtPortListener implements KubevirtPortListener {
+
+ private boolean isRelevantHelper() {
+ return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+ }
+
+ @Override
+ public void event(KubevirtPortEvent event) {
+ switch (event.type()) {
+ case KUBEVIRT_PORT_CREATED:
+ eventExecutor.execute(() -> processPortCreation(event.subject()));
+ break;
+ case KUBEVIRT_PORT_UPDATED:
+ eventExecutor.execute(() -> processPortUpdate(event.subject()));
+ break;
+ case KUBEVIRT_PORT_REMOVED:
+ eventExecutor.execute(() -> processPortDeletion(event.subject()));
+ break;
+ default:
+ //do nothing
+ break;
+ }
+ }
+
+ private void processPortCreation(KubevirtPort kubevirtPort) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ KubevirtRouter router = getRouterForKubevirtPort(kubevirtRouterService, kubevirtPort);
+ if (router == null) {
+ return;
+ }
+
+ KubevirtNode gwNode = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+ if (gwNode != null) {
+
+ router.internal().forEach(srcNetwork -> {
+ if (srcNetwork.equals(kubevirtPort.networkId())
+ || kubevirtNetworkService.network(srcNetwork) == null) {
+ return;
+ }
+ setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+ kubevirtPort, gwNode, true);
+ });
+ }
+ }
+
+ private void processPortUpdate(KubevirtPort kubevirtPort) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ KubevirtRouter router = getRouterForKubevirtPort(kubevirtRouterService, kubevirtPort);
+ if (router == null) {
+ return;
+ }
+
+ KubevirtNode gwNode = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+ if (gwNode != null) {
+
+ router.internal().forEach(srcNetwork -> {
+ if (srcNetwork.equals(kubevirtPort.networkId())
+ || kubevirtNetworkService.network(srcNetwork) == null) {
+ return;
+ }
+ setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+ kubevirtPort, gwNode, true);
+ });
+ }
+ }
+
+ private void processPortDeletion(KubevirtPort kubevirtPort) {
+ if (!isRelevantHelper()) {
+ return;
+ }
+
+ KubevirtRouter router = getRouterForKubevirtPort(kubevirtRouterService, kubevirtPort);
+ if (router == null) {
+ return;
+ }
+
+ KubevirtNode gwNode = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+ if (gwNode != null) {
+
+ router.internal().forEach(srcNetwork -> {
+ if (srcNetwork.equals(kubevirtPort.networkId())
+ || kubevirtNetworkService.network(srcNetwork) == null) {
+ return;
+ }
+ setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+ kubevirtPort, gwNode, false);
+ });
+ }
+
+ }
+ }
}