Refactored OpenstackSwitching app
[DONE]
- Restructured to activate or deactivate switching and routing app separately
- Fixed to add or remove host when port is detected or vanished
- Use openstack node service to get integration bridges and data IP
[TODO]
- Remove use of OpenstackPortInfo
- Support installing flow rules for exising VMs
- Call security group update method when port update triggered from OpenStack
Change-Id: Ic0b2ac3f7ab07f0e20c97c6edfdd1928b9767baf
diff --git a/apps/openstacknetworking/routing/BUCK b/apps/openstacknetworking/routing/BUCK
new file mode 100644
index 0000000..fe48a99
--- /dev/null
+++ b/apps/openstacknetworking/routing/BUCK
@@ -0,0 +1,28 @@
+COMPILE_DEPS = [
+ '//lib:CORE_DEPS',
+ '//core/store/serializers:onos-core-serializers',
+ '//apps/openstackinterface/api:onos-apps-openstackinterface-api',
+ '//apps/openstacknetworking/api:onos-apps-openstacknetworking-api',
+ '//apps/scalablegateway:onos-apps-scalablegateway',
+ '//apps/openstacknode:onos-apps-openstacknode',
+]
+
+BUNDLES = [
+ '//apps/openstacknetworking/api:onos-apps-openstacknetworking-api',
+ '//apps/openstacknetworking/web:onos-apps-openstacknetworking-web',
+ '//apps/openstacknetworking/routing:onos-apps-openstacknetworking-routing',
+]
+
+osgi_jar_with_tests (
+ deps = COMPILE_DEPS,
+)
+
+onos_app (
+ app_name = 'org.onosproject.openstackrouting',
+ title = 'OpenStack Routing App',
+ category = 'Utility',
+ url = 'http://onosproject.org',
+ description = 'OpenStack routing application.',
+ included_bundles = BUNDLES,
+ required_apps = [ 'org.onosproject.openstackinterface', 'org.onosproject.openstacknode', 'org.onosproject.scalablegateway' ]
+)
diff --git a/apps/openstacknetworking/routing/features.xml b/apps/openstacknetworking/routing/features.xml
new file mode 100644
index 0000000..f9be32e
--- /dev/null
+++ b/apps/openstacknetworking/routing/features.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Copyright 2016-present Open Networking Laboratory
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<features xmlns="http://karaf.apache.org/xmlns/features/v1.2.0" name="${project.artifactId}-${project.version}">
+ <feature name="${project.artifactId}" version="${project.version}"
+ description="${project.description}">
+ <feature>onos-api</feature>
+ <bundle>mvn:${project.groupId}/onos-app-openstacknetworking-api/${project.version}</bundle>
+ <bundle>mvn:${project.groupId}/onos-app-openstacknetworking-web/${project.version}</bundle>
+ <bundle>mvn:${project.groupId}/onos-app-openstackrouting/${project.version}</bundle>
+ </feature>
+</features>
diff --git a/apps/openstacknetworking/routing/pom.xml b/apps/openstacknetworking/routing/pom.xml
new file mode 100644
index 0000000..cc4f7d0
--- /dev/null
+++ b/apps/openstacknetworking/routing/pom.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2016-present Open Networking Laboratory
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.onosproject</groupId>
+ <artifactId>onos-app-openstacknetworking</artifactId>
+ <version>1.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>onos-app-openstackrouting</artifactId>
+ <packaging>bundle</packaging>
+
+ <properties>
+ <onos.app.name>org.onosproject.openstackrouting</onos.app.name>
+ <onos.app.title>Openstack Switching App</onos.app.title>
+ <onos.app.category>Traffic Steering</onos.app.category>
+ <onos.app.url>http://onosproject.org</onos.app.url>
+ <onos.app.requires>
+ org.onosproject.openstackinterface,
+ org.onosproject.openstacknode
+ </onos.app.requires>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.osgi</groupId>
+ <artifactId>org.osgi.compendium</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.onosproject</groupId>
+ <artifactId>onos-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.onosproject</groupId>
+ <artifactId>onos-core-serializers</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.onosproject</groupId>
+ <artifactId>onos-app-openstacknetworking-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.onosproject</groupId>
+ <artifactId>onos-app-openstackinterface-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.onosproject</groupId>
+ <artifactId>onos-scalablegateway</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+</project>
+
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIPHandler.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIPHandler.java
new file mode 100644
index 0000000..073a47f
--- /dev/null
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIPHandler.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import org.onosproject.openstackinterface.OpenstackFloatingIP;
+import org.onosproject.openstacknetworking.OpenstackPortInfo;
+
+/**
+ * Handle FloatingIP Event for Managing Flow Rules In Openstack Nodes.
+ */
+public class OpenstackFloatingIPHandler implements Runnable {
+
+ private final OpenstackFloatingIP floatingIP;
+ private final OpenstackRoutingRulePopulator rulePopulator;
+ private boolean associate;
+ private final OpenstackPortInfo portInfo;
+
+ OpenstackFloatingIPHandler(OpenstackRoutingRulePopulator rulePopulator,
+ OpenstackFloatingIP openstackFloatingIP, boolean associate, OpenstackPortInfo portInfo) {
+ this.floatingIP = openstackFloatingIP;
+ this.rulePopulator = rulePopulator;
+ this.associate = associate;
+ this.portInfo = portInfo;
+ }
+
+ @Override
+ public void run() {
+ if (associate) {
+ rulePopulator.populateFloatingIpRules(floatingIP);
+ } else {
+ rulePopulator.removeFloatingIpRules(floatingIP, portInfo);
+ }
+
+ }
+}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
new file mode 100644
index 0000000..9678598
--- /dev/null
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
@@ -0,0 +1,344 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import com.google.common.collect.Maps;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.ICMP;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.MacAddress;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.OutboundPacket;
+import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketPriority;
+import org.onosproject.net.packet.PacketService;
+import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackPort;
+import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.openstacknetworking.OpenstackPortInfo;
+import org.onosproject.openstacknetworking.OpenstackSwitchingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.slf4j.Logger;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+
+/**
+ * Handle ICMP packet sent from Openstack Gateway nodes.
+ */
+public class OpenstackIcmpHandler {
+ protected final Logger log = getLogger(getClass());
+
+ private final PacketService packetService;
+ private final DeviceService deviceService;
+ private final Map<String, OpenstackPortInfo> icmpInfoMap = Maps.newHashMap();
+ private final OpenstackSwitchingService openstackSwitchingService;
+ private final OpenstackInterfaceService openstackService;
+ private final ScalableGatewayService gatewayService;
+ private final OpenstackNetworkingConfig config;
+ private static final MacAddress GATEWAY_MAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
+ private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface";
+ private static final String PORTNAME = "portName";
+ private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
+ private static final String NETWORK_FLOATING_IP = "network:floatingip";
+ private static final String EXTERNAL_NODE_NULL = "There is no external node about this deviceId []";
+ /**
+ * Default constructor.
+ *
+ * @param packetService packet service
+ * @param deviceService device service
+ * @param openstackService openstackInterface service
+ * @param config openstackRoutingConfig
+ * @param openstackSwitchingService openstackSwitching service
+ * @param gatewayService scalable gateway service
+ */
+ OpenstackIcmpHandler(PacketService packetService, DeviceService deviceService,
+ OpenstackInterfaceService openstackService, OpenstackNetworkingConfig config,
+ OpenstackSwitchingService openstackSwitchingService, ScalableGatewayService gatewayService) {
+ this.packetService = packetService;
+ this.deviceService = deviceService;
+ this.openstackService = checkNotNull(openstackService);
+ this.config = checkNotNull(config);
+ this.openstackSwitchingService = checkNotNull(openstackSwitchingService);
+ this.gatewayService = gatewayService;
+ }
+
+ /**
+ * Requests ICMP packet.
+ *
+ * @param appId Application Id
+ */
+ public void requestPacket(ApplicationId appId) {
+ TrafficSelector icmpSelector = DefaultTrafficSelector.builder()
+ .matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(IPv4.PROTOCOL_ICMP)
+ .build();
+
+ Map<DeviceId, PortNumber> externalInfoMap = getExternalInfo();
+
+ externalInfoMap.keySet().forEach(deviceId ->
+ packetService.requestPackets(icmpSelector,
+ PacketPriority.CONTROL,
+ appId,
+ Optional.of(deviceId)));
+ }
+
+ /**
+ * Handles ICMP packet.
+ *
+ * @param context packet context
+ * @param ethernet ethernet
+ */
+ public void processIcmpPacket(PacketContext context, Ethernet ethernet) {
+ checkNotNull(context, "context can not be null");
+ checkNotNull(ethernet, "ethernet can not be null");
+
+ IPv4 ipPacket = (IPv4) ethernet.getPayload();
+
+ log.debug("icmpEvent called from ip {}, mac {}", Ip4Address.valueOf(ipPacket.getSourceAddress()).toString(),
+ ethernet.getSourceMAC().toString());
+
+ ICMP icmp = (ICMP) ipPacket.getPayload();
+ short icmpId = getIcmpId(icmp);
+
+ DeviceId deviceId = context.inPacket().receivedFrom().deviceId();
+ PortNumber portNumber = context.inPacket().receivedFrom().port();
+ if (icmp.getIcmpType() == ICMP.TYPE_ECHO_REQUEST) {
+ //TODO: Considers icmp between internal subnets which are belonged to the same router.
+
+ OpenstackPortInfo openstackPortInfo =
+ getOpenstackPortInfo(Ip4Address.valueOf(ipPacket.getSourceAddress()), ethernet.getSourceMAC());
+
+ //checkNotNull(openstackPortInfo, "openstackPortInfo can not be null");
+ if (requestToOpenstackRoutingNetwork(ipPacket.getDestinationAddress())) {
+ if (openstackPortInfo == null) {
+ if (config.gatewayBridgeId().equals(context.inPacket().receivedFrom().deviceId().toString())) {
+ if (portNumber.equals(getPortForAnnotationPortName(deviceId,
+ config.gatewayExternalInterfaceName()))) {
+ processIcmpPacketSentToExtenal(ipPacket, icmp, ipPacket.getSourceAddress(),
+ ethernet.getSourceMAC(), deviceId, portNumber);
+ return;
+ }
+ }
+ return;
+ } else {
+ processIcmpPacketSentToGateway(ipPacket, icmp, openstackPortInfo);
+ return;
+ }
+ }
+
+ if (ipPacket.getDestinationAddress() == openstackPortInfo.gatewayIP().toInt()) {
+ processIcmpPacketSentToGateway(ipPacket, icmp, openstackPortInfo);
+ } else {
+ Ip4Address pNatIpAddress = pNatIpForPort(openstackPortInfo);
+ checkNotNull(pNatIpAddress, "pNatIpAddress can not be null");
+
+ sendRequestPacketToExt(ipPacket, icmp, deviceId, pNatIpAddress);
+
+ String icmpInfoKey = String.valueOf(icmpId)
+ .concat(String.valueOf(pNatIpAddress.toInt()))
+ .concat(String.valueOf(ipPacket.getDestinationAddress()));
+ icmpInfoMap.putIfAbsent(icmpInfoKey, openstackPortInfo);
+ }
+ } else if (icmp.getIcmpType() == ICMP.TYPE_ECHO_REPLY) {
+ String icmpInfoKey = String.valueOf(icmpId)
+ .concat(String.valueOf(ipPacket.getDestinationAddress()))
+ .concat(String.valueOf(ipPacket.getSourceAddress()));
+
+ processResponsePacketFromExternalToHost(ipPacket, icmp, icmpInfoMap.get(icmpInfoKey));
+
+ icmpInfoMap.remove(icmpInfoKey);
+ }
+ }
+
+ private void processIcmpPacketSentToExtenal(IPv4 icmpRequestIpv4, ICMP icmpRequest,
+ int destAddr, MacAddress destMac,
+ DeviceId deviceId, PortNumber portNumber) {
+ icmpRequest.setChecksum((short) 0);
+ icmpRequest.setIcmpType(ICMP.TYPE_ECHO_REPLY).resetChecksum();
+ icmpRequestIpv4.setSourceAddress(icmpRequestIpv4.getDestinationAddress())
+ .setDestinationAddress(destAddr).resetChecksum();
+ icmpRequestIpv4.setPayload(icmpRequest);
+ Ethernet icmpResponseEth = new Ethernet();
+ icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
+ .setSourceMACAddress(config.gatewayExternalInterfaceMac())
+ .setDestinationMACAddress(destMac).setPayload(icmpRequestIpv4);
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder().setOutput(portNumber).build();
+ OutboundPacket packet = new DefaultOutboundPacket(deviceId,
+ treatment, ByteBuffer.wrap(icmpResponseEth.serialize()));
+ packetService.emit(packet);
+ }
+
+ private void processIcmpPacketSentToGateway(IPv4 icmpRequestIpv4, ICMP icmpRequest,
+ OpenstackPortInfo openstackPortInfo) {
+ icmpRequest.setChecksum((short) 0);
+ icmpRequest.setIcmpType(ICMP.TYPE_ECHO_REPLY)
+ .resetChecksum();
+
+ icmpRequestIpv4.setSourceAddress(icmpRequestIpv4.getDestinationAddress())
+ .setDestinationAddress(openstackPortInfo.ip().toInt())
+ .resetChecksum();
+
+ icmpRequestIpv4.setPayload(icmpRequest);
+
+ Ethernet icmpResponseEth = new Ethernet();
+
+ icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
+ .setSourceMACAddress(GATEWAY_MAC)
+ .setDestinationMACAddress(openstackPortInfo.mac())
+ .setPayload(icmpRequestIpv4);
+
+ sendResponsePacketToHost(icmpResponseEth, openstackPortInfo);
+ }
+
+ private void sendRequestPacketToExt(IPv4 icmpRequestIpv4, ICMP icmpRequest, DeviceId deviceId,
+ Ip4Address pNatIpAddress) {
+ icmpRequest.resetChecksum();
+ icmpRequestIpv4.setSourceAddress(pNatIpAddress.toInt())
+ .resetChecksum();
+ icmpRequestIpv4.setPayload(icmpRequest);
+
+ Ethernet icmpRequestEth = new Ethernet();
+
+ icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
+ .setPayload(icmpRequestIpv4);
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ Map<DeviceId, PortNumber> externalInforMap = getExternalInfo();
+
+ if (externalInforMap.size() == 0 || !externalInforMap.containsKey(deviceId)) {
+ log.error(EXTERNAL_NODE_NULL, deviceId.toString());
+ return;
+ }
+ tBuilder.setOutput(externalInforMap.get(deviceId));
+
+ TrafficTreatment treatment = tBuilder.build();
+
+ OutboundPacket packet = new DefaultOutboundPacket(deviceId,
+ treatment, ByteBuffer.wrap(icmpRequestEth.serialize()));
+
+ packetService.emit(packet);
+ }
+
+ private void processResponsePacketFromExternalToHost(IPv4 icmpResponseIpv4, ICMP icmpResponse,
+ OpenstackPortInfo openstackPortInfo) {
+ icmpResponse.resetChecksum();
+
+ icmpResponseIpv4.setDestinationAddress(openstackPortInfo.ip().toInt())
+ .resetChecksum();
+ icmpResponseIpv4.setPayload(icmpResponse);
+
+ Ethernet icmpResponseEth = new Ethernet();
+
+ icmpResponseEth.setEtherType(Ethernet.TYPE_IPV4)
+ .setSourceMACAddress(GATEWAY_MAC)
+ .setDestinationMACAddress(openstackPortInfo.mac())
+ .setPayload(icmpResponseIpv4);
+
+ sendResponsePacketToHost(icmpResponseEth, openstackPortInfo);
+ }
+
+ private void sendResponsePacketToHost(Ethernet icmpResponseEth, OpenstackPortInfo openstackPortInfo) {
+ Map.Entry<String, OpenstackPortInfo> entry = openstackSwitchingService.openstackPortInfo().entrySet().stream()
+ .filter(e -> e.getValue().mac().equals(openstackPortInfo.mac()))
+ .findAny().orElse(null);
+
+ if (entry == null) {
+ return;
+ }
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(getPortForAnnotationPortName(openstackPortInfo.deviceId(), entry.getKey()))
+ .build();
+
+ OutboundPacket packet = new DefaultOutboundPacket(openstackPortInfo.deviceId(),
+ treatment, ByteBuffer.wrap(icmpResponseEth.serialize()));
+
+ packetService.emit(packet);
+ }
+
+ private OpenstackPortInfo getOpenstackPortInfo(Ip4Address sourceIp, MacAddress sourceMac) {
+ checkNotNull(openstackSwitchingService.openstackPortInfo(), "openstackportinfo collection can not be null");
+
+ return openstackSwitchingService.openstackPortInfo().values()
+ .stream().filter(p -> p.ip().equals(sourceIp) && p.mac().equals(sourceMac))
+ .findAny().orElse(null);
+ }
+
+ private short getIcmpId(ICMP icmp) {
+ return ByteBuffer.wrap(icmp.serialize(), 4, 2).getShort();
+ }
+
+ private Ip4Address pNatIpForPort(OpenstackPortInfo openstackPortInfo) {
+
+ OpenstackPort openstackPort = openstackService.ports().stream()
+ .filter(p -> p.deviceOwner().equals(NETWORK_ROUTER_INTERFACE) &&
+ p.networkId().equals(openstackPortInfo.networkId()))
+ .findAny().orElse(null);
+
+ checkNotNull(openstackPort, "openstackPort can not be null");
+
+ return openstackService.router(openstackPort.deviceId())
+ .gatewayExternalInfo().externalFixedIps().values()
+ .stream().findAny().orElse(null);
+ }
+
+ private PortNumber getPortForAnnotationPortName(DeviceId deviceId, String match) {
+ Port port = deviceService.getPorts(deviceId).stream()
+ .filter(p -> p.annotations().value(PORTNAME).equals(match))
+ .findAny().orElse(null);
+
+ checkNotNull(port, "port cannot be null");
+
+ return port.number();
+ }
+
+ private boolean requestToOpenstackRoutingNetwork(int destAddr) {
+ OpenstackPort port = openstackService.ports().stream()
+ .filter(p -> p.deviceOwner().equals(NETWORK_ROUTER_GATEWAY) ||
+ p.deviceOwner().equals(NETWORK_FLOATING_IP))
+ .filter(p -> p.fixedIps().containsValue(
+ Ip4Address.valueOf(destAddr)))
+ .findAny().orElse(null);
+ if (port == null) {
+ return false;
+ }
+ return true;
+ }
+ private Map<DeviceId, PortNumber> getExternalInfo() {
+ Map<DeviceId, PortNumber> externalInfoMap = Maps.newHashMap();
+ gatewayService.getGatewayDeviceIds().forEach(deviceId ->
+ externalInfoMap.putIfAbsent(deviceId, gatewayService.getGatewayExternalPorts(deviceId).get(0)));
+ return externalInfoMap;
+ }
+}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
new file mode 100644
index 0000000..3f11e6c
--- /dev/null
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.TCP;
+import org.onlab.packet.UDP;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.InboundPacket;
+import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketService;
+import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackPort;
+import org.onosproject.openstackinterface.OpenstackRouter;
+import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onlab.osgi.DefaultServiceDirectory.getService;
+
+
+/**
+ * Handle NAT packet processing for Managing Flow Rules In Openstack Nodes.
+ */
+public class OpenstackPnatHandler implements Runnable {
+
+ volatile PacketContext context;
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ protected PacketService packetService;
+
+ private final OpenstackRoutingRulePopulator rulePopulator;
+ private final int portNum;
+ private final OpenstackPort openstackPort;
+ private final Port port;
+ private OpenstackNetworkingConfig config;
+
+ private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
+ private static final String EXTERNAL_PORT_NULL = "There is no external port in this deviceId []";
+
+ OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
+ int portNum, OpenstackPort openstackPort, Port port, OpenstackNetworkingConfig config) {
+ this.rulePopulator = checkNotNull(rulePopulator);
+ this.context = checkNotNull(context);
+ this.portNum = checkNotNull(portNum);
+ this.openstackPort = checkNotNull(openstackPort);
+ this.port = checkNotNull(port);
+ this.config = checkNotNull(config);
+ }
+
+ @Override
+ public void run() {
+ InboundPacket inboundPacket = context.inPacket();
+ Ethernet ethernet = checkNotNull(inboundPacket.parsed());
+
+ //TODO: Considers IPV6
+ if (ethernet.getEtherType() != Ethernet.TYPE_IPV4) {
+ log.warn("Now, we just consider IP version 4");
+ return;
+ }
+
+ OpenstackRouter router = getOpenstackRouter(openstackPort);
+
+ MacAddress externalMac = MacAddress.NONE;
+ MacAddress routerMac = MacAddress.NONE;
+
+ rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
+ getExternalIp(router), externalMac, routerMac);
+
+ packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router);
+ }
+
+ private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) {
+ OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class);
+
+ OpenstackPort port = networkingService.ports()
+ .stream()
+ .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
+ .filter(p -> checkSameSubnet(p, openstackPort))
+ .findAny()
+ .orElse(null);
+
+ return checkNotNull(networkingService.router(port.deviceId()));
+ }
+
+ private boolean checkSameSubnet(OpenstackPort p, OpenstackPort openstackPort) {
+ String key1 = checkNotNull(p.fixedIps().keySet().stream().findFirst().orElse(null)).toString();
+ String key2 = checkNotNull(openstackPort.fixedIps().keySet().stream().findFirst().orElse(null)).toString();
+ return key1.equals(key2) ? true : false;
+ }
+
+ private Ip4Address getExternalIp(OpenstackRouter router) {
+ return router.gatewayExternalInfo().externalFixedIps().values().stream().findAny().orElse(null);
+ }
+
+ private void packetOut(Ethernet ethernet, DeviceId deviceId, int portNum, OpenstackRouter router) {
+ PacketService packetService = getService(PacketService.class);
+
+ IPv4 iPacket = (IPv4) ethernet.getPayload();
+
+ TrafficTreatment.Builder treatment = DefaultTrafficTreatment.builder();
+
+ switch (iPacket.getProtocol()) {
+ case IPv4.PROTOCOL_TCP:
+ TCP tcpPacket = (TCP) iPacket.getPayload();
+ tcpPacket.setSourcePort(portNum);
+ tcpPacket.resetChecksum();
+ tcpPacket.setParent(iPacket);
+ iPacket.setPayload(tcpPacket);
+ break;
+ case IPv4.PROTOCOL_UDP:
+ UDP udpPacket = (UDP) iPacket.getPayload();
+ udpPacket.setSourcePort(portNum);
+ udpPacket.resetChecksum();
+ udpPacket.setParent(iPacket);
+ iPacket.setPayload(udpPacket);
+ break;
+ default:
+ log.error("Temporally, this method can process UDP and TCP protocol.");
+ return;
+ }
+
+ iPacket.setSourceAddress(getExternalIp(router).toString());
+ iPacket.resetChecksum();
+ iPacket.setParent(ethernet);
+ ethernet.setPayload(iPacket);
+ ScalableGatewayService gatewayService = getService(ScalableGatewayService.class);
+ GatewayNode gatewayNode = gatewayService.getGatewayNode(deviceId);
+ if (gatewayNode.getGatewayExternalInterfaceNames().size() == 0) {
+ log.error(EXTERNAL_PORT_NULL, deviceId.toString());
+ return;
+ }
+ treatment.setOutput(gatewayService.getGatewayExternalPorts(deviceId).get(0));
+
+ ethernet.resetChecksum();
+
+
+ packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(),
+ ByteBuffer.wrap(ethernet.serialize())));
+ }
+}
\ No newline at end of file
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
new file mode 100644
index 0000000..9e23b5d
--- /dev/null
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingArpHandler.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import com.google.common.collect.Lists;
+import org.onlab.packet.ARP;
+import org.onlab.packet.EthType;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.MacAddress;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketPriority;
+import org.onosproject.net.packet.PacketService;
+import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackPort;
+import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.slf4j.Logger;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Optional;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Handle ARP packet sent from Openstack Gateway nodes.
+ */
+public class OpenstackRoutingArpHandler {
+ protected final Logger log = getLogger(getClass());
+
+ private final PacketService packetService;
+ private final OpenstackInterfaceService openstackService;
+ private final OpenstackNetworkingConfig config;
+ private final ScalableGatewayService gatewayService;
+ private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
+ private static final String NETWORK_FLOATING_IP = "network:floatingip";
+
+ /**
+ * Default constructor.
+ * @param packetService packet service
+ * @param openstackService openstackInterface service
+ * @param config openstackRoutingConfig
+ * @param gatewayService
+ */
+ OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService,
+ OpenstackNetworkingConfig config, ScalableGatewayService gatewayService) {
+ this.packetService = packetService;
+ this.openstackService = checkNotNull(openstackService);
+ this.config = checkNotNull(config);
+ this.gatewayService = gatewayService;
+ }
+
+ /**
+ * Requests ARP packet to GatewayNode.
+ *
+ * @param appId application id
+ */
+ public void requestPacket(ApplicationId appId) {
+
+ TrafficSelector arpSelector = DefaultTrafficSelector.builder()
+ .matchEthType(EthType.EtherType.ARP.ethType().toShort())
+ .build();
+
+ getExternalInfo().forEach(deviceId ->
+ packetService.requestPackets(arpSelector,
+ PacketPriority.CONTROL,
+ appId,
+ Optional.of(deviceId))
+ );
+ }
+
+ /**
+ * Handles ARP packet.
+ *
+ * @param context packet context
+ * @param ethernet ethernet
+ */
+ public void processArpPacketFromRouter(PacketContext context, Ethernet ethernet) {
+ checkNotNull(context, "context can not be null");
+ checkNotNull(ethernet, "ethernet can not be null");
+
+
+ ARP arp = (ARP) ethernet.getPayload();
+
+ log.debug("arpEvent called from {} to {}",
+ Ip4Address.valueOf(arp.getSenderProtocolAddress()).toString(),
+ Ip4Address.valueOf(arp.getTargetProtocolAddress()).toString());
+
+ if (arp.getOpCode() != ARP.OP_REQUEST) {
+ return;
+ }
+
+ IpAddress targetIp = Ip4Address.valueOf(arp.getTargetProtocolAddress());
+
+ if (getTargetMacForTargetIp(targetIp.getIp4Address()) == MacAddress.NONE) {
+ return;
+ }
+ MacAddress targetMac = MacAddress.valueOf(config.gatewayExternalInterfaceMac());
+
+ Ethernet ethReply = ARP.buildArpReply(targetIp.getIp4Address(),
+ targetMac, ethernet);
+
+ TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+ .setOutput(context.inPacket().receivedFrom().port())
+ .build();
+
+ packetService.emit(new DefaultOutboundPacket(
+ context.inPacket().receivedFrom().deviceId(),
+ treatment,
+ ByteBuffer.wrap(ethReply.serialize())));
+ }
+
+ private MacAddress getTargetMacForTargetIp(Ip4Address targetIp) {
+ OpenstackPort port = openstackService.ports().stream()
+ .filter(p -> p.deviceOwner().equals(NETWORK_ROUTER_GATEWAY) ||
+ p.deviceOwner().equals(NETWORK_FLOATING_IP))
+ .filter(p -> p.fixedIps().containsValue(targetIp.getIp4Address()))
+ .findAny().orElse(null);
+
+ if (port == null) {
+ return MacAddress.NONE;
+ }
+ return port.macAddress();
+ }
+
+ private List<DeviceId> getExternalInfo() {
+ List<DeviceId> externalInfoList = Lists.newArrayList();
+ gatewayService.getGatewayDeviceIds().forEach(externalInfoList::add);
+ return externalInfoList;
+ }
+}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
new file mode 100644
index 0000000..6da1523
--- /dev/null
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
@@ -0,0 +1,677 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import com.google.common.collect.Lists;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.UDP;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.config.ConfigFactory;
+import org.onosproject.net.config.NetworkConfigEvent;
+import org.onosproject.net.config.NetworkConfigListener;
+import org.onosproject.net.config.NetworkConfigRegistry;
+import org.onosproject.net.config.NetworkConfigService;
+import org.onosproject.net.device.DeviceEvent;
+import org.onosproject.net.device.DeviceListener;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.packet.InboundPacket;
+import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketProcessor;
+import org.onosproject.net.packet.PacketService;
+import org.onosproject.openstackinterface.OpenstackFloatingIP;
+import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackPort;
+import org.onosproject.openstackinterface.OpenstackRouter;
+import org.onosproject.openstackinterface.OpenstackRouterInterface;
+import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.openstacknetworking.OpenstackPortInfo;
+import org.onosproject.openstacknetworking.OpenstackRoutingService;
+import org.onosproject.openstacknetworking.OpenstackSubjectFactories;
+import org.onosproject.openstacknetworking.OpenstackSwitchingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.onosproject.store.service.ConsistentMap;
+import org.onosproject.store.service.Serializer;
+import org.onosproject.store.service.StorageService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onlab.util.Tools.groupedThreads;
+
+@Component(immediate = true)
+@Service
+/**
+ * Populates flow rules about L3 functionality for VMs in Openstack.
+ */
+public class OpenstackRoutingManager implements OpenstackRoutingService {
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected CoreService coreService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected PacketService packetService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DeviceService deviceService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackInterfaceService openstackService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected OpenstackSwitchingService openstackSwitchingService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected FlowObjectiveService flowObjectiveService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected DriverService driverService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected NetworkConfigService configService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected NetworkConfigRegistry configRegistry;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected StorageService storageService;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected ScalableGatewayService gatewayService;
+
+ private ApplicationId appId;
+ private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
+ private ConsistentMap<String, OpenstackFloatingIP> floatingIpMap; // Map<FloatingIp`s Id, FloatingIp object>
+ // Map<RouterInterface`s portId, Corresponded port`s network id>
+ private ConsistentMap<String, String> routerInterfaceMap;
+ private static final String APP_ID = "org.onosproject.openstackrouting";
+ private static final String PORT_NAME = "portName";
+ private static final String PORTNAME_PREFIX_VM = "tap";
+ private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
+ private static final String FLOATING_IP_MAP_NAME = "openstackrouting-floatingip";
+ private static final String TP_PORT_MAP_NAME = "openstackrouting-tpportnum";
+ private static final String ROUTER_INTERFACE_MAP_NAME = "openstackrouting-routerinterface";
+ private static final String COLON = ":";
+ private static final int PNAT_PORT_EXPIRE_TIME = 1200 * 1000;
+ private static final int TP_PORT_MINIMUM_NUM = 1024;
+ private static final int TP_PORT_MAXIMUM_NUM = 65535;
+
+ private final ConfigFactory configFactory =
+ new ConfigFactory(OpenstackSubjectFactories.USER_DEFINED_SUBJECT_FACTORY, OpenstackNetworkingConfig.class,
+ "config") {
+ @Override
+ public OpenstackNetworkingConfig createConfig() {
+ return new OpenstackNetworkingConfig();
+ }
+ };
+
+ private final NetworkConfigListener configListener = new InternalConfigListener();
+
+ private OpenstackNetworkingConfig config;
+ private static final KryoNamespace.Builder FLOATING_IP_SERIALIZER = KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API)
+ .register(OpenstackFloatingIP.FloatingIpStatus.class)
+ .register(OpenstackFloatingIP.class);
+
+ private static final KryoNamespace.Builder NUMBER_SERIALIZER = KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API);
+
+ private static final KryoNamespace.Builder ROUTER_INTERFACE_SERIALIZER = KryoNamespace.newBuilder()
+ .register(KryoNamespaces.API);
+
+ private InternalPacketProcessor internalPacketProcessor = new InternalPacketProcessor();
+ private InternalDeviceListener internalDeviceListener = new InternalDeviceListener();
+ private ExecutorService l3EventExecutorService =
+ Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "L3-event"));
+ private ExecutorService icmpEventExecutorService =
+ Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "icmp-event"));
+ private ExecutorService arpEventExecutorService =
+ Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "arp-event"));
+ private OpenstackIcmpHandler openstackIcmpHandler;
+ private OpenstackRoutingArpHandler openstackArpHandler;
+ private OpenstackRoutingRulePopulator rulePopulator;
+ private Map<DeviceId, Ip4Address> computeNodeMap;
+
+ @Activate
+ protected void activate() {
+ appId = coreService.registerApplication(APP_ID);
+ packetService.addProcessor(internalPacketProcessor, PacketProcessor.director(1));
+ configRegistry.registerConfigFactory(configFactory);
+ configService.addListener(configListener);
+ deviceService.addListener(internalDeviceListener);
+
+ floatingIpMap = storageService.<String, OpenstackFloatingIP>consistentMapBuilder()
+ .withSerializer(Serializer.using(FLOATING_IP_SERIALIZER.build()))
+ .withName(FLOATING_IP_MAP_NAME)
+ .withApplicationId(appId)
+ .build();
+ tpPortNumMap = storageService.<Integer, String>consistentMapBuilder()
+ .withSerializer(Serializer.using(NUMBER_SERIALIZER.build()))
+ .withName(TP_PORT_MAP_NAME)
+ .withApplicationId(appId)
+ .build();
+ routerInterfaceMap = storageService.<String, String>consistentMapBuilder()
+ .withSerializer(Serializer.using(ROUTER_INTERFACE_SERIALIZER.build()))
+ .withName(ROUTER_INTERFACE_MAP_NAME)
+ .withApplicationId(appId)
+ .build();
+
+ readConfiguration();
+
+ log.info("started");
+ }
+
+ @Deactivate
+ protected void deactivate() {
+ packetService.removeProcessor(internalPacketProcessor);
+ deviceService.removeListener(internalDeviceListener);
+ l3EventExecutorService.shutdown();
+ icmpEventExecutorService.shutdown();
+ arpEventExecutorService.shutdown();
+
+ floatingIpMap.clear();
+ tpPortNumMap.clear();
+ routerInterfaceMap.clear();
+
+ log.info("stopped");
+ }
+
+
+ @Override
+ public void createFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
+ floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
+ }
+
+ @Override
+ public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIp) {
+ if (!floatingIpMap.containsKey(openstackFloatingIp.id())) {
+ log.warn("There`s no information about {} in FloatingIpMap", openstackFloatingIp.id());
+ return;
+ }
+ if (openstackFloatingIp.portId() == null || openstackFloatingIp.portId().equals("null")) {
+ OpenstackFloatingIP floatingIp = floatingIpMap.get(openstackFloatingIp.id()).value();
+ OpenstackPortInfo portInfo = openstackSwitchingService.openstackPortInfo()
+ .get(PORTNAME_PREFIX_VM.concat(floatingIp.portId().substring(0, 11)));
+ if (portInfo == null) {
+ log.warn("There`s no portInfo information about portId {}", floatingIp.portId());
+ return;
+ }
+ l3EventExecutorService.execute(
+ new OpenstackFloatingIPHandler(rulePopulator, floatingIp, false, portInfo));
+ floatingIpMap.replace(floatingIp.id(), openstackFloatingIp);
+ } else {
+ floatingIpMap.put(openstackFloatingIp.id(), openstackFloatingIp);
+ l3EventExecutorService.execute(
+ new OpenstackFloatingIPHandler(rulePopulator, openstackFloatingIp, true, null));
+ }
+ }
+
+ @Override
+ public void deleteFloatingIP(String id) {
+ floatingIpMap.remove(id);
+ }
+
+ @Override
+ public void createRouter(OpenstackRouter openstackRouter) {
+ }
+
+ @Override
+ public void updateRouter(OpenstackRouter openstackRouter) {
+ if (openstackRouter.gatewayExternalInfo().externalFixedIps().size() > 0) {
+ checkExternalConnection(openstackRouter, getOpenstackRouterInterface(openstackRouter));
+ } else {
+ unsetExternalConnection();
+ }
+ }
+
+ private void unsetExternalConnection() {
+ Collection<OpenstackRouter> internalRouters = getExternalRouter(false);
+ internalRouters.forEach(r ->
+ getOpenstackRouterInterface(r).forEach(i -> rulePopulator.removeExternalRules(i)));
+ }
+
+ private Collection<OpenstackRouter> getExternalRouter(boolean externalConnection) {
+ List<OpenstackRouter> routers;
+ if (externalConnection) {
+ routers = openstackService.routers()
+ .stream()
+ .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() > 0))
+ .collect(Collectors.toList());
+ } else {
+ routers = openstackService.routers()
+ .stream()
+ .filter(r -> (r.gatewayExternalInfo().externalFixedIps().size() == 0))
+ .collect(Collectors.toList());
+ }
+ return routers;
+ }
+
+ @Override
+ public void deleteRouter(String id) {
+ //TODO : In now, there`s nothing to do for deleteRouter process. It is reserved.
+ }
+
+ @Override
+ public void updateRouterInterface(OpenstackRouterInterface routerInterface) {
+ List<OpenstackRouterInterface> routerInterfaces = Lists.newArrayList();
+ routerInterfaces.add(routerInterface);
+ checkExternalConnection(getOpenstackRouter(routerInterface.id()), routerInterfaces);
+ setL3Connection(getOpenstackRouter(routerInterface.id()), null);
+ routerInterfaceMap.put(routerInterface.portId(), openstackService.port(routerInterface.portId()).networkId());
+ }
+
+ /**
+ * Set flow rules for traffic between two different subnets when more than one subnets
+ * connected to a router.
+ *
+ * @param openstackRouter OpenstackRouter Info
+ * @param openstackPort OpenstackPort Info
+ */
+ private void setL3Connection(OpenstackRouter openstackRouter, OpenstackPort openstackPort) {
+ Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(openstackRouter);
+
+ if (interfaceList.size() < 2) {
+ return;
+ }
+ if (openstackPort == null) {
+ interfaceList.forEach(i -> {
+ OpenstackPort interfacePort = openstackService.port(i.portId());
+ openstackService.ports()
+ .stream()
+ .filter(p -> p.networkId().equals(interfacePort.networkId())
+ && !p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE))
+ .forEach(p -> rulePopulator.populateL3Rules(p,
+ getL3ConnectionList(p.networkId(), interfaceList)));
+
+ });
+ } else {
+ rulePopulator.populateL3Rules(openstackPort, getL3ConnectionList(openstackPort.networkId(), interfaceList));
+ }
+
+ }
+
+ private List<OpenstackRouterInterface> getL3ConnectionList(String networkId,
+ Collection<OpenstackRouterInterface> interfaceList) {
+ List<OpenstackRouterInterface> targetList = Lists.newArrayList();
+ interfaceList.forEach(i -> {
+ OpenstackPort port = openstackService.port(i.portId());
+ if (!port.networkId().equals(networkId)) {
+ targetList.add(i);
+ }
+ });
+ return targetList;
+ }
+
+ @Override
+ public void removeRouterInterface(OpenstackRouterInterface routerInterface) {
+ OpenstackRouter router = openstackService.router(routerInterface.id());
+ Collection<OpenstackRouterInterface> interfaceList = getOpenstackRouterInterface(router);
+ if (interfaceList.size() == 1) {
+ List<OpenstackRouterInterface> newList = Lists.newArrayList();
+ newList.add(routerInterface);
+ interfaceList.forEach(i -> removeL3RulesForRouterInterface(i, router, newList));
+ }
+ removeL3RulesForRouterInterface(routerInterface, router, null);
+ rulePopulator.removeExternalRules(routerInterface);
+ routerInterfaceMap.remove(routerInterface.portId());
+ }
+
+ private void removeL3RulesForRouterInterface(OpenstackRouterInterface routerInterface, OpenstackRouter router,
+ List<OpenstackRouterInterface> newList) {
+ openstackService.ports(routerInterfaceMap.get(routerInterface.portId()).value()).forEach(p -> {
+ Ip4Address vmIp = (Ip4Address) p.fixedIps().values().toArray()[0];
+ if (newList == null) {
+ rulePopulator.removeL3Rules(vmIp,
+ getL3ConnectionList(p.networkId(), getOpenstackRouterInterface(router)));
+ } else {
+ rulePopulator.removeL3Rules(vmIp, newList);
+ }
+ }
+ );
+ }
+
+ @Override
+ public void checkDisassociatedFloatingIp(String portId, OpenstackPortInfo portInfo) {
+ if (floatingIpMap.size() < 1) {
+ log.info("No information in FloatingIpMap");
+ return;
+ }
+ OpenstackFloatingIP floatingIp = associatedFloatingIps()
+ .stream()
+ .filter(fIp -> fIp.portId().equals(portId))
+ .findAny()
+ .orElse(null);
+ if (floatingIp != null && portInfo != null) {
+ l3EventExecutorService.execute(
+ new OpenstackFloatingIPHandler(rulePopulator, floatingIp, false, portInfo));
+ OpenstackFloatingIP.Builder fBuilder = new OpenstackFloatingIP.Builder()
+ .floatingIpAddress(floatingIp.floatingIpAddress())
+ .id(floatingIp.id())
+ .networkId(floatingIp.networkId())
+ .status(floatingIp.status())
+ .tenantId(floatingIp.tenantId());
+ floatingIpMap.replace(floatingIp.id(), fBuilder.build());
+ } else if (portInfo == null) {
+ log.warn("portInfo is null as timing issue between ovs port update event and openstack deletePort event");
+ }
+ }
+
+ @Override
+ public String networkIdForRouterInterface(String portId) {
+ return routerInterfaceMap.get(portId).value();
+ }
+
+ private Collection<OpenstackFloatingIP> associatedFloatingIps() {
+ List<OpenstackFloatingIP> fIps = Lists.newArrayList();
+ floatingIpMap.values()
+ .stream()
+ .filter(fIp -> fIp.value().portId() != null)
+ .forEach(fIp -> fIps.add(fIp.value()));
+ return fIps;
+ }
+
+ private void reloadInitL3Rules() {
+
+ l3EventExecutorService.execute(() ->
+ openstackService.ports()
+ .stream()
+ .forEach(p ->
+ {
+ if (p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)) {
+ updateRouterInterface(portToRouterInterface(p));
+ } else {
+ Optional<Ip4Address> vmIp = p.fixedIps().values().stream().findAny();
+ if (vmIp.isPresent()) {
+ OpenstackFloatingIP floatingIP = getOpenstackFloatingIp(vmIp.get());
+ if (floatingIP != null) {
+ updateFloatingIP(floatingIP);
+ }
+ }
+ }
+ })
+ );
+ }
+
+ private OpenstackRouterInterface portToRouterInterface(OpenstackPort p) {
+ OpenstackRouterInterface.Builder osBuilder = new OpenstackRouterInterface.Builder()
+ .id(checkNotNull(p.deviceId()))
+ .tenantId(checkNotNull(openstackService.network(p.networkId()).tenantId()))
+ .subnetId(checkNotNull(p.fixedIps().keySet().stream().findFirst().orElse(null)).toString())
+ .portId(checkNotNull(p.id()));
+
+ return osBuilder.build();
+ }
+
+ private class InternalPacketProcessor implements PacketProcessor {
+
+ @Override
+ public void process(PacketContext context) {
+
+ if (context.isHandled()) {
+ return;
+ } else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
+ return;
+ }
+
+ InboundPacket pkt = context.inPacket();
+ Ethernet ethernet = pkt.parsed();
+
+ //TODO: Considers IPv6 later.
+ if (ethernet == null) {
+ return;
+ } else if (ethernet.getEtherType() == Ethernet.TYPE_IPV4) {
+ IPv4 iPacket = (IPv4) ethernet.getPayload();
+ switch (iPacket.getProtocol()) {
+ case IPv4.PROTOCOL_ICMP:
+
+ icmpEventExecutorService.execute(() ->
+ openstackIcmpHandler.processIcmpPacket(context, ethernet));
+ break;
+ case IPv4.PROTOCOL_UDP:
+ // don't process DHCP
+ UDP udpPacket = (UDP) iPacket.getPayload();
+ if (udpPacket.getDestinationPort() == UDP.DHCP_SERVER_PORT &&
+ udpPacket.getSourcePort() == UDP.DHCP_CLIENT_PORT) {
+ break;
+ }
+ default:
+ int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
+ DeviceId deviceId = pkt.receivedFrom().deviceId();
+ Port port = null;
+ port = deviceService.getPort(deviceId,
+ gatewayService.getGatewayExternalPorts(deviceId).get(0));
+ if (port != null) {
+ OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
+ Ip4Address.valueOf(iPacket.getSourceAddress()));
+ l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
+ portNum, openstackPort, port, config));
+ } else {
+ log.warn("There`s no external interface");
+ }
+
+ break;
+ }
+ } else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
+ arpEventExecutorService.execute(() ->
+ openstackArpHandler.processArpPacketFromRouter(context, ethernet));
+ }
+ }
+
+ private int getPortNum(MacAddress sourceMac, int destinationAddress) {
+ int portNum = findUnusedPortNum();
+ if (portNum == 0) {
+ clearPortNumMap();
+ portNum = findUnusedPortNum();
+ }
+ tpPortNumMap.put(portNum, sourceMac.toString().concat(COLON).concat(String.valueOf(destinationAddress)));
+ return portNum;
+ }
+
+ private int findUnusedPortNum() {
+ for (int i = TP_PORT_MINIMUM_NUM; i < TP_PORT_MAXIMUM_NUM; i++) {
+ if (!tpPortNumMap.containsKey(i)) {
+ return i;
+ }
+ }
+ return 0;
+ }
+
+ }
+
+ private boolean checkGatewayNode(DeviceId deviceId) {
+ return gatewayService.getGatewayDeviceIds().contains(deviceId);
+ }
+
+ private void clearPortNumMap() {
+ tpPortNumMap.entrySet().forEach(e -> {
+ if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
+ tpPortNumMap.remove(e.getKey());
+ }
+ });
+ }
+
+ private Optional<Port> getExternalPort(DeviceId deviceId, String interfaceName) {
+ return deviceService.getPorts(deviceId)
+ .stream()
+ .filter(p -> p.annotations().value(PORT_NAME).equals(interfaceName))
+ .findAny();
+ }
+
+ private void checkExternalConnection(OpenstackRouter router,
+ Collection<OpenstackRouterInterface> interfaces) {
+ checkNotNull(router, "Router can not be null");
+ checkNotNull(interfaces, "routerInterfaces can not be null");
+ Ip4Address externalIp = router.gatewayExternalInfo().externalFixedIps()
+ .values().stream().findFirst().orElse(null);
+ if ((externalIp == null) || (!router.gatewayExternalInfo().isEnablePnat())) {
+ log.debug("Not satisfied to set pnat configuration");
+ return;
+ }
+ interfaces.forEach(this::initiateL3Rule);
+ }
+
+ private Optional<OpenstackRouter> getRouterfromExternalIp(Ip4Address externalIp) {
+ return getExternalRouter(true)
+ .stream()
+ .filter(r -> r.gatewayExternalInfo()
+ .externalFixedIps()
+ .values()
+ .stream()
+ .findAny()
+ .get()
+ .equals(externalIp))
+ .findAny();
+ }
+
+ private void initiateL3Rule(OpenstackRouterInterface routerInterface) {
+ long vni = Long.parseLong(openstackService.network(openstackService
+ .port(routerInterface.portId()).networkId()).segmentId());
+ rulePopulator.populateExternalRules(vni);
+ }
+
+ private Collection<OpenstackRouterInterface> getOpenstackRouterInterface(OpenstackRouter router) {
+ List<OpenstackRouterInterface> interfaces = Lists.newArrayList();
+ openstackService.ports()
+ .stream()
+ .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
+ && p.deviceId().equals(router.id()))
+ .forEach(p -> interfaces.add(portToRouterInterface(p)));
+ return interfaces;
+ }
+
+ private OpenstackRouter getOpenstackRouter(String id) {
+ return openstackService.routers().stream().filter(r ->
+ r.id().equals(id)).iterator().next();
+ }
+
+ private OpenstackPort getOpenstackPort(MacAddress sourceMac, Ip4Address ip4Address) {
+ OpenstackPort openstackPort = openstackService.ports().stream()
+ .filter(p -> p.macAddress().equals(sourceMac)).iterator().next();
+ return openstackPort.fixedIps().values().stream().filter(ip ->
+ ip.equals(ip4Address)).count() > 0 ? openstackPort : null;
+ }
+
+ private OpenstackFloatingIP getOpenstackFloatingIp(Ip4Address vmIp) {
+ Optional<OpenstackFloatingIP> floatingIp = floatingIpMap.asJavaMap().values().stream()
+ .filter(f -> f.portId() != null && f.fixedIpAddress().equals(vmIp))
+ .findAny();
+
+ if (floatingIp.isPresent()) {
+ return floatingIp.get();
+ }
+ log.debug("There is no floating IP information for VM IP {}", vmIp);
+
+ return null;
+ }
+
+ private void readConfiguration() {
+ config = configService.getConfig("openstacknetworking", OpenstackNetworkingConfig.class);
+ if (config == null) {
+ log.error("No configuration found");
+ return;
+ }
+
+ rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
+ deviceService, driverService, config, gatewayService);
+ openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService,
+ openstackService, config, openstackSwitchingService, gatewayService);
+ openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService,
+ config, gatewayService);
+ openstackIcmpHandler.requestPacket(appId);
+ openstackArpHandler.requestPacket(appId);
+
+ openstackService.floatingIps().stream()
+ .forEach(f -> floatingIpMap.put(f.id(), f));
+
+ reloadInitL3Rules();
+
+ log.info("OpenstackRouting configured");
+ }
+
+ private class InternalConfigListener implements NetworkConfigListener {
+
+ @Override
+ public void event(NetworkConfigEvent event) {
+ if (!event.configClass().equals(OpenstackNetworkingConfig.class)) {
+ return;
+ }
+
+ if (event.type().equals(NetworkConfigEvent.Type.CONFIG_ADDED) ||
+ event.type().equals(NetworkConfigEvent.Type.CONFIG_UPDATED)) {
+ l3EventExecutorService.execute(OpenstackRoutingManager.this::readConfiguration);
+ }
+ }
+ }
+
+ private class InternalDeviceListener implements DeviceListener {
+
+ @Override
+ public void event(DeviceEvent deviceEvent) {
+ if (deviceEvent.type() == DeviceEvent.Type.PORT_UPDATED) {
+ Port port = deviceEvent.port();
+ OpenstackPortInfo portInfo = openstackSwitchingService.openstackPortInfo()
+ .get(port.annotations().value(PORT_NAME));
+ OpenstackPort openstackPort = openstackService.port(port);
+ OpenstackPort interfacePort = openstackService.ports()
+ .stream()
+ .filter(p -> p.deviceOwner().equals(DEVICE_OWNER_ROUTER_INTERFACE)
+ && p.networkId().equals(openstackPort.networkId()))
+ .findAny()
+ .orElse(null);
+ if (portInfo == null && openstackPort == null) {
+ log.warn("As delete event timing issue between routing and switching, Can`t delete L3 rules");
+ return;
+ }
+ if ((port.isEnabled()) && (interfacePort != null)) {
+ OpenstackRouterInterface routerInterface = portToRouterInterface(interfacePort);
+ l3EventExecutorService.execute(() ->
+ setL3Connection(getOpenstackRouter(routerInterface.id()), openstackPort));
+ } else if (interfacePort != null) {
+ OpenstackRouterInterface routerInterface = portToRouterInterface(interfacePort);
+ l3EventExecutorService.execute(() -> rulePopulator.removeL3Rules(portInfo.ip(),
+ getL3ConnectionList(portInfo.networkId(),
+ getOpenstackRouterInterface(getOpenstackRouter(routerInterface.id())))));
+ }
+ }
+ }
+ }
+
+}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
new file mode 100644
index 0000000..57444d0
--- /dev/null
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
@@ -0,0 +1,673 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import com.google.common.collect.Lists;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.TCP;
+import org.onlab.packet.TpPort;
+import org.onlab.packet.UDP;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.GroupId;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DefaultDriverData;
+import org.onosproject.net.driver.DefaultDriverHandler;
+import org.onosproject.net.driver.Driver;
+import org.onosproject.net.driver.DriverHandler;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionPropertyException;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatmentType;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.net.packet.InboundPacket;
+import org.onosproject.openstackinterface.OpenstackInterfaceService;
+import org.onosproject.openstackinterface.OpenstackPort;
+import org.onosproject.openstackinterface.OpenstackRouterInterface;
+import org.onosproject.openstackinterface.OpenstackSubnet;
+import org.onosproject.openstackinterface.OpenstackFloatingIP;
+import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.openstacknetworking.OpenstackPortInfo;
+import org.onosproject.openstacknetworking.OpenstackRoutingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.stream.StreamSupport;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onlab.osgi.DefaultServiceDirectory.getService;
+
+/**
+ * Populates Routing Flow Rules.
+ */
+public class OpenstackRoutingRulePopulator {
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ private final ApplicationId appId;
+ private final FlowObjectiveService flowObjectiveService;
+ private final OpenstackInterfaceService openstackService;
+ private final DeviceService deviceService;
+ private final DriverService driverService;
+ private final OpenstackNetworkingConfig config;
+ private final ScalableGatewayService gatewayService;
+
+ private static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
+ private static final String PORTNAME = "portName";
+ private static final String PORTNAME_PREFIX_VM = "tap";
+
+ private static final String PORTNOTNULL = "Port can not be null";
+ private static final String DEVICENOTNULL = "Device can not be null";
+ private static final String EXTPORTNOTNULL = "External port can not be null";
+ private static final String TUNNEL_DESTINATION = "tunnelDst";
+ private static final int ROUTING_RULE_PRIORITY = 25000;
+ private static final int FLOATING_RULE_PRIORITY = 42000;
+ private static final int PNAT_RULE_PRIORITY = 26000;
+ private static final int PNAT_TIMEOUT = 120;
+ private static final int PREFIX_LENGTH = 32;
+ private static final MacAddress GATEWAYMAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
+
+ private InboundPacket inboundPacket;
+ private OpenstackPort openstackPort;
+ private int portNum;
+ private MacAddress externalInterface;
+ private MacAddress externalRouter;
+
+ /**
+ * The constructor of openstackRoutingRulePopulator.
+ *
+ * @param appId Caller`s appId
+ * @param openstackService Opestack REST request handler
+ * @param flowObjectiveService FlowObjectiveService
+ * @param deviceService DeviceService
+ * @param driverService DriverService
+ * @param config Configuration for openstack environment
+ * @param gatewayService scalable gateway service
+ */
+ public OpenstackRoutingRulePopulator(ApplicationId appId, OpenstackInterfaceService openstackService,
+ FlowObjectiveService flowObjectiveService, DeviceService deviceService,
+ DriverService driverService, OpenstackNetworkingConfig config,
+ ScalableGatewayService gatewayService) {
+ this.appId = appId;
+ this.flowObjectiveService = flowObjectiveService;
+ this.openstackService = checkNotNull(openstackService);
+ this.deviceService = deviceService;
+ this.driverService = driverService;
+ this.config = config;
+ this.gatewayService = gatewayService;
+ }
+
+ /**
+ * Populates flow rules for Pnat configurations.
+ *
+ * @param inboundPacket Packet-in event packet
+ * @param openstackPort Target VM information
+ * @param portNum Pnat port number
+ * @param externalIp external ip address
+ * @param externalInterfaceMacAddress Gateway external interface macaddress
+ * @param externalRouterMacAddress Outer(physical) router`s macaddress
+ */
+ public void populatePnatFlowRules(InboundPacket inboundPacket, OpenstackPort openstackPort, int portNum,
+ Ip4Address externalIp, MacAddress externalInterfaceMacAddress,
+ MacAddress externalRouterMacAddress) {
+ this.inboundPacket = inboundPacket;
+ this.openstackPort = openstackPort;
+ this.portNum = portNum;
+ this.externalInterface = externalInterfaceMacAddress;
+ this.externalRouter = externalRouterMacAddress;
+
+ long vni = getVni(openstackPort.networkId());
+
+ populatePnatIncomingFlowRules(vni, externalIp);
+ populatePnatOutgoingFlowRules(vni, externalIp);
+ }
+
+ private void populatePnatOutgoingFlowRules(long vni, Ip4Address externalIp) {
+ IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(iPacket.getProtocol())
+ .matchTunnelId(vni)
+ .matchIPSrc(IpPrefix.valueOf(iPacket.getSourceAddress(), 32))
+ .matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ switch (iPacket.getProtocol()) {
+ case IPv4.PROTOCOL_TCP:
+ TCP tcpPacket = (TCP) iPacket.getPayload();
+ sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getSourcePort()))
+ .matchTcpDst(TpPort.tpPort(tcpPacket.getDestinationPort()));
+ tBuilder.setTcpSrc(TpPort.tpPort(portNum));
+ break;
+ case IPv4.PROTOCOL_UDP:
+ UDP udpPacket = (UDP) iPacket.getPayload();
+ sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getSourcePort()))
+ .matchUdpDst(TpPort.tpPort(udpPacket.getDestinationPort()));
+ tBuilder.setUdpSrc(TpPort.tpPort(portNum));
+ break;
+ default:
+ log.debug("Unsupported IPv4 protocol {}");
+ break;
+ }
+
+ tBuilder.setIpSrc(externalIp);
+ gatewayService.getGatewayNodes().forEach(node -> {
+ tBuilder.setOutput(gatewayService.getGatewayExternalPorts(node.getGatewayDeviceId()).get(0));
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(PNAT_RULE_PRIORITY)
+ .makeTemporary(PNAT_TIMEOUT)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(node.getGatewayDeviceId(), fo);
+ });
+
+ }
+
+ private Port getPortOfExternalInterface() {
+ return deviceService.getPorts(getGatewayNode().id()).stream()
+ .filter(p -> p.annotations().value(PORTNAME).equals(config.gatewayExternalInterfaceName()))
+ .findAny().orElse(null);
+ }
+
+
+ private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp) {
+ IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPProtocol(iPacket.getProtocol())
+ .matchIPDst(IpPrefix.valueOf(externalIp, 32))
+ .matchIPSrc(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
+
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+ tBuilder.setTunnelId(vni)
+ .setEthDst(inboundPacket.parsed().getSourceMAC())
+ .setIpDst(IpAddress.valueOf(iPacket.getSourceAddress()));
+
+ switch (iPacket.getProtocol()) {
+ case IPv4.PROTOCOL_TCP:
+ TCP tcpPacket = (TCP) iPacket.getPayload();
+ sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getDestinationPort()))
+ .matchTcpDst(TpPort.tpPort(portNum));
+ tBuilder.setTcpDst(TpPort.tpPort(tcpPacket.getSourcePort()));
+ break;
+ case IPv4.PROTOCOL_UDP:
+ UDP udpPacket = (UDP) iPacket.getPayload();
+ sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getDestinationPort()))
+ .matchUdpDst(TpPort.tpPort(portNum));
+ tBuilder.setUdpDst(TpPort.tpPort(udpPacket.getSourcePort()));
+ break;
+ default:
+ break;
+ }
+
+ getGatewayNodeList().forEach(node -> {
+ DeviceId deviceId = node.id();
+ tBuilder.extension(buildNiciraExtenstion(deviceId, getHostIpfromOpenstackPort(openstackPort)), deviceId)
+ .setOutput(getTunnelPort(deviceId));
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(PNAT_RULE_PRIORITY)
+ .makeTemporary(PNAT_TIMEOUT)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ });
+ }
+
+ private List<Device> getGatewayNodeList() {
+ List<Device> devices = Lists.newArrayList();
+ gatewayService.getGatewayDeviceIds().forEach(deviceId ->
+ devices.add(checkNotNull(deviceService.getDevice(deviceId))));
+ return devices;
+ }
+
+ private Ip4Address getHostIpfromOpenstackPort(OpenstackPort openstackPort) {
+ Device device = getDevicefromOpenstackPort(openstackPort);
+ return config.nodes().get(device.id());
+ }
+
+ private Device getDevicefromOpenstackPort(OpenstackPort openstackPort) {
+ String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11);
+ Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false)
+ .filter(d -> findPortinDevice(d.id(), openstackPortName))
+ .iterator()
+ .next();
+ checkNotNull(device, DEVICENOTNULL);
+ return device;
+ }
+
+ private boolean findPortinDevice(DeviceId deviceId, String openstackPortName) {
+ Port port = deviceService.getPorts(deviceId)
+ .stream()
+ .filter(p -> p.isEnabled() && p.annotations().value(PORTNAME).equals(openstackPortName))
+ .findAny()
+ .orElse(null);
+ return port != null;
+ }
+
+ /**
+ * Builds Nicira extension for tagging remoteIp of vxlan.
+ *
+ * @param deviceId Device Id of vxlan source device
+ * @param hostIp Remote Ip of vxlan destination device
+ * @return NiciraExtension Treatment
+ */
+ public ExtensionTreatment buildNiciraExtenstion(DeviceId deviceId, Ip4Address hostIp) {
+ Driver driver = driverService.getDriver(deviceId);
+ DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, deviceId));
+ ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class);
+
+ ExtensionTreatment extensionInstruction =
+ resolver.getExtensionInstruction(
+ ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST.type());
+
+ try {
+ extensionInstruction.setPropertyValue(TUNNEL_DESTINATION, hostIp);
+ } catch (ExtensionPropertyException e) {
+ log.error("Error setting Nicira extension setting {}", e);
+ }
+
+ return extensionInstruction;
+ }
+
+ /**
+ * Returns port number of vxlan tunnel.
+ *
+ * @param deviceId Target Device Id
+ * @return PortNumber
+ */
+ public PortNumber getTunnelPort(DeviceId deviceId) {
+ Port port = deviceService.getPorts(deviceId).stream()
+ .filter(p -> p.annotations().value(PORTNAME).equals(PORTNAME_PREFIX_TUNNEL))
+ .findAny().orElse(null);
+
+ if (port == null) {
+ log.error("No TunnelPort was created.");
+ return null;
+ }
+ return port.number();
+
+ }
+
+ /**
+ * Populates flow rules from openstackComputeNode to GatewayNode.
+ *
+ * @param vni Target network
+ */
+ public void populateExternalRules(long vni) {
+
+ // 1. computeNode to gateway
+ populateComputeNodeRules(vni);
+ // 2. gatewayNode to controller
+ populateRuleGatewaytoController(vni);
+ }
+
+ private void populateRuleGatewaytoController(long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchEthDst(GATEWAYMAC);
+ tBuilder.setOutput(PortNumber.CONTROLLER);
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ getGatewayNodeList().forEach(device -> flowObjectiveService.forward(device.id(), fo));
+ }
+
+ private void populateComputeNodeRules(long vni) {
+ StreamSupport.stream(deviceService.getDevices().spliterator(), false)
+ .filter(d -> !checkGatewayNode(d.id()))
+ .forEach(d -> populateRuleToGatewayBySgw(d.id(),
+ gatewayService.getGroupIdForGatewayLoadBalance(d.id()), vni));
+ }
+
+ private void populateRuleToGatewayBySgw(DeviceId deviceId, GroupId groupId, long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchEthDst(GATEWAYMAC);
+ tBuilder.group(groupId);
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private void populateRuleToGateway(DeviceId deviceId, Device gatewayDevice, long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchEthDst(GATEWAYMAC);
+ tBuilder.extension(buildNiciraExtenstion(deviceId, config.nodes().get(gatewayDevice.id())), deviceId)
+ .setOutput(getTunnelPort(deviceId));
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private Device getGatewayNode() {
+ return checkNotNull(deviceService.getDevice(DeviceId.deviceId(config.gatewayBridgeId())));
+ }
+
+ private boolean checkGatewayNode(DeviceId deviceId) {
+ return gatewayService.getGatewayDeviceIds().stream().anyMatch(dId -> dId.equals(deviceId));
+ }
+
+ private long getVni(String netId) {
+ return Long.parseLong(openstackService.network(netId).segmentId());
+ }
+
+ /**
+ * Remove flow rules for external connection.
+ *
+ * @param routerInterface Corresponding routerInterface
+ */
+ public void removeExternalRules(OpenstackRouterInterface routerInterface) {
+ OpenstackSubnet openstackSubnet = openstackService.subnet(routerInterface.subnetId());
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(getVni(openstackSubnet.networkId()))
+ .matchEthDst(GATEWAYMAC);
+
+ StreamSupport.stream(deviceService.getDevices().spliterator(), false)
+ .forEach(d -> {
+ ForwardingObjective.Flag flag = checkGatewayNode(d.id()) ?
+ ForwardingObjective.Flag.VERSATILE :
+ ForwardingObjective.Flag.SPECIFIC;
+ removeRule(d.id(), sBuilder, flag, ROUTING_RULE_PRIORITY);
+ });
+
+ }
+
+ private void removeRule(DeviceId deviceId, TrafficSelector.Builder sBuilder,
+ ForwardingObjective.Flag flag, int priority) {
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(flag)
+ .withPriority(priority)
+ .fromApp(appId)
+ .remove();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ /**
+ * Populates flow rules for floatingIp configuration.
+ *
+ * @param floatingIP Corresponding floating ip information
+ */
+ public void populateFloatingIpRules(OpenstackFloatingIP floatingIP) {
+ OpenstackPort port = openstackService.port(floatingIP.portId());
+ //1. incoming rules
+ populateFloatingIpIncomingRules(floatingIP, port);
+ //2. outgoing rules
+ populateFloatingIpOutgoingRules(floatingIP, port);
+ }
+
+ private void populateFloatingIpIncomingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
+ DeviceId portDeviceId = getDevicefromOpenstackPort(port).id();
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
+
+ getGatewayNodeList().forEach(device -> {
+ DeviceId deviceId = device.id();
+ tBuilder.setEthSrc(GATEWAYMAC)
+ .setEthDst(port.macAddress())
+ .setIpDst(floatingIP.fixedIpAddress())
+ .setTunnelId(getVni(port.networkId()))
+ .extension(buildNiciraExtenstion(deviceId, config.nodes().get(portDeviceId)), deviceId)
+ .setOutput(getTunnelPort(deviceId));
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(FLOATING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ });
+ }
+
+ private void populateFloatingIpOutgoingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(getVni(port.networkId()))
+ .matchIPSrc(IpPrefix.valueOf(floatingIP.fixedIpAddress(), 32));
+
+ getGatewayNodeList().forEach(device -> {
+ DeviceId deviceId = device.id();
+ tBuilder.setIpSrc(floatingIP.floatingIpAddress())
+ .setOutput(getExternalPortNum(deviceId));
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withFlag(ForwardingObjective.Flag.VERSATILE)
+ .withPriority(FLOATING_RULE_PRIORITY)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ });
+ }
+
+ private PortNumber getExternalPortNum(DeviceId deviceId) {
+ return checkNotNull(gatewayService.getGatewayExternalPorts(deviceId).get(0), PORTNOTNULL);
+ }
+
+ /**
+ * Removes flow rules for floating ip configuration.
+ *
+ * @param floatingIP Corresponding floating ip information
+ * @param portInfo stored information about deleted vm
+ */
+ public void removeFloatingIpRules(OpenstackFloatingIP floatingIP, OpenstackPortInfo portInfo) {
+ TrafficSelector.Builder sOutgoingBuilder = DefaultTrafficSelector.builder();
+ TrafficSelector.Builder sIncomingBuilder = DefaultTrafficSelector.builder();
+
+ sOutgoingBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(portInfo.vni())
+ .matchIPSrc(IpPrefix.valueOf(portInfo.ip(), PREFIX_LENGTH));
+
+ sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
+
+ getGatewayNodeList().forEach(device -> {
+ removeRule(device.id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
+ removeRule(device.id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
+ });
+ }
+
+ /**
+ * Populates L3 rules for east to west traffic.
+ *
+ * @param openstackPort target VM
+ * @param targetList target openstackRouterInterfaces
+ */
+ public void populateL3Rules(OpenstackPort openstackPort, List<OpenstackRouterInterface> targetList) {
+ Device device = getDevicefromOpenstackPort(openstackPort);
+ Port port = getPortFromOpenstackPort(device, openstackPort);
+ Ip4Address vmIp = openstackPort.fixedIps().values().iterator().next();
+
+ if (port == null) {
+ return;
+ }
+
+ targetList.forEach(routerInterface -> {
+ long vni = getVni(openstackService.port(routerInterface.portId()).networkId());
+
+ if (vmIp == null) {
+ return;
+ }
+
+ populateL3RulestoSameNode(vmIp, openstackPort, port, device, vni);
+
+ deviceService.getAvailableDevices().forEach(d -> {
+ if (!d.equals(device) && !checkGatewayNode(d.id())) {
+ populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(openstackPort));
+ }
+ });
+
+ });
+ }
+
+ private void populateL3RulestoDifferentNode(Ip4Address vmIp, long vni, DeviceId deviceId, Ip4Address hostIp) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchTunnelId(vni)
+ .matchIPDst(vmIp.toIpPrefix());
+ tBuilder.extension(buildNiciraExtenstion(deviceId, hostIp), deviceId)
+ .setOutput(getTunnelPort(deviceId));
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(deviceId, fo);
+ }
+
+ private void populateL3RulestoSameNode(Ip4Address vmIp, OpenstackPort p, Port port, Device device, long vni) {
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+ TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(vmIp.toIpPrefix())
+ .matchTunnelId(vni);
+
+ tBuilder.setEthDst(p.macAddress())
+ .setOutput(port.number());
+
+ ForwardingObjective fo = DefaultForwardingObjective.builder()
+ .withSelector(sBuilder.build())
+ .withTreatment(tBuilder.build())
+ .withPriority(ROUTING_RULE_PRIORITY)
+ .withFlag(ForwardingObjective.Flag.SPECIFIC)
+ .fromApp(appId)
+ .add();
+
+ flowObjectiveService.forward(device.id(), fo);
+ }
+
+ private Port getPortFromOpenstackPort(Device device, OpenstackPort p) {
+ String openstackPortName = PORTNAME_PREFIX_VM + p.id().substring(0, 11);
+ return deviceService.getPorts(device.id())
+ .stream()
+ .filter(pt -> pt.annotations().value(PORTNAME).equals(openstackPortName))
+ .findAny()
+ .orElse(null);
+ }
+
+ /**
+ * Removes L3 rules for routerInterface events.
+ *
+ * @param vmIp Corresponding Vm ip
+ * @param routerInterfaces Corresponding routerInterfaces
+ */
+ public void removeL3Rules(Ip4Address vmIp, List<OpenstackRouterInterface> routerInterfaces) {
+ if (vmIp == null) {
+ return;
+ }
+
+ OpenstackRoutingService routingService = getService(OpenstackRoutingService.class);
+
+ deviceService.getAvailableDevices().forEach(d -> {
+ if (!checkGatewayNode(d.id())) {
+ routerInterfaces.forEach(routerInterface -> {
+ String networkId = routingService.networkIdForRouterInterface(routerInterface.portId());
+ long vni = getVni(networkId);
+
+ TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+
+ sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+ .matchIPDst(vmIp.toIpPrefix())
+ .matchTunnelId(vni);
+
+ removeRule(d.id(), sBuilder, ForwardingObjective.Flag.SPECIFIC, ROUTING_RULE_PRIORITY);
+ });
+ }
+ });
+ }
+}
diff --git a/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java
new file mode 100644
index 0000000..78d05a9
--- /dev/null
+++ b/apps/openstacknetworking/routing/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2016-present Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Application for OpenstackRouting.
+ */
+package org.onosproject.openstacknetworking.routing;
\ No newline at end of file