Refactoring of OpenstackSwitching and OpenstackRouting

Change-Id: Ib7caea98006274dcdfebfe27c07e3533730ab23e
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIPHandler.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIPHandler.java
new file mode 100644
index 0000000..3ef1c88
--- /dev/null
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackFloatingIPHandler.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2016 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import org.onosproject.event.AbstractEvent;
+
+/**
+ * Handle FloatingIP Event for Managing Flow Rules In Openstack Nodes.
+ */
+public class OpenstackFloatingIPHandler implements Runnable {
+
+    volatile AbstractEvent event;
+    OpenstackFloatingIPHandler(AbstractEvent event) {
+        this.event = event;
+    }
+
+    @Override
+    public void run() {
+
+    }
+}
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
new file mode 100644
index 0000000..3cc11f0
--- /dev/null
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackIcmpHandler.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2016 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import org.onosproject.net.packet.PacketContext;
+
+/**
+ * Handle ICMP packet processing for Managing Flow Rules In Openstack Nodes.
+ */
+public class OpenstackIcmpHandler implements Runnable {
+
+    volatile PacketContext context;
+    private OpenstackRoutingRulePopulator rulePopulator;
+    OpenstackIcmpHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context) {
+        this.context = context;
+        this.rulePopulator = rulePopulator;
+    }
+
+    @Override
+    public void run() {
+    }
+}
\ No newline at end of file
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
new file mode 100644
index 0000000..40eed7f
--- /dev/null
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackPnatHandler.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright 2016 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.TCP;
+import org.onlab.packet.UDP;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.packet.DefaultOutboundPacket;
+import org.onosproject.net.packet.InboundPacket;
+import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketService;
+import org.onosproject.openstacknetworking.OpenstackPort;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+
+/**
+ * Handle NAT packet processing for Managing Flow Rules In Openstack Nodes.
+ */
+public class OpenstackPnatHandler implements Runnable {
+
+    volatile PacketContext context;
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
+    protected PacketService packetService;
+
+    private final OpenstackRoutingRulePopulator rulePopulator;
+    private final int portNum;
+    private final OpenstackPort openstackPort;
+
+    OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
+                         int portNum, OpenstackPort openstackPort) {
+        this.rulePopulator = checkNotNull(rulePopulator);
+        this.context = checkNotNull(context);
+        this.portNum = checkNotNull(portNum);
+        this.openstackPort = checkNotNull(openstackPort);
+    }
+
+    @Override
+    public void run() {
+        InboundPacket inboundPacket = context.inPacket();
+        Ethernet ethernet = checkNotNull(inboundPacket.parsed());
+
+        //TODO: Considers IPV6
+        if (ethernet.getEtherType() != Ethernet.TYPE_IPV4) {
+            log.warn("Now, we just consider IP version 4");
+            return;
+        }
+
+        packetOut(inboundPacket, portNum);
+
+        rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
+                getExternalInterfaceMacAddress(), getExternalRouterMacAddress());
+    }
+
+    private void packetOut(InboundPacket inboundPacket, int portNum) {
+        Ethernet ethernet = checkNotNull(inboundPacket.parsed());
+        IPv4 iPacket = (IPv4) ethernet.getPayload();
+
+        TrafficTreatment.Builder treatment = DefaultTrafficTreatment.builder();
+
+        switch (iPacket.getProtocol()) {
+            case IPv4.PROTOCOL_TCP:
+                TCP tcpPacket = (TCP) iPacket.getPayload();
+                tcpPacket.setSourcePort(portNum);
+                tcpPacket.resetChecksum();
+                tcpPacket.setParent(iPacket);
+                iPacket.setPayload(tcpPacket);
+                break;
+            case IPv4.PROTOCOL_UDP:
+                UDP udpPacket = (UDP) iPacket.getPayload();
+                udpPacket.setSourcePort(portNum);
+                udpPacket.resetChecksum();
+                udpPacket.setParent(iPacket);
+                iPacket.setPayload(udpPacket);
+                break;
+            default:
+                break;
+        }
+
+        iPacket.resetChecksum();
+        iPacket.setPayload(ethernet);
+        ethernet.setSourceMACAddress(getExternalInterfaceMacAddress())
+                .setDestinationMACAddress(getExternalRouterMacAddress());
+        ethernet.resetChecksum();
+
+        treatment.setOutput(getExternalPort(inboundPacket.receivedFrom().deviceId()));
+
+        packetService.emit(new DefaultOutboundPacket(inboundPacket.receivedFrom().deviceId(),
+                treatment.build(), ByteBuffer.wrap(ethernet.serialize())));
+    }
+
+    private PortNumber getExternalPort(DeviceId deviceId) {
+        // TODO
+        return null;
+    }
+    private MacAddress getExternalInterfaceMacAddress() {
+        // TODO
+        return null;
+    }
+    private MacAddress getExternalRouterMacAddress() {
+        // TODO
+        return null;
+    }
+}
\ No newline at end of file
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
new file mode 100644
index 0000000..2ea5935
--- /dev/null
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingManager.java
@@ -0,0 +1,244 @@
+/*
+ * Copyright 2016 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.MacAddress;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.CoreService;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.packet.InboundPacket;
+import org.onosproject.net.packet.PacketContext;
+import org.onosproject.net.packet.PacketProcessor;
+import org.onosproject.net.packet.PacketService;
+import org.onosproject.openstacknetworking.OpenstackFloatingIP;
+import org.onosproject.openstacknetworking.OpenstackNetworkingService;
+import org.onosproject.openstacknetworking.OpenstackPort;
+import org.onosproject.openstacknetworking.OpenstackRouter;
+import org.onosproject.openstacknetworking.OpenstackRouterInterface;
+import org.onosproject.openstacknetworking.OpenstackRoutingService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onlab.util.Tools.groupedThreads;
+
+@Service
+@Component(immediate = true)
+/**
+ * Populates flow rules about L3 functionality for VMs in Openstack.
+ */
+public class OpenstackRoutingManager implements OpenstackRoutingService {
+    private final Logger log = LoggerFactory
+            .getLogger(getClass());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected PacketService packetService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected OpenstackNetworkingService openstackService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected FlowObjectiveService flowObjectiveService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DriverService driverService;
+
+    private ApplicationId appId;
+    private Map<String, OpenstackRouterInterface> routerInterfaceMap = Maps.newHashMap();
+    private Map<Integer, String> portNumMap = initPortNumMap();
+    private static final String APP_ID = "org.onosproject.openstackrouting";
+    private Map<Integer, String> initPortNumMap() {
+        Map<Integer, String> map = Maps.newHashMap();
+        for (int i = 1024; i < 65535; i++) {
+            map.put(i, "");
+        }
+        return map;
+    }
+
+    private InternalPacketProcessor internalPacketProcessor = new InternalPacketProcessor();
+    private ExecutorService l3EventExecutorService =
+            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "L3-event"));
+    private ExecutorService icmpEventExecutorService =
+            Executors.newSingleThreadExecutor(groupedThreads("onos/openstackrouting", "icmp-event"));
+
+    @Activate
+    protected void activate() {
+        appId = coreService.registerApplication(APP_ID);
+        packetService.addProcessor(internalPacketProcessor, PacketProcessor.director(1));
+
+        log.info("onos-openstackrouting started");
+    }
+
+    @Deactivate
+    protected void deactivate() {
+        packetService.removeProcessor(internalPacketProcessor);
+        log.info("onos-openstackrouting stopped");
+    }
+
+
+    @Override
+    public void createFloatingIP(OpenstackFloatingIP openstackFloatingIP) {
+
+    }
+
+    @Override
+    public void updateFloatingIP(OpenstackFloatingIP openstackFloatingIP) {
+
+    }
+
+    @Override
+    public void deleteFloatingIP(String id) {
+
+    }
+
+    @Override
+    public void createRouter(OpenstackRouter openstackRouter) {
+        checkExternalConnection(openstackRouter, getOpenstackRouterInterface(openstackRouter));
+    }
+
+    @Override
+    public void updateRouter(OpenstackRouter openstackRouter) {
+        checkExternalConnection(openstackRouter, getOpenstackRouterInterface(openstackRouter));
+    }
+
+    @Override
+    public void deleteRouter(String id) {
+        //TODO
+    }
+
+    @Override
+    public void updateRouterInterface(OpenstackRouterInterface routerInterface) {
+        routerInterfaceMap.putIfAbsent(routerInterface.portId(), routerInterface);
+        List<OpenstackRouterInterface> routerInterfaces = Lists.newArrayList();
+        routerInterfaces.add(routerInterface);
+        checkExternalConnection(getOpenstackRouter(routerInterface.tenantId()), routerInterfaces);
+    }
+
+    @Override
+    public void removeRouterInterface(OpenstackRouterInterface routerInterface) {
+        OpenstackRoutingRulePopulator rulePopulator = new OpenstackRoutingRulePopulator(appId,
+                openstackService, flowObjectiveService, deviceService, driverService);
+        rulePopulator.removeExternalRules(routerInterface);
+        routerInterfaceMap.remove(routerInterface.portId());
+    }
+    private class InternalPacketProcessor implements PacketProcessor {
+
+        @Override
+        public void process(PacketContext context) {
+
+            if (context.isHandled()) {
+                return;
+            }
+
+            InboundPacket pkt = context.inPacket();
+            Ethernet ethernet = pkt.parsed();
+
+            if (ethernet != null && ethernet.getEtherType() == Ethernet.TYPE_IPV4) {
+                IPv4 iPacket = (IPv4) ethernet.getPayload();
+                OpenstackRoutingRulePopulator rulePopulator = new OpenstackRoutingRulePopulator(appId,
+                        openstackService, flowObjectiveService, deviceService,
+                        driverService);
+                switch (iPacket.getProtocol()) {
+                    case IPv4.PROTOCOL_ICMP:
+                        icmpEventExecutorService.execute(new OpenstackIcmpHandler(rulePopulator, context));
+                        break;
+                    default:
+                        int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
+                        OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
+                                Ip4Address.valueOf(iPacket.getSourceAddress()));
+                        l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
+                                portNum, openstackPort));
+                        break;
+                }
+
+            }
+        }
+
+        private int getPortNum(MacAddress sourceMac, int destinationAddress) {
+            int portNum = portNumMap.keySet().stream()
+                    .filter(k -> portNumMap.get(k).equals("")).findFirst().orElse(0);
+            portNumMap.replace(portNum, sourceMac.toString().concat(":").concat(String.valueOf(destinationAddress)));
+            return portNum;
+        }
+    }
+
+    private void checkExternalConnection(OpenstackRouter router,
+                                         Collection<OpenstackRouterInterface> routerInterfaces) {
+        checkNotNull(router, "Router can not be null");
+        checkNotNull(routerInterfaces, "RouterInterfaces can not be null");
+        Ip4Address externalIp = router.gatewayExternalInfo().externalFixedIps()
+                .values().stream().findFirst().orElse(null);
+        if ((externalIp == null) || (!router.gatewayExternalInfo().isEnablePnat())) {
+            log.debug("Failed to set pnat configuration");
+            return;
+        }
+        routerInterfaces.forEach(routerInterface -> {
+            initiateL3Rule(router, routerInterface);
+        });
+    }
+
+    private void initiateL3Rule(OpenstackRouter router, OpenstackRouterInterface routerInterface) {
+        long vni = Long.parseLong(openstackService.network(openstackService
+                .port(routerInterface.portId()).networkId()).segmentId());
+        OpenstackRoutingRulePopulator rulePopulator = new OpenstackRoutingRulePopulator(appId,
+                openstackService, flowObjectiveService, deviceService, driverService);
+        rulePopulator.populateExternalRules(vni, router, routerInterface);
+    }
+
+    private Collection<OpenstackRouterInterface> getOpenstackRouterInterface(OpenstackRouter router) {
+        return routerInterfaceMap.values().stream().filter(i -> i.id().equals(router.id()))
+                .collect(Collectors.toList());
+    }
+
+    private OpenstackRouter getOpenstackRouter(String tenantId) {
+        return openstackService.routers().stream().filter(r ->
+                r.tenantId().equals(tenantId)).findFirst().orElse(null);
+    }
+
+    private OpenstackPort getOpenstackPort(MacAddress sourceMac, Ip4Address ip4Address) {
+        OpenstackPort openstackPort = openstackService.ports("").stream()
+                .filter(p -> p.macAddress().equals(sourceMac)).findFirst().orElse(null);
+        return openstackPort.fixedIps().values().stream().findFirst().orElse(null)
+                .equals(ip4Address) ? openstackPort : null;
+    }
+
+}
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
new file mode 100644
index 0000000..dc10216
--- /dev/null
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/OpenstackRoutingRulePopulator.java
@@ -0,0 +1,371 @@
+/*
+ * Copyright 2016 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstacknetworking.routing;
+
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.Ip4Address;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.TCP;
+import org.onlab.packet.TpPort;
+import org.onlab.packet.UDP;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.net.Device;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.Port;
+import org.onosproject.net.PortNumber;
+import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.driver.DefaultDriverData;
+import org.onosproject.net.driver.DefaultDriverHandler;
+import org.onosproject.net.driver.Driver;
+import org.onosproject.net.driver.DriverHandler;
+import org.onosproject.net.driver.DriverService;
+import org.onosproject.net.flow.DefaultTrafficSelector;
+import org.onosproject.net.flow.DefaultTrafficTreatment;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.instructions.ExtensionPropertyException;
+import org.onosproject.net.flow.instructions.ExtensionTreatment;
+import org.onosproject.net.flow.instructions.ExtensionTreatmentType;
+import org.onosproject.net.flowobjective.DefaultForwardingObjective;
+import org.onosproject.net.flowobjective.FlowObjectiveService;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.net.packet.InboundPacket;
+import org.onosproject.openstacknetworking.OpenstackNetworkingService;
+import org.onosproject.openstacknetworking.OpenstackPort;
+import org.onosproject.openstacknetworking.OpenstackRouter;
+import org.onosproject.openstacknetworking.OpenstackRouterInterface;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.stream.StreamSupport;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Populates Routing Flow Rules.
+ */
+public class OpenstackRoutingRulePopulator {
+
+    private static Logger log = LoggerFactory
+            .getLogger(OpenstackRoutingRulePopulator.class);
+    private ApplicationId appId;
+    private FlowObjectiveService flowObjectiveService;
+    private OpenstackNetworkingService openstackService;
+    private DeviceService deviceService;
+    private DriverService driverService;
+
+    public static final String PORTNAME_PREFIX_VM = "tap";
+    public static final String PORTNAME_PREFIX_ROUTER = "qr";
+    public static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
+    public static final String PORTNAME = "portName";
+
+    private static final int ROUTING_RULE_PRIORITY = 25000;
+    private static final int PNAT_RULE_PRIORITY = 24000;
+    private static final int PNAT_TIMEOUT = 120;
+    private static final MacAddress GATEWAYMAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
+
+    private InboundPacket inboundPacket;
+    private OpenstackPort openstackPort;
+    private int portNum;
+    private MacAddress externalInterface;
+    private MacAddress externalRouter;
+    private OpenstackRouter router;
+    private OpenstackRouterInterface routerInterface;
+
+    // TODO: This will be replaced to get the information from openstackswitchingservice.
+    private static final String EXTERNAL_INTERFACE_NAME = "eth3";
+
+    public OpenstackRoutingRulePopulator(ApplicationId appId, OpenstackNetworkingService openstackService,
+                                         FlowObjectiveService flowObjectiveService,
+                                         DeviceService deviceService, DriverService driverService) {
+        this.appId = appId;
+        this.flowObjectiveService = flowObjectiveService;
+        this.openstackService = openstackService;
+        this.deviceService = deviceService;
+        this.driverService = driverService;
+    }
+
+    public void populatePnatFlowRules(InboundPacket inboundPacket, OpenstackPort openstackPort, int portNum,
+                                      MacAddress externalInterfaceMacAddress, MacAddress externalRouterMacAddress) {
+        this.inboundPacket = inboundPacket;
+        this.openstackPort = openstackPort;
+        this.portNum = portNum;
+        this.externalInterface = externalInterfaceMacAddress;
+        this.externalRouter = externalRouterMacAddress;
+
+        long vni = getVni(openstackPort);
+
+        populatePnatIncomingFlowRules(vni);
+        populatePnatOutgoingFlowRules(vni);
+    }
+
+    private void populatePnatOutgoingFlowRules(long vni) {
+        IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
+
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPProtocol(iPacket.getProtocol())
+                .matchTunnelId(vni)
+                .matchIPSrc(IpPrefix.valueOf(iPacket.getSourceAddress(), 32))
+                .matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
+
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+        tBuilder.setEthSrc(externalInterface)
+                .setEthDst(externalRouter);
+
+        switch (iPacket.getProtocol()) {
+            case IPv4.PROTOCOL_TCP:
+                TCP tcpPacket = (TCP) iPacket.getPayload();
+                sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getSourcePort()))
+                        .matchTcpDst(TpPort.tpPort(tcpPacket.getDestinationPort()));
+                tBuilder.setTcpDst(TpPort.tpPort(portNum));
+                break;
+            case IPv4.PROTOCOL_UDP:
+                UDP udpPacket = (UDP) iPacket.getPayload();
+                sBuilder.matchUdpDst(TpPort.tpPort(udpPacket.getSourcePort()))
+                        .matchUdpDst(TpPort.tpPort(udpPacket.getDestinationPort()));
+                tBuilder.setUdpDst(TpPort.tpPort(portNum));
+                break;
+            default:
+                break;
+        }
+
+        Port port = getPortNumOfExternalInterface();
+        checkNotNull(port, "Port can not be null");
+        tBuilder.setOutput(port.number());
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withFlag(ForwardingObjective.Flag.VERSATILE)
+                .withPriority(PNAT_RULE_PRIORITY)
+                .makeTemporary(PNAT_TIMEOUT)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
+    }
+
+    private Port getPortNumOfExternalInterface() {
+        return deviceService.getPorts(inboundPacket.receivedFrom().deviceId()).stream()
+                .filter(p -> p.annotations().value("portName").equals(EXTERNAL_INTERFACE_NAME))
+                .findAny().orElse(null);
+    }
+
+
+    private void populatePnatIncomingFlowRules(long vni) {
+        IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
+        DeviceId deviceId = inboundPacket.receivedFrom().deviceId();
+
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPProtocol(iPacket.getProtocol())
+                .matchIPSrc(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
+
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+        tBuilder.setTunnelId(vni)
+                .setIpDst(IpAddress.valueOf(iPacket.getSourceAddress()));
+
+        switch (iPacket.getProtocol()) {
+            case IPv4.PROTOCOL_TCP:
+                TCP tcpPacket = (TCP) iPacket.getPayload();
+                sBuilder.matchTcpSrc(TpPort.tpPort(tcpPacket.getDestinationPort()))
+                        .matchTcpDst(TpPort.tpPort(portNum));
+                tBuilder.setTcpDst(TpPort.tpPort(tcpPacket.getSourcePort()));
+                break;
+            case IPv4.PROTOCOL_UDP:
+                UDP udpPacket = (UDP) iPacket.getPayload();
+                sBuilder.matchUdpSrc(TpPort.tpPort(udpPacket.getDestinationPort()))
+                        .matchUdpDst(TpPort.tpPort(portNum));
+                tBuilder.setUdpDst(TpPort.tpPort(udpPacket.getSourcePort()));
+                break;
+            default:
+                break;
+        }
+
+        tBuilder.extension(buildNiciraExtenstion(deviceId, Ip4Address.valueOf(iPacket.getSourceAddress())), deviceId)
+                .setOutput(getTunnelPort(deviceId));
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withFlag(ForwardingObjective.Flag.VERSATILE)
+                .withPriority(PNAT_RULE_PRIORITY)
+                .makeTemporary(PNAT_TIMEOUT)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo);
+    }
+
+    private ExtensionTreatment buildNiciraExtenstion(DeviceId id, Ip4Address hostIp) {
+        Driver driver = driverService.getDriver(id);
+        DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, id));
+        ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class);
+
+        ExtensionTreatment extensionInstruction =
+                resolver.getExtensionInstruction(
+                        ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST.type());
+
+        try {
+            extensionInstruction.setPropertyValue("tunnelDst", hostIp);
+        } catch (ExtensionPropertyException e) {
+            log.error("Error setting Nicira extension setting {}", e);
+        }
+
+        return extensionInstruction;
+    }
+
+    private PortNumber getTunnelPort(DeviceId deviceId) {
+        Port port = deviceService.getPorts(deviceId).stream()
+                .filter(p -> p.annotations().value("portName").equals(PORTNAME_PREFIX_TUNNEL))
+                .findAny().orElse(null);
+
+        if (port == null) {
+            log.error("No TunnelPort was created.");
+            return null;
+        }
+        return port.number();
+
+    }
+
+    public void populateExternalRules(long vni, OpenstackRouter router,
+                                      OpenstackRouterInterface routerInterface) {
+        this.router = router;
+        this.routerInterface = routerInterface;
+
+        // 1. computeNode to gateway
+        populateComputeNodeRules(vni);
+        // 2. gatewayNode to controller
+        populateRuleGatewaytoController(vni);
+    }
+
+    private void populateRuleGatewaytoController(long vni) {
+        Device gatewayDevice = getGatewayNode();
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(vni)
+                .matchEthDst(GATEWAYMAC);
+        tBuilder.setOutput(PortNumber.CONTROLLER);
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withFlag(ForwardingObjective.Flag.VERSATILE)
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(gatewayDevice.id(), fo);
+    }
+
+    private void populateComputeNodeRules(long vni) {
+        Device gatewayDevice = getGatewayNode();
+
+        StreamSupport.stream(deviceService.getAvailableDevices().spliterator(), false)
+                .filter(d -> !checkGatewayNode(d.id()))
+                .forEach(d -> populateRuleToGateway(d, gatewayDevice, vni));
+        /*deviceService.getAvailableDevices().forEach(d -> {
+            if (!checkGatewayNode(d.id())) {
+                populateRuleToGateway(d, gatewayDevice, vni);
+            }
+        });*/
+    }
+
+    private void populateRuleToGateway(Device d, Device gatewayDevice, long vni) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(vni)
+                .matchEthDst(GATEWAYMAC);
+        tBuilder.extension(buildNiciraExtenstion(d.id(), getIPAddressforDevice(gatewayDevice)), d.id())
+                .setOutput(getTunnelPort(d.id()));
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(d.id(), fo);
+    }
+
+    private Ip4Address getIPAddressforDevice(Device device) {
+        return Ip4Address.valueOf(device.annotations().value("channelId").split(":")[0]);
+    }
+
+    private Device getGatewayNode() {
+        final Device[] device = new Device[1];
+        deviceService.getAvailableDevices().forEach(d -> {
+            if (checkGatewayNode(d.id())) {
+                device[0] = d;
+            }
+        });
+        return device[0];
+    }
+
+    private boolean checkGatewayNode(DeviceId deviceId) {
+        return !deviceService.getPorts(deviceId).stream().anyMatch(port ->
+                port.annotations().value("portName").startsWith(PORTNAME_PREFIX_ROUTER) ||
+                        port.annotations().value("portName").startsWith(PORTNAME_PREFIX_VM));
+    }
+
+    private long getVni(OpenstackPort openstackPort) {
+        return Long.parseLong(openstackService.network(openstackPort.networkId()).segmentId());
+    }
+
+    public void removeExternalRules(OpenstackRouterInterface routerInterface) {
+        OpenstackPort openstackPort = openstackService.port(routerInterface.portId());
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(getVni(openstackPort))
+                .matchEthDst(GATEWAYMAC);
+
+        StreamSupport.stream(deviceService.getAvailableDevices().spliterator(), false)
+                .forEach(d -> {
+                    if (checkGatewayNode(d.id())) {
+                        removeExternalRule(d.id(), sBuilder, ForwardingObjective.Flag.VERSATILE);
+                    } else {
+                        removeExternalRule(d.id(), sBuilder, ForwardingObjective.Flag.SPECIFIC);
+                    }
+                });
+
+    }
+
+    private void removeExternalRule(DeviceId id, TrafficSelector.Builder sBuilder, ForwardingObjective.Flag flag) {
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withFlag(flag)
+                .withPriority(ROUTING_RULE_PRIORITY)
+                .fromApp(appId)
+                .remove();
+
+        flowObjectiveService.forward(id, fo);
+    }
+
+}
diff --git a/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java
new file mode 100644
index 0000000..9592065
--- /dev/null
+++ b/apps/openstacknetworking/openstackrouting/src/main/java/org/onosproject/openstacknetworking/routing/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2016 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Application for OpenstackRouting.
+ */
+package org.onosproject.openstacknetworking.routing;
\ No newline at end of file