[ONOS-3696] Remove flow rules when a VM is terminated
- ProcessPortAdded method name is changed to ProcessPortUpdated
- PortNumber is removed in OpenstackPortInfo class
- Checks doNotPushFlows in OpenstackSwitchingRulePopulator
- etc
- rebased

Change-Id: I215892102669799af0fd8c8ac8ea377ab7be2aad
diff --git a/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackPortInfo.java b/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackPortInfo.java
new file mode 100644
index 0000000..4a7605f
--- /dev/null
+++ b/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackPortInfo.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2014-2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openstackswitching;
+
+import org.onlab.packet.Ip4Address;
+import org.onosproject.net.DeviceId;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Contains OpenstackPort Information.
+ */
+public final class OpenstackPortInfo {
+
+    private final Ip4Address hostIp;
+    private final DeviceId deviceId;
+    private final long vni;
+
+    public OpenstackPortInfo(Ip4Address hostIp, DeviceId deviceId,
+                             long vni) {
+        this.hostIp = hostIp;
+        this.deviceId = deviceId;
+        this.vni = vni;
+    }
+
+    public Ip4Address ip() {
+        return hostIp;
+    }
+
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    public long vni() {
+        return vni;
+    }
+
+    public static OpenstackPortInfo.Builder builder() {
+        return new Builder();
+    }
+
+    public static final class Builder {
+        private Ip4Address hostIp;
+        private DeviceId deviceId;
+        private long vni;
+
+        public Builder setHostIp(Ip4Address hostIp) {
+            this.hostIp = checkNotNull(hostIp, "hostIp cannot be null");
+            return this;
+        }
+
+        public Builder setDeviceId(DeviceId deviceId) {
+            this.deviceId = checkNotNull(deviceId, "deviceId cannot be null");
+            return this;
+        }
+
+        public Builder setVNI(long vni) {
+            this.vni = checkNotNull(vni, "vni cannot be null");
+            return this;
+        }
+        public OpenstackPortInfo build() {
+            return new OpenstackPortInfo(this);
+        }
+    }
+
+    private OpenstackPortInfo(Builder builder) {
+        hostIp = builder.hostIp;
+        deviceId = builder.deviceId;
+        vni = builder.vni;
+    }
+}
diff --git a/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingManager.java b/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingManager.java
index 24d5f58..564ae99 100644
--- a/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingManager.java
+++ b/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingManager.java
@@ -17,6 +17,7 @@
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -25,13 +26,11 @@
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ip4Address;
-import org.onlab.packet.IpAddress;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.CoreService;
 import org.onosproject.dhcp.DhcpService;
 import org.onosproject.event.AbstractEvent;
 import org.onosproject.net.Device;
-import org.onosproject.net.DeviceId;
 import org.onosproject.net.Host;
 import org.onosproject.net.Port;
 import org.onosproject.net.config.ConfigFactory;
@@ -42,12 +41,6 @@
 import org.onosproject.net.device.DeviceListener;
 import org.onosproject.net.device.DeviceService;
 import org.onosproject.net.driver.DriverService;
-import org.onosproject.net.flow.FlowEntry;
-import org.onosproject.net.flow.FlowRuleService;
-import org.onosproject.net.flow.criteria.Criterion;
-import org.onosproject.net.flow.criteria.IPCriterion;
-import org.onosproject.net.flow.instructions.Instruction;
-import org.onosproject.net.flow.instructions.L2ModificationInstruction;
 import org.onosproject.net.flowobjective.FlowObjectiveService;
 import org.onosproject.net.host.HostEvent;
 import org.onosproject.net.host.HostListener;
@@ -60,6 +53,7 @@
 import org.slf4j.LoggerFactory;
 import java.util.List;
 import java.util.Collection;
+import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
@@ -104,9 +98,6 @@
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected DriverService driverService;
 
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected FlowRuleService flowRuleService;
-
     private ApplicationId appId;
     private boolean doNotPushFlows;
     private Ip4Address neutronServer;
@@ -136,6 +127,9 @@
             }
     );
 
+
+    private Map<String, OpenstackPortInfo> openstackPortInfoMap = Maps.newHashMap();
+
     @Activate
     protected void activate() {
         appId = coreService
@@ -258,13 +252,39 @@
         log.debug("device {} is added", device.id());
     }
 
-    private void processPortAdded(Device device, Port port) {
-        if (!port.annotations().value("portName").equals("vxlan")
-            && port.isEnabled() && !doNotPushFlows) {
-            OpenstackSwitchingRulePopulator rulePopulator =
-                    new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
-                            deviceService, restHandler, driverService);
-            rulePopulator.populateSwitchingRules(device, port);
+    private void processPortUpdated(Device device, Port port) {
+        if (!port.annotations().value("portName").equals("vxlan") && !doNotPushFlows) {
+            if (port.isEnabled()) {
+                OpenstackSwitchingRulePopulator rulePopulator =
+                        new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
+                                deviceService, restHandler, driverService);
+                rulePopulator.populateSwitchingRules(device, port);
+
+                OpenstackPort openstackPort = port(port);
+
+                long vni = Long.parseLong(restHandler.getNetworks().stream()
+                        .filter(n -> n.id().equals(openstackPort.networkId()))
+                        .findAny().orElse(null).segmentId());
+
+                OpenstackPortInfo.Builder portBuilder = OpenstackPortInfo.builder()
+                        .setDeviceId(device.id())
+                        .setHostIp((Ip4Address) openstackPort.fixedIps().values().stream().findFirst().orElse(null))
+                        .setVNI(vni);
+
+                openstackPortInfoMap.putIfAbsent(port.annotations().value("portName"),
+                        portBuilder.build());
+            }
+
+            //In case portupdate event is driven by vm shutoff from openstack
+            if (!port.isEnabled() && openstackPortInfoMap.containsKey(port.annotations().value("portName"))) {
+                log.debug("Flowrules according to the port {} were removed", port.number().toString());
+                OpenstackSwitchingRulePopulator rulePopulator =
+                        new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
+                                deviceService, restHandler, driverService);
+                openstackPortInfoMap.get(port.annotations().value("portName"));
+                rulePopulator.removeSwitchingRules(port, openstackPortInfoMap);
+                openstackPortInfoMap.remove(port.annotations().value("portName"));
+            }
         }
     }
 
@@ -300,46 +320,6 @@
 
     private void processHostRemoved(Host host) {
         log.debug("host {} was removed", host.toString());
-
-        try {
-            if (!doNotPushFlows) {
-                IpAddress hostIp = host.ipAddresses().stream().
-                        filter(ip -> ip.isIp4()).findAny().orElse(null);
-                OpenstackSwitchingRulePopulator rulePopulator =
-                        new OpenstackSwitchingRulePopulator(appId, flowObjectiveService,
-                                deviceService, restHandler, driverService);
-                rulePopulator.removeSwitchingRules(host.location().deviceId(),
-                        hostIp.getIp4Address());
-            }
-
-            dhcpService.removeStaticMapping(host.mac());
-        } catch (NoSuchElementException e) {
-            log.error("No IP address is assigned.");
-        }
-    }
-
-    private long getVniFromFlowRules(DeviceId deviceId, Ip4Address hostIp) {
-
-        for (FlowEntry flowEntry: flowRuleService.getFlowEntries(deviceId)) {
-            Criterion c = flowEntry.selector().getCriterion(Criterion.Type.IPV4_DST);
-            if (c != null) {
-                IPCriterion destIpCriterion = (IPCriterion) c;
-                if (destIpCriterion.ip().getIp4Prefix().address().equals(hostIp)) {
-                    for (Instruction i : flowEntry.treatment().immediate()) {
-                        if (i.type().equals(Instruction.Type.L2MODIFICATION)) {
-                            L2ModificationInstruction l2m = (L2ModificationInstruction) i;
-                            if (l2m.subtype().equals(L2ModificationInstruction.L2SubType.TUNNEL_ID)) {
-                                L2ModificationInstruction.ModTunnelIdInstruction setTunnelInstr =
-                                        (L2ModificationInstruction.ModTunnelIdInstruction) l2m;
-                                return setTunnelInstr.tunnelId();
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        return 0;
     }
 
     private void registerDhcpInfo(OpenstackPort openstackPort) {
@@ -350,7 +330,7 @@
         Ip4Address domainServer;
         OpenstackSubnet openstackSubnet;
 
-        ip4Address = (Ip4Address) openstackPort.fixedIps().values().toArray()[0];
+        ip4Address = (Ip4Address) openstackPort.fixedIps().values().stream().findFirst().orElse(null);
 
         openstackSubnet = restHandler.getSubnets().stream()
                 .filter(n -> n.networkId().equals(openstackPort.networkId()))
@@ -442,7 +422,7 @@
                     case DEVICE_UPDATED:
                         Port port = (Port) deviceEvent.subject();
                         if (port.isEnabled()) {
-                            processPortAdded((Device) deviceEvent.subject(), deviceEvent.port());
+                            processPortUpdated((Device) deviceEvent.subject(), deviceEvent.port());
                         }
                         break;
                     case DEVICE_AVAILABILITY_CHANGED:
@@ -452,10 +432,10 @@
                         }
                         break;
                     case PORT_ADDED:
-                        processPortAdded((Device) deviceEvent.subject(), deviceEvent.port());
+                        processPortUpdated((Device) deviceEvent.subject(), deviceEvent.port());
                         break;
                     case PORT_UPDATED:
-                        processPortAdded((Device) deviceEvent.subject(), deviceEvent.port());
+                        processPortUpdated((Device) deviceEvent.subject(), deviceEvent.port());
                         break;
                     case PORT_REMOVED:
                         processPortRemoved((Device) deviceEvent.subject(), deviceEvent.port());
@@ -502,19 +482,4 @@
         }
 
     }
-
-    private final class PortInfo {
-        DeviceId deviceId;
-        String portName;
-        Ip4Address fixedIp;
-        Ip4Address hostIp;
-
-        private PortInfo(DeviceId deviceId, String portName, Ip4Address fixedIp,
-                         Ip4Address hostIp) {
-            this.deviceId = deviceId;
-            this.portName = portName;
-            this.fixedIp = fixedIp;
-            this.hostIp = hostIp;
-        }
-    }
 }
\ No newline at end of file
diff --git a/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingRulePopulator.java b/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingRulePopulator.java
index 8d5c780..f5f93dc 100644
--- a/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingRulePopulator.java
+++ b/apps/openstackswitching/app/src/main/java/org/onosproject/openstackswitching/OpenstackSwitchingRulePopulator.java
@@ -18,7 +18,6 @@
 
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ip4Address;
-import org.onlab.packet.MacAddress;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.Device;
 import org.onosproject.net.DeviceId;
@@ -45,6 +44,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.Collection;
+import java.util.Map;
 
 /**
  * Populates switching flow rules.
@@ -118,31 +118,25 @@
         }
     }
 
-    /**
-     * Returns OpenstackPort object for the Port reference given.
-     *
-     * @param port Port object
-     * @return OpenstackPort reference, or null when not found
-     */
-    public OpenstackPort openstackPort(Port port) {
-        String uuid = port.annotations().value("portName").substring(3);
-        return openstackPortList.stream().filter(p -> p.id().startsWith(uuid))
-                .findAny().orElse(null);
-    }
+    private void setFlowRuleForTunnelTag(DeviceId deviceId, Port port, String vni) {
 
-    /**
-     * Remove flows rules for the VM removed.
-     *
-     * @param deviceId device to remove rules
-     * @param vmIp IP address of the VM removed
-     */
-    public void removeSwitchingRules(DeviceId deviceId, Ip4Address vmIp) {
-        removeFlowRuleForVMsInSameCnode(deviceId, vmIp);
-        deviceService.getAvailableDevices().forEach(device -> {
-            if (!device.id().equals(deviceId)) {
-                removeVxLanFlowRule(device.id(), vmIp);
-            }
-        });
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchInPort(port.number());
+
+        tBuilder.setTunnelId(Long.parseLong(vni));
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(TUNNELTAG_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
     }
 
     /**
@@ -155,14 +149,45 @@
         Ip4Address vmIp = getFixedIpAddressForPort(port.annotations().value("portName"));
         String portName = port.annotations().value("portName");
         String vni = getVniForPort(portName);
-        MacAddress vmMacAddress = getVmMacAddressForPort(portName);
 
         if (vmIp != null) {
-            setFlowRuleForVMsInSameCnode(vmIp, device.id(), port, vni, vmMacAddress);
+            setFlowRuleForVMsInSameCnode(vmIp, device.id(), port, vni);
         }
     }
 
     /**
+     * Sets the flow rules for traffic between VMs in the same Cnode.
+     *
+     * @param ip4Address VM IP address
+     * @param id device ID to put rules
+     * @param port VM port
+     * @param vni VM VNI
+     */
+    private void setFlowRuleForVMsInSameCnode(Ip4Address ip4Address, DeviceId id,
+                                              Port port, String vni) {
+
+        //For L2 Switching Case
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(ip4Address.toIpPrefix())
+                .matchTunnelId(Long.parseLong(vni));
+
+        tBuilder.setOutput(port.number());
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(id, fo);
+    }
+
+    /**
      * Populates the flow rules for traffic to VMs in different Cnode using
      * Nicira extention.
      *
@@ -174,7 +199,6 @@
         String channelId = device.annotations().value("channelId");
         Ip4Address hostIpAddress = Ip4Address.valueOf(channelId.split(":")[0]);
         Ip4Address fixedIp = getFixedIpAddressForPort(portName);
-        MacAddress vmMac = getVmMacAddressForPort(portName);
         String vni = getVniForPort(portName);
         deviceService.getAvailableDevices().forEach(d -> {
             if (!d.equals(device)) {
@@ -183,11 +207,10 @@
                     if (!p.equals(port) && vni.equals(getVniForPort(pName))) {
                         String cidx = d.annotations().value("channelId");
                         Ip4Address hostIpx = Ip4Address.valueOf(cidx.split(":")[0]);
-                        MacAddress vmMacx = getVmMacAddressForPort(pName);
                         Ip4Address fixedIpx = getFixedIpAddressForPort(pName);
                         if (port.isEnabled()) {
-                            setVxLanFlowRule(vni, device.id(), hostIpx, fixedIpx, vmMacx);
-                            setVxLanFlowRule(vni, d.id(), hostIpAddress, fixedIp, vmMac);
+                            setVxLanFlowRule(vni, device.id(), hostIpx, fixedIpx);
+                            setVxLanFlowRule(vni, d.id(), hostIpAddress, fixedIp);
                         }
                     }
                 });
@@ -196,6 +219,190 @@
     }
 
     /**
+     * Sets the flow rules between traffic from VMs in different Cnode.
+     *
+     * @param vni  VNI
+     * @param deviceId device ID
+     * @param hostIp host IP of the VM
+     * @param vmIp fixed IP of the VM
+     */
+    private void setVxLanFlowRule(String vni, DeviceId deviceId, Ip4Address hostIp,
+                                  Ip4Address vmIp) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(Long.parseLong(vni))
+                .matchIPDst(vmIp.toIpPrefix());
+        tBuilder.extension(buildNiciraExtenstion(deviceId, hostIp), deviceId)
+                .setOutput(getTunnelPort(deviceId));
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .add();
+
+        flowObjectiveService.forward(deviceId, fo);
+    }
+
+    /**
+     * Returns OpenstackPort object for the Port reference given.
+     *
+     * @param port Port object
+     * @return OpenstackPort reference, or null when not found
+     */
+    public OpenstackPort openstackPort(Port port) {
+        String uuid = port.annotations().value("portName").substring(3);
+        return openstackPortList.stream().filter(p -> p.id().startsWith(uuid))
+                .findAny().orElse(null);
+    }
+
+    /**
+     * Remove flows rules for the removed VM.
+     *
+     * @param removedPort removedport info
+     * @param openstackPortInfoMap openstackPortInfoMap
+     */
+    public void removeSwitchingRules(Port removedPort, Map<String, OpenstackPortInfo> openstackPortInfoMap) {
+
+        OpenstackPortInfo openstackPortInfo = openstackPortInfoMap
+                .get(removedPort.annotations().value("portName"));
+
+        DeviceId deviceId = openstackPortInfo.deviceId();
+        Ip4Address vmIp = openstackPortInfo.ip();
+        PortNumber portNumber = removedPort.number();
+        long vni = openstackPortInfo.vni();
+
+        removeFlowRuleForTunnelTag(deviceId, portNumber);
+        removeFlowRuleForVMsInSameCnode(deviceId, vmIp, vni);
+        removeFlowRuleForVMsInDiffrentCnode(removedPort, deviceId, vmIp, vni, openstackPortInfoMap);
+    }
+
+    /**
+     * Removes flow rules for tagging tunnelId.
+     *
+     * @param deviceId device id
+     * @param portNumber port number
+     */
+    private void removeFlowRuleForTunnelTag(DeviceId deviceId, PortNumber portNumber) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchInPort(portNumber);
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(tBuilder.build())
+                .withPriority(TUNNELTAG_RULE_PRIORITY)
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .fromApp(appId)
+                .remove();
+
+        flowObjectiveService.forward(deviceId, fo);
+    }
+
+    /**
+     * Removes the flow rules for traffic between VMs in the same Cnode.
+     *
+     * @param deviceId device id on which removed VM was run
+     * @param vmIp ip of the removed VM
+     * @param vni vni which removed VM was belonged
+     */
+    private void removeFlowRuleForVMsInSameCnode(DeviceId deviceId, Ip4Address vmIp, long vni) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(vmIp.toIpPrefix())
+                .matchTunnelId(vni);
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(DefaultTrafficTreatment.builder().build())
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .fromApp(appId)
+                .remove();
+
+        flowObjectiveService.forward(deviceId, fo);
+    }
+
+    /**
+     * Removes the flow rules for traffic between VMs in the different Cnode.
+     *
+     * @param removedPort removedport info
+     * @param deviceId device id on which removed VM was run
+     * @param vmIp ip of the removed VM
+     * @param vni vni which removed VM was belonged
+     * @param openstackPortInfoMap openstackPortInfoMap
+     */
+    private void removeFlowRuleForVMsInDiffrentCnode(Port removedPort, DeviceId deviceId, Ip4Address vmIp,
+                                                     long vni, Map<String, OpenstackPortInfo> openstackPortInfoMap) {
+
+        final boolean anyPortRemainedInSameCnode
+                = checkIfAnyPortRemainedInSameCnode(removedPort, deviceId, vni, openstackPortInfoMap);
+
+        openstackPortInfoMap.forEach((port, portInfo) -> {
+            if (portInfo.vni() == vni && !portInfo.deviceId().equals(deviceId)) {
+                removeVxLanFlowRule(portInfo.deviceId(), vmIp, vni);
+                if (!anyPortRemainedInSameCnode) {
+                    removeVxLanFlowRule(deviceId, portInfo.ip(), vni);
+                }
+            }
+        });
+    }
+
+    /**
+     * Removes the flow rules between traffic from VMs in different Cnode.
+     *
+     * @param deviceId device id
+     * @param vmIp ip
+     * @param vni vni which removed VM was belonged
+     */
+    private void removeVxLanFlowRule(DeviceId deviceId, Ip4Address vmIp, long vni) {
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+
+        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
+                .matchTunnelId(vni)
+                .matchIPDst(vmIp.toIpPrefix());
+
+        ForwardingObjective fo = DefaultForwardingObjective.builder()
+                .withSelector(sBuilder.build())
+                .withTreatment(DefaultTrafficTreatment.builder().build())
+                .withFlag(ForwardingObjective.Flag.SPECIFIC)
+                .withPriority(SWITCHING_RULE_PRIORITY)
+                .fromApp(appId)
+                .remove();
+
+        flowObjectiveService.forward(deviceId, fo);
+    }
+
+    /**
+     * Checks if there is any port remained with same vni at the device, on which the removed VM was run.
+     *
+     * @param removedPort removedport info
+     * @param deviceId device id on which removed VM was run
+     * @param vni vni which removed VM was belonged
+     * @param openstackPortInfoMap openstackPortInfoMap
+     * @return true if there is, false otherwise
+     */
+    private boolean checkIfAnyPortRemainedInSameCnode(Port removedPort, DeviceId deviceId, long vni,
+                                                    Map<String, OpenstackPortInfo> openstackPortInfoMap) {
+
+        for (Map.Entry<String, OpenstackPortInfo> entry : openstackPortInfoMap.entrySet()) {
+            if (!removedPort.annotations().value("portName").equals(entry.getKey())) {
+                if (entry.getValue().vni() == vni && entry.getValue().deviceId().equals(deviceId)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    /**
      * Returns the VNI of the VM of the port.
      *
      * @param portName VM port
@@ -248,149 +455,6 @@
         return (Ip4Address) port.fixedIps().values().toArray()[0];
     }
 
-    /**
-     * Returns the MAC address of the VM of the port.
-     *
-     * @param portName VM port
-     * @return MAC address of the VM
-     */
-    private MacAddress getVmMacAddressForPort(String portName) {
-
-        String uuid = portName.substring(3);
-        OpenstackPort port = openstackPortList.stream()
-                .filter(p -> p.id().startsWith(uuid))
-                .findFirst().orElse(null);
-
-        if (port == null) {
-            log.error("There is no port information for port name {}", portName);
-            return null;
-        }
-
-        return port.macAddress();
-    }
-
-    private void setFlowRuleForTunnelTag(DeviceId deviceId, Port port, String vni) {
-
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchInPort(port.number());
-
-        tBuilder.setTunnelId(Long.parseLong(vni));
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withPriority(TUNNELTAG_RULE_PRIORITY)
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(deviceId, fo);
-    }
-
-
-    /**
-     * Sets the flow rules for traffic between VMs in the same Cnode.
-     *
-     * @param ip4Address VM IP address
-     * @param id device ID to put rules
-     * @param port VM port
-     * @param vni VM VNI
-     * @param vmMacAddress VM MAC address
-     */
-    private void setFlowRuleForVMsInSameCnode(Ip4Address ip4Address, DeviceId id,
-                                              Port port, String vni, MacAddress vmMacAddress) {
-
-        //For L2 Switching Case
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPDst(ip4Address.toIpPrefix())
-                .matchTunnelId(Long.parseLong(vni));
-
-        tBuilder.setOutput(port.number());
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withPriority(SWITCHING_RULE_PRIORITY)
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(id, fo);
-    }
-
-
-    /**
-     * Sets the flow rules between traffic from VMs in different Cnode.
-     *
-     * @param vni  VNI
-     * @param id device ID
-     * @param hostIp host IP of the VM
-     * @param vmIp fixed IP of the VM
-     * @param vmMac MAC address of the VM
-     */
-    private void setVxLanFlowRule(String vni, DeviceId id, Ip4Address hostIp,
-                                  Ip4Address vmIp, MacAddress vmMac) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchTunnelId(Long.parseLong(vni))
-                .matchIPDst(vmIp.toIpPrefix());
-
-        tBuilder.extension(buildNiciraExtenstion(id, hostIp), id)
-                .setOutput(getTunnelPort(id));
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(tBuilder.build())
-                .withPriority(SWITCHING_RULE_PRIORITY)
-                .withFlag(ForwardingObjective.Flag.SPECIFIC)
-                .fromApp(appId)
-                .add();
-
-        flowObjectiveService.forward(id, fo);
-    }
-
-    private void removeFlowRuleForVMsInSameCnode(DeviceId id, Ip4Address vmIp) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPDst(vmIp.toIpPrefix());
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(DefaultTrafficTreatment.builder().build())
-                .withFlag(ForwardingObjective.Flag.VERSATILE)
-                .withPriority(SWITCHING_RULE_PRIORITY)
-                .fromApp(appId)
-                .remove();
-
-        flowObjectiveService.forward(id, fo);
-    }
-
-    private void removeVxLanFlowRule(DeviceId id, Ip4Address vmIp) {
-        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
-        // XXX: Later, more matches will be added when multiple table is implemented.
-        sBuilder.matchEthType(Ethernet.TYPE_IPV4)
-                .matchIPDst(vmIp.toIpPrefix());
-
-        ForwardingObjective fo = DefaultForwardingObjective.builder()
-                .withSelector(sBuilder.build())
-                .withTreatment(DefaultTrafficTreatment.builder().build())
-                .withFlag(ForwardingObjective.Flag.VERSATILE)
-                .withPriority(SWITCHING_RULE_PRIORITY)
-                .fromApp(appId)
-                .remove();
-
-        flowObjectiveService.forward(id, fo);
-    }
-
     private ExtensionTreatment buildNiciraExtenstion(DeviceId id, Ip4Address hostIp) {
         Driver driver = driverService.getDriver(id);
         DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, id));
@@ -409,8 +473,8 @@
         return  extensionInstruction;
     }
 
-    private PortNumber getTunnelPort(DeviceId id) {
-        Port port = deviceService.getPorts(id).stream()
+    private PortNumber getTunnelPort(DeviceId deviceId) {
+        Port port = deviceService.getPorts(deviceId).stream()
                 .filter(p -> p.annotations().value("portName").equals("vxlan"))
                 .findAny().orElse(null);