Implement Internal Routing Capability.

Change-Id: I0e3017f730a046f099c71116ded023aa2f45d154
diff --git a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
index 81fd064..012fb6c 100644
--- a/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
+++ b/apps/kubevirt-networking/api/src/main/java/org/onosproject/kubevirtnetworking/api/Constants.java
@@ -92,4 +92,5 @@
     public static final int CLI_MARGIN_LENGTH = 2;
 
     public static final int PRIORITY_STATEFUL_SNAT_RULE = 40500;
+    public static final int PRIORITY_INTERNAL_ROUTING_RULE = 41000;
 }
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
index b734226..a8867f2 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
@@ -36,6 +36,10 @@
 import org.onosproject.kubevirtnetworking.api.KubevirtNetworkEvent;
 import org.onosproject.kubevirtnetworking.api.KubevirtNetworkListener;
 import org.onosproject.kubevirtnetworking.api.KubevirtNetworkService;
+import org.onosproject.kubevirtnetworking.api.KubevirtPort;
+import org.onosproject.kubevirtnetworking.api.KubevirtPortEvent;
+import org.onosproject.kubevirtnetworking.api.KubevirtPortListener;
+import org.onosproject.kubevirtnetworking.api.KubevirtPortService;
 import org.onosproject.kubevirtnetworking.api.KubevirtRouter;
 import org.onosproject.kubevirtnetworking.api.KubevirtRouterAdminService;
 import org.onosproject.kubevirtnetworking.api.KubevirtRouterEvent;
@@ -80,12 +84,15 @@
 import static org.onlab.packet.ICMP.TYPE_ECHO_REPLY;
 import static org.onlab.packet.ICMP.TYPE_ECHO_REQUEST;
 import static org.onlab.util.Tools.groupedThreads;
+import static org.onosproject.kubevirtnetworking.api.Constants.FLAT_TABLE;
+import static org.onosproject.kubevirtnetworking.api.Constants.FORWARDING_TABLE;
 import static org.onosproject.kubevirtnetworking.api.Constants.KUBEVIRT_NETWORKING_APP_ID;
 import static org.onosproject.kubevirtnetworking.api.Constants.PRE_FLAT_TABLE;
 import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
 import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_DHCP_RULE;
 import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_FORWARDING_RULE;
 import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_ICMP_RULE;
+import static org.onosproject.kubevirtnetworking.api.Constants.PRIORITY_INTERNAL_ROUTING_RULE;
 import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_ARP_TABLE;
 import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_DHCP_TABLE;
 import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_FORWARDING_TABLE;
@@ -94,6 +101,8 @@
 import static org.onosproject.kubevirtnetworking.api.Constants.TENANT_TO_TUNNEL_PREFIX;
 import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
 import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedRouter;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtNetwork;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtPort;
 import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getbrIntMacAddress;
 import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.segmentIdHex;
 import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.NXM_NX_IP_TTL;
@@ -150,8 +159,18 @@
     @Reference(cardinality = ReferenceCardinality.MANDATORY)
     protected KubevirtRouterAdminService kubevirtRouterService;
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    protected KubevirtPortService kubevirtPortService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    protected KubevirtNetworkService kubevirtNetworkService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    protected KubevirtNodeService kubevirtNodeService;
+
     private final KubevirtNetworkListener networkListener = new InternalNetworkEventListener();
     private final KubevirtNodeListener nodeListener = new InternalNodeEventListener();
+    private final KubevirtPortListener portListener = new InternalKubevirtPortListener();
 
     private final InternalRouterEventListener kubevirtRouterlistener =
             new InternalRouterEventListener();
@@ -166,10 +185,11 @@
     protected void activate() {
         appId = coreService.registerApplication(KUBEVIRT_NETWORKING_APP_ID);
         localNodeId = clusterService.getLocalNode().id();
-        networkService.addListener(networkListener);
-        nodeService.addListener(nodeListener);
         leadershipService.runForLeadership(appId.name());
 
+        networkService.addListener(networkListener);
+        nodeService.addListener(nodeListener);
+        kubevirtPortService.addListener(portListener);
         kubevirtRouterService.addListener(kubevirtRouterlistener);
 
         log.info("Started");
@@ -179,9 +199,9 @@
     protected void deactivate() {
         networkService.removeListener(networkListener);
         nodeService.removeListener(nodeListener);
-        leadershipService.withdraw(appId.name());
-
+        kubevirtPortService.removeListener(portListener);
         kubevirtRouterService.removeListener(kubevirtRouterlistener);
+        leadershipService.withdraw(appId.name());
         eventExecutor.shutdown();
 
         log.info("Stopped");
@@ -228,7 +248,7 @@
         deviceService.removeDevice(network.tenantDeviceId(node.hostname()));
     }
 
-    private void createPatchInterface(KubevirtNode node, KubevirtNetwork network) {
+    private void createPatchInterfaceInCaseOveray(KubevirtNode node, KubevirtNetwork network) {
         Device device = deviceService.getDevice(node.ovsdb());
 
         if (device == null || !device.is(InterfaceConfig.class)) {
@@ -278,6 +298,38 @@
         ifaceConfig.removePatchMode(tunToIntIntf);
     }
 
+    private void setArpRules(KubevirtNode node, KubevirtNetwork network) {
+
+        KubevirtRouter router = getRouterForKubevirtNetwork(kubevirtRouterService, network);
+        if (router == null) {
+            return;
+        }
+
+        KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(nodeService, router);
+        if (electedGw == null) {
+            return;
+        }
+
+        setGatewayArpRuleForInternalNetworkInCaseOveray(network, TENANT_ARP_TABLE, electedGw.intgBridge(),
+                network.tenantDeviceId(node.hostname()), true);
+    }
+
+    private void setIcmpRules(KubevirtNode node, KubevirtNetwork network) {
+        KubevirtRouter router = getRouterForKubevirtNetwork(kubevirtRouterService, network);
+        if (router == null) {
+            return;
+        }
+
+        KubevirtNode electedGw = gatewayNodeForSpecifiedRouter(nodeService, router);
+        if (electedGw == null) {
+            return;
+        }
+
+        setGatewayIcmpRuleForInternalNetworkInCaseOveray(network, TENANT_ICMP_TABLE, electedGw.intgBridge(),
+                network.tenantDeviceId(node.hostname()), true);
+    }
+
+
     private void setDefaultRules(KubevirtNode node, KubevirtNetwork network) {
         DeviceId deviceId = network.tenantDeviceId(node.hostname());
 
@@ -298,10 +350,6 @@
 
         setDhcpRule(deviceId, true);
         setForwardingRule(deviceId, true);
-        setGatewayArpRuleForInternalNetwork(network, TENANT_ARP_TABLE,
-                network.tenantDeviceId(node.hostname()), true);
-        setGatewayIcmpRuleForInternalNetwork(network, TENANT_ICMP_TABLE,
-                network.tenantDeviceId(node.hostname()), true);
 
         log.info("Install default flow rules for tenant bridge {}", network.tenantBridgeName());
     }
@@ -348,29 +396,203 @@
                                                    KubevirtRouter router,
                                                    KubevirtNode electedGateway,
                                                    boolean install) {
-        setGatewayArpRuleForInternalNetwork(network, PRE_FLAT_TABLE, electedGateway.intgBridge(), install);
-        setGatewayIcmpRuleForInternalNetwork(network, PRE_FLAT_TABLE, electedGateway.intgBridge(), install);
+        switch (network.type()) {
+            case VXLAN:
+            case GRE:
+            case GENEVE:
+
+                kubevirtNodeService.completeNodes(WORKER).forEach(node -> {
+                    setGatewayArpRuleForInternalNetworkInCaseOveray(network, TENANT_ARP_TABLE,
+                            electedGateway.intgBridge(),
+                            network.tenantDeviceId(node.hostname()), install);
+                    setGatewayIcmpRuleForInternalNetworkInCaseOveray(network, TENANT_ICMP_TABLE,
+                            electedGateway.intgBridge(),
+                            network.tenantDeviceId(node.hostname()), install);
+                });
+                break;
+            case FLAT:
+            case VLAN:
+                setGatewayArpRuleForInternalNetworkInCaseVlanFlat(network, PRE_FLAT_TABLE,
+                        electedGateway.intgBridge(), install);
+                setGatewayIcmpRuleForInternalNetworkInCaseVlanFlat(network, PRE_FLAT_TABLE,
+                        electedGateway.intgBridge(), install);
+                setGatewayInterNetworkRoutingWithinSameRouter(network, router, electedGateway, install);
+                break;
+            default:
+                // do nothing
+                break;
+        }
+
     }
 
+    private void setGatewayIcmpRuleForInternalNetworkInCaseOveray(KubevirtNetwork network,
+                                                                  int tableNum,
+                                                                  DeviceId gwDeviceId,
+                                                                  DeviceId tenantDeviceId,
+                                                                  boolean install) {
+        MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gwDeviceId);
 
-    private void setGatewayInterNetworkRouting(KubevirtNetwork network, KubevirtRouter router, boolean install) {
+        if (brIntMacAddress == null) {
+            log.warn("Setting gateway ICMP rule for internal network because " +
+                    "there's no br-int port for device {}", gwDeviceId);
+            return;
+        }
+
+        Device device = deviceService.getDevice(tenantDeviceId);
+
+        if (device == null) {
+            log.warn("Setting gateway icmp rule for internal network because " +
+                            "there's no tenant device for {} to install gateway arp rule",
+                    tenantDeviceId);
+            return;
+        }
+
+
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+                .matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPProtocol(IPv4.PROTOCOL_ICMP)
+                .matchIcmpType(TYPE_ECHO_REQUEST)
+                .matchIcmpCode(CODE_ECHO_REQEUST)
+                .matchIPDst(IpPrefix.valueOf(network.gatewayIp(), 32));
+
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+                .extension(buildMoveEthSrcToDstExtension(device), device.id())
+                .extension(buildMoveIpSrcToDstExtension(device), device.id())
+                .extension(buildLoadExtension(device,
+                        NXM_NX_IP_TTL, DEFAULT_TTL), device.id())
+                .extension(buildLoadExtension(device,
+                        NXM_OF_ICMP_TYPE, TYPE_ECHO_REPLY), device.id())
+                .setIpSrc(network.gatewayIp())
+                .setEthSrc(brIntMacAddress)
+                .setOutput(PortNumber.IN_PORT);
+
+        flowService.setRule(
+                appId,
+                tenantDeviceId,
+                sBuilder.build(),
+                tBuilder.build(),
+                PRIORITY_ICMP_RULE,
+                tableNum,
+                install);
+    }
+
+    private void setGatewayArpRuleForInternalNetworkInCaseOveray(KubevirtNetwork network,
+                                                                 int tableNum,
+                                                                 DeviceId gwDeviceId,
+                                                                 DeviceId tenantDeviceId,
+                                                                 boolean install) {
+
+        MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gwDeviceId);
+
+        if (brIntMacAddress == null) {
+            log.warn("Setting gateway arp rule for internal network because " +
+                    "there's no br-int port for device {}", gwDeviceId);
+            return;
+        }
+
+        Device device = deviceService.getDevice(tenantDeviceId);
+
+        if (device == null) {
+            log.warn("Setting gateway arp rule for internal network because " +
+                    "there's no tenant device for {} to install gateway arp rule",
+                    tenantDeviceId);
+            return;
+        }
+
+
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
+        sBuilder.matchEthType(EthType.EtherType.ARP.ethType().toShort())
+                .matchArpOp(ARP.OP_REQUEST)
+                .matchArpTpa(Ip4Address.valueOf(network.gatewayIp().toString()));
+
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+        tBuilder.extension(buildMoveEthSrcToDstExtension(device), device.id())
+                .extension(buildMoveArpShaToThaExtension(device), device.id())
+                .extension(buildMoveArpSpaToTpaExtension(device), device.id())
+                .setArpOp(ARP.OP_REPLY)
+                .setArpSha(brIntMacAddress)
+                .setArpSpa(Ip4Address.valueOf(network.gatewayIp().toString()))
+                .setEthSrc(brIntMacAddress)
+                .setOutput(PortNumber.IN_PORT);
+
+        flowService.setRule(
+                appId,
+                device.id(),
+                sBuilder.build(),
+                tBuilder.build(),
+                PRIORITY_ARP_GATEWAY_RULE,
+                tableNum,
+                install
+        );
+    }
+
+    private void setGatewayInterNetworkRoutingWithinSameRouter(
+            KubevirtNetwork network, KubevirtRouter router, KubevirtNode gatewayNode, boolean install) {
         router.internal().forEach(srcNetwork -> {
-            if (srcNetwork.equals(network.networkId())) {
+            if (srcNetwork.equals(network.networkId())
+                    || kubevirtNetworkService.network(srcNetwork) == null) {
                 return;
             }
 
+            kubevirtPortService.ports(network.networkId()).forEach(port -> {
+                setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+                        port, gatewayNode, install);
+            });
         });
     }
 
-    private void setGatewayArpRuleForInternalNetwork(KubevirtNetwork network,
-                                                     int tableNum, DeviceId deviceId, boolean install) {
-        Device device = deviceService.getDevice(deviceId);
+    private void setGatewayInterNetworkRoutingFromNetworkToPort(KubevirtNetwork srcNetwork, KubevirtPort dstPort,
+                                                                KubevirtNode gatewayNode, boolean install) {
 
-        MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, deviceId);
+        Device gwDevice = deviceService.getDevice(gatewayNode.intgBridge());
+
+        if (gwDevice == null) {
+            log.warn("Failed to set internal network routing rule because " +
+                    "there's no device Id for device {}", gatewayNode.intgBridge());
+            return;
+        }
+
+        MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gatewayNode.intgBridge());
 
         if (brIntMacAddress == null) {
-            log.error("Setting gateway arp rule for internal network because " +
-                    "there's no br-int port for device {}", deviceId);
+            log.warn("Failed to set internal network routing rule because " +
+                    "there's no br-int port for device {}", gatewayNode.intgBridge());
+            return;
+        }
+
+        TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+                .matchEthType(Ethernet.TYPE_IPV4)
+                .matchEthDst(brIntMacAddress)
+                .matchIPSrc(IpPrefix.valueOf(srcNetwork.cidr()))
+                .matchIPDst(IpPrefix.valueOf(dstPort.ipAddress(), 32));
+
+        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+                .setEthSrc(brIntMacAddress)
+                .setEthDst(dstPort.macAddress())
+                .transition(FORWARDING_TABLE)
+                .build();
+
+        flowService.setRule(
+                appId,
+                gwDevice.id(),
+                sBuilder.build(),
+                treatment,
+                PRIORITY_INTERNAL_ROUTING_RULE,
+                FLAT_TABLE,
+                install
+        );
+    }
+
+    private void setGatewayArpRuleForInternalNetworkInCaseVlanFlat(KubevirtNetwork network,
+                                                                   int tableNum, DeviceId gwDeviceId, boolean install) {
+
+
+        Device device = deviceService.getDevice(gwDeviceId);
+        MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, gwDeviceId);
+
+        if (brIntMacAddress == null) {
+            log.warn("Setting gateway arp rule for internal network because " +
+                    "there's no br-int port for device {}", gwDeviceId);
             return;
         }
 
@@ -408,8 +630,8 @@
      * @param deviceId device id of the selected gateway for the network
      * @param install install if true, remove otherwise
      */
-    private void setGatewayIcmpRuleForInternalNetwork(KubevirtNetwork network,
-                                              int tableNum, DeviceId deviceId, boolean install) {
+    private void setGatewayIcmpRuleForInternalNetworkInCaseVlanFlat(KubevirtNetwork network,
+                                                                    int tableNum, DeviceId deviceId, boolean install) {
         MacAddress brIntMacAddress = getbrIntMacAddress(deviceService, deviceId);
 
         if (brIntMacAddress == null) {
@@ -586,8 +808,19 @@
                 if (network != null) {
                     initGatewayNodeForInternalNetwork(network, router, electedGw, false);
                 }
-            });
 
+                removeDetachedInternalNetworkRules(network, router, electedGw);
+            });
+        }
+
+        private void removeDetachedInternalNetworkRules(KubevirtNetwork removedNetwork, KubevirtRouter router,
+                                                        KubevirtNode electedGw) {
+            router.internal().forEach(networkName -> {
+                kubevirtPortService.ports(networkName).forEach(kubevirtPort -> {
+                    setGatewayInterNetworkRoutingFromNetworkToPort(
+                            removedNetwork, kubevirtPort, electedGw, false);
+                });
+            });
         }
 
         private void processRouterGatewayNodeAttached(KubevirtRouter router,
@@ -733,7 +966,7 @@
 
             nodeService.completeNodes().forEach(n -> {
                 createBridge(n, network);
-                createPatchInterface(n, network);
+                createPatchInterfaceInCaseOveray(n, network);
                 setDefaultRules(n, network);
             });
         }
@@ -788,8 +1021,11 @@
                                 continue;
                             }
                             createBridge(node, network);
-                            createPatchInterface(node, network);
+                            createPatchInterfaceInCaseOveray(node, network);
                             setDefaultRules(node, network);
+                            setArpRules(node, network);
+                            setIcmpRules(node, network);
+
                             break;
                         case FLAT:
                         case VLAN:
@@ -850,4 +1086,105 @@
             });
         }
     }
+
+    private class InternalKubevirtPortListener implements KubevirtPortListener {
+
+        private boolean isRelevantHelper() {
+            return Objects.equals(localNodeId, leadershipService.getLeader(appId.name()));
+        }
+
+        @Override
+        public void event(KubevirtPortEvent event) {
+            switch (event.type()) {
+                case KUBEVIRT_PORT_CREATED:
+                    eventExecutor.execute(() -> processPortCreation(event.subject()));
+                    break;
+                case KUBEVIRT_PORT_UPDATED:
+                    eventExecutor.execute(() -> processPortUpdate(event.subject()));
+                    break;
+                case KUBEVIRT_PORT_REMOVED:
+                    eventExecutor.execute(() -> processPortDeletion(event.subject()));
+                    break;
+                default:
+                    //do nothing
+                    break;
+            }
+        }
+
+        private void processPortCreation(KubevirtPort kubevirtPort) {
+            if (!isRelevantHelper()) {
+                return;
+            }
+
+            KubevirtRouter router = getRouterForKubevirtPort(kubevirtRouterService, kubevirtPort);
+            if (router == null) {
+                return;
+            }
+
+            KubevirtNode gwNode = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+            if (gwNode != null) {
+
+                router.internal().forEach(srcNetwork -> {
+                    if (srcNetwork.equals(kubevirtPort.networkId())
+                            || kubevirtNetworkService.network(srcNetwork) == null) {
+                        return;
+                    }
+                    setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+                            kubevirtPort, gwNode, true);
+                });
+            }
+        }
+
+        private void processPortUpdate(KubevirtPort kubevirtPort) {
+            if (!isRelevantHelper()) {
+                return;
+            }
+
+            KubevirtRouter router = getRouterForKubevirtPort(kubevirtRouterService, kubevirtPort);
+            if (router == null) {
+                return;
+            }
+
+            KubevirtNode gwNode = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+            if (gwNode != null) {
+
+                router.internal().forEach(srcNetwork -> {
+                    if (srcNetwork.equals(kubevirtPort.networkId())
+                            || kubevirtNetworkService.network(srcNetwork) == null) {
+                        return;
+                    }
+                    setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+                            kubevirtPort, gwNode, true);
+                });
+            }
+        }
+
+        private void processPortDeletion(KubevirtPort kubevirtPort) {
+            if (!isRelevantHelper()) {
+                return;
+            }
+
+            KubevirtRouter router = getRouterForKubevirtPort(kubevirtRouterService, kubevirtPort);
+            if (router == null) {
+                return;
+            }
+
+            KubevirtNode gwNode = gatewayNodeForSpecifiedRouter(kubevirtNodeService, router);
+
+            if (gwNode != null) {
+
+                router.internal().forEach(srcNetwork -> {
+                    if (srcNetwork.equals(kubevirtPort.networkId())
+                            || kubevirtNetworkService.network(srcNetwork) == null) {
+                        return;
+                    }
+                    setGatewayInterNetworkRoutingFromNetworkToPort(kubevirtNetworkService.network(srcNetwork),
+                            kubevirtPort, gwNode, false);
+                });
+            }
+
+        }
+    }
 }
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtRoutingSnatHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtRoutingSnatHandler.java
index 808a323..30cca24 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtRoutingSnatHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtRoutingSnatHandler.java
@@ -77,6 +77,7 @@
 import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.gatewayNodeForSpecifiedRouter;
 import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterSnatIpAddress;
 import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getbrIntMacAddress;
+import static org.onosproject.kubevirtnetworking.util.KubevirtNetworkingUtil.getRouterForKubevirtPort;
 import static org.onosproject.kubevirtnetworking.util.RulePopulatorUtil.CT_NAT_SRC_FLAG;
 import static org.onosproject.net.AnnotationKeys.PORT_NAME;
 import static org.slf4j.LoggerFactory.getLogger;
@@ -250,7 +251,7 @@
                 selector,
                 tBuilder.build(),
                 PRIORITY_STATEFUL_SNAT_RULE,
-                PRE_FLAT_TABLE,
+                FLAT_TABLE,
                 install);
     }
 
@@ -320,7 +321,7 @@
                 .niciraConnTrackTreatmentBuilder(driverService, gatewayNode.intgBridge())
                 .commit(false)
                 .natAction(true)
-                .table((short) PRE_FLAT_TABLE)
+                .table((short) FLAT_TABLE)
                 .build();
 
         TrafficTreatment treatment = DefaultTrafficTreatment.builder()
@@ -334,7 +335,7 @@
                 sBuilder.build(),
                 treatment,
                 PRIORITY_STATEFUL_SNAT_RULE,
-                PRE_FLAT_TABLE,
+                FLAT_TABLE,
                 install);
 
         router.internal().forEach(networkName -> {
@@ -549,7 +550,7 @@
                 return;
             }
 
-            KubevirtRouter router = routerForKubevirtPort(kubevirtPort);
+            KubevirtRouter router = getRouterForKubevirtPort(kubevirtRouterService, kubevirtPort);
             if (router == null) {
                 return;
             }
@@ -570,7 +571,7 @@
                 return;
             }
 
-            KubevirtRouter router = routerForKubevirtPort(kubevirtPort);
+            KubevirtRouter router = getRouterForKubevirtPort(kubevirtRouterService, kubevirtPort);
             if (router == null) {
                 return;
             }
@@ -591,7 +592,7 @@
                 return;
             }
 
-            KubevirtRouter router = routerForKubevirtPort(kubevirtPort);
+            KubevirtRouter router = getRouterForKubevirtPort(kubevirtRouterService, kubevirtPort);
             if (router == null) {
                 return;
             }
@@ -606,15 +607,5 @@
                 setStatefulSnatDownStreamRuleForKubevirtPort(gwNode, gatewaySnatIp, kubevirtPort, false);
             }
         }
-
-        private KubevirtRouter routerForKubevirtPort(KubevirtPort kubevirtPort) {
-            if (kubevirtPort.ipAddress() != null) {
-                return kubevirtRouterService.routers().stream()
-                        .filter(r -> r.internal().contains(kubevirtPort.networkId()))
-                        .findAny().orElse(null);
-            }
-
-            return null;
-        }
     }
 }
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
index f40d42c..8a9ee6c 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/util/KubevirtNetworkingUtil.java
@@ -59,6 +59,7 @@
 
 import static org.onosproject.kubevirtnetworking.api.Constants.TUNNEL_TO_TENANT_PREFIX;
 import static org.onosproject.kubevirtnode.api.KubevirtNode.Type.GATEWAY;
+import static org.onosproject.net.AnnotationKeys.PORT_MAC;
 import static org.onosproject.net.AnnotationKeys.PORT_NAME;
 
 /**
@@ -78,6 +79,7 @@
     private static final String NETWORK_PREFIX = "default/";
     private static final String MAC = "mac";
     private static final String IPS = "ips";
+    private static final String BR_INT = "br-int";
 
     /**
      * Prevents object installation from external.
@@ -468,8 +470,8 @@
     public static MacAddress getbrIntMacAddress(DeviceService deviceService,
                                                 DeviceId deviceId) {
         return MacAddress.valueOf(deviceService.getPorts(deviceId).stream()
-                .filter(port -> Objects.equals(port.annotations().value(PORT_NAME), "br-int"))
-                .map(port -> port.annotations().value("portMac"))
+                .filter(port -> Objects.equals(port.annotations().value(PORT_NAME), BR_INT))
+                .map(port -> port.annotations().value(PORT_MAC))
                 .findAny().orElse(null));
     }
 
@@ -498,4 +500,35 @@
 
         return Ip4Address.valueOf(routerSnatIp);
     }
+
+    /**
+     * Returns the kubevirt router with specified kubevirt port.
+     *
+     * @param routerService kubevirt router service
+     * @param kubevirtPort  kubevirt port
+     * @return kubevirt router
+     */
+    public static KubevirtRouter getRouterForKubevirtPort(KubevirtRouterService routerService,
+                                                          KubevirtPort kubevirtPort) {
+        if (kubevirtPort.ipAddress() != null) {
+            return routerService.routers().stream()
+                    .filter(r -> r.internal().contains(kubevirtPort.networkId()))
+                    .findAny().orElse(null);
+        }
+        return null;
+    }
+
+    /**
+     * Returns the kubevirt router with specified kubevirt network.
+     *
+     * @param routerService kubevirt router service
+     * @param kubevirtNetwork kubevirt network
+     * @return kubevirt router
+     */
+    public static KubevirtRouter getRouterForKubevirtNetwork(KubevirtRouterService routerService,
+                                                             KubevirtNetwork kubevirtNetwork) {
+        return routerService.routers().stream()
+                .filter(router -> router.internal().contains(kubevirtNetwork.networkId()))
+                .findAny().orElse(null);
+    }
 }