Fix: resolve an event ordering issue on removing ingress rules

Change-Id: Ie7cfe328ea99592e05d40ee1cfdcce159fcc4a32
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
index 1d0a2e9..b82aa3c 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtNetworkHandler.java
@@ -335,9 +335,57 @@
 
     private void removeAllFlows(KubevirtNode node, KubevirtNetwork network) {
         DeviceId deviceId = network.tenantDeviceId(node.hostname());
+        removeIngressRules(network);
         flowService.purgeRules(deviceId);
     }
 
+    private void removeIngressRules(KubevirtNetwork network) {
+        if (network == null) {
+            return;
+        }
+
+        if (network.type() == FLAT || network.type() == VLAN) {
+            return;
+        }
+
+        if (network.segmentId() == null) {
+            return;
+        }
+
+        for (KubevirtNode localNode : kubevirtNodeService.completeNodes(WORKER)) {
+
+            while (true) {
+                KubevirtNode updatedNode = kubevirtNodeService.node(localNode.hostname());
+                if (tunnelToTenantPort(deviceService, updatedNode, network) != null) {
+                    break;
+                } else {
+                    log.info("Waiting for tunnel to tenant patch port creation " +
+                            "on ingress rule setup on node {}", updatedNode);
+                    waitFor(3);
+                }
+            }
+
+            PortNumber patchPortNumber = tunnelToTenantPort(deviceService, localNode, network);
+
+            TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder()
+                    .matchTunnelId(Long.parseLong(network.segmentId()));
+
+            TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder()
+                    .setOutput(patchPortNumber);
+
+            flowService.setRule(
+                    appId,
+                    localNode.tunBridge(),
+                    sBuilder.build(),
+                    tBuilder.build(),
+                    PRIORITY_TUNNEL_RULE,
+                    TUNNEL_DEFAULT_TABLE,
+                    false);
+
+            log.debug("Set ingress rules for segment ID {}", network.segmentId());
+        }
+    }
+
     private void removePatchInterface(KubevirtNode node, KubevirtNetwork network) {
         Device device = deviceService.getDevice(node.ovsdb());
 
diff --git a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java
index 8c76ed4..dcfb4f4 100644
--- a/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java
+++ b/apps/kubevirt-networking/app/src/main/java/org/onosproject/kubevirtnetworking/impl/KubevirtSwitchingTenantHandler.java
@@ -137,6 +137,11 @@
     }
 
     private void setIngressRules(KubevirtNetwork network, boolean install) {
+        // FIXME: due to the event ordering issue, we remove all ingress rules from KubevirtNetworkHandler class.
+        if (!install) {
+            return;
+        }
+
         if (network == null) {
             return;
         }
@@ -179,7 +184,7 @@
                     TUNNEL_DEFAULT_TABLE,
                     install);
 
-            log.debug("Install ingress rules for segment ID {}", network.segmentId());
+            log.debug("Set ingress rules for segment ID {}", network.segmentId());
         }
     }
 
@@ -218,7 +223,7 @@
                     TUNNEL_DEFAULT_TABLE,
                     install);
 
-            log.debug("Install ingress rules for segment ID {}", network.segmentId());
+            log.debug("Set ingress rules for segment ID {}", network.segmentId());
         }
     }
 
@@ -308,7 +313,7 @@
                     install);
         }
 
-        log.debug("Install egress rules for instance {}, segment ID {}",
+        log.debug("Set egress rules for instance {}, segment ID {}",
                 port.ipAddress(), network.segmentId());
     }