Fix: retry purging the flow rules due to various failures

Change-Id: I3bf6b909a95db5b6dc0e9eac66506c088bd20145
diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackPurgeRulesCommand.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackPurgeRulesCommand.java
index 707b5e6..e12c3ed 100644
--- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackPurgeRulesCommand.java
+++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackPurgeRulesCommand.java
@@ -23,6 +23,9 @@
 import org.onosproject.net.flow.FlowRuleService;
 import org.onosproject.openstacknetworking.api.Constants;
 
+import static java.lang.Thread.sleep;
+import static java.util.stream.StreamSupport.stream;
+
 /**
  * Purges all existing network states.
  */
@@ -31,16 +34,56 @@
         description = "Purges all flow rules installed by OpenStack networking app")
 public class OpenstackPurgeRulesCommand extends AbstractShellCommand {
 
+    private static final long TIMEOUT_MS = 10000; // we wait 10s
+    private static final long SLEEP_MS = 2000; // we wait 2s for init each node
+
     @Override
     protected void doExecute() {
         FlowRuleService flowRuleService = AbstractShellCommand.get(FlowRuleService.class);
         CoreService coreService = AbstractShellCommand.get(CoreService.class);
         ApplicationId appId = coreService.getAppId(Constants.OPENSTACK_NETWORKING_APP_ID);
+
         if (appId == null) {
             error("Failed to purge OpenStack networking flow rules.");
             return;
         }
+
         flowRuleService.removeFlowRulesById(appId);
         print("Successfully purged flow rules installed by OpenStack networking application.");
+
+        boolean result = true;
+        long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
+
+        // we make sure all flow rules are removed from the store
+        while (stream(flowRuleService.getFlowEntriesById(appId)
+                                     .spliterator(), false).count() > 0) {
+
+            long  waitMs = timeoutExpiredMs - System.currentTimeMillis();
+
+            try {
+                sleep(SLEEP_MS);
+            } catch (InterruptedException e) {
+                log.error("Exception caused during rule purging...");
+            }
+
+            if (stream(flowRuleService.getFlowEntriesById(appId)
+                                      .spliterator(), false).count() == 0) {
+                break;
+            } else {
+                flowRuleService.removeFlowRulesById(appId);
+                print("Failed to purging flow rules, retrying rule purging...");
+            }
+
+            if (waitMs <= 0) {
+                result = false;
+                break;
+            }
+        }
+
+        if (result) {
+            print("Successfully purged flow rules!");
+        } else {
+            error("Failed to purge flow rules.");
+        }
     }
 }
diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java
index 2c36b2e..bd828aa 100644
--- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java
+++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java
@@ -55,6 +55,7 @@
 import java.util.Optional;
 
 import static java.lang.Thread.sleep;
+import static java.util.stream.StreamSupport.stream;
 import static org.onlab.util.Tools.nullIsIllegal;
 import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.addRouterIface;
 import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.checkActivationFlag;
@@ -76,6 +77,7 @@
     private static final String USE_SECURITY_GROUP_NAME = "useSecurityGroup";
 
     private static final long SLEEP_MS = 3000; // we wait 3s for init each node
+    private static final long TIMEOUT_MS = 10000; // we wait 10s
 
     private static final String DEVICE_OWNER_IFACE = "network:router_interface";
 
@@ -221,8 +223,11 @@
     @Path("purge/rules")
     public Response purgeRules() {
 
-        purgeRulesBase();
-        return ok(mapper().createObjectNode()).build();
+        if (purgeRulesBase()) {
+            return ok(mapper().createObjectNode()).build();
+        } else {
+            return Response.serverError().build();
+        }
     }
 
     /**
@@ -353,12 +358,50 @@
         }
     }
 
-    private void purgeRulesBase() {
+    private boolean purgeRulesBase() {
         ApplicationId appId = coreService.getAppId(Constants.OPENSTACK_NETWORKING_APP_ID);
         if (appId == null) {
             throw new ItemNotFoundException("application not found");
         }
+
         flowRuleService.removeFlowRulesById(appId);
+
+        boolean result = true;
+        long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
+
+        // we make sure all flow rules are removed from the store
+        while (stream(flowRuleService.getFlowEntriesById(appId)
+                                     .spliterator(), false).count() > 0) {
+
+            long  waitMs = timeoutExpiredMs - System.currentTimeMillis();
+
+            try {
+                sleep(SLEEP_MS);
+            } catch (InterruptedException e) {
+                log.error("Exception caused during rule purging...");
+            }
+
+            if (stream(flowRuleService.getFlowEntriesById(appId)
+                                      .spliterator(), false).count() == 0) {
+                break;
+            } else {
+                flowRuleService.removeFlowRulesById(appId);
+                log.info("Failed to purging flow rules, retrying rule purging...");
+            }
+
+            if (waitMs <= 0) {
+                result = false;
+                break;
+            }
+        }
+
+        if (result) {
+            log.info("Successfully purged flow rules!");
+        } else {
+            log.warn("Failed to purge flow rules.");
+        }
+
+        return result;
     }
 
     private void configArpModeBase(String arpMode) {