Fix: retry purging the flow rules due to various failures

Change-Id: I3bf6b909a95db5b6dc0e9eac66506c088bd20145
diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackPurgeRulesCommand.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackPurgeRulesCommand.java
index 49c31cb..f79462e 100644
--- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackPurgeRulesCommand.java
+++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackPurgeRulesCommand.java
@@ -22,8 +22,7 @@
 import org.onosproject.net.flow.FlowRuleService;
 import org.onosproject.openstacknetworking.api.Constants;
 
-import java.util.stream.Stream;
-
+import static java.lang.Thread.sleep;
 import static java.util.stream.StreamSupport.stream;
 
 /**
@@ -33,25 +32,53 @@
         description = "Purges all flow rules installed by OpenStack networking app")
 public class OpenstackPurgeRulesCommand extends AbstractShellCommand {
 
+    private static final long TIMEOUT_MS = 10000; // we wait 10s
+    private static final long SLEEP_MS = 2000; // we wait 2s for init each node
+
     @Override
     protected void execute() {
         FlowRuleService flowRuleService = AbstractShellCommand.get(FlowRuleService.class);
         CoreService coreService = AbstractShellCommand.get(CoreService.class);
         ApplicationId appId = coreService.getAppId(Constants.OPENSTACK_NETWORKING_APP_ID);
+
         if (appId == null) {
             error("Failed to purge OpenStack networking flow rules.");
             return;
         }
-        flowRuleService.removeFlowRulesById(appId);
+
+        boolean result = true;
+        long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
 
         // we make sure all flow rules are removed from the store
-        while (true) {
-            Stream stream = stream(flowRuleService.getFlowEntriesById(appId).spliterator(), false);
-            if (stream.count() == 0) {
+        while (stream(flowRuleService.getFlowEntriesById(appId)
+                                     .spliterator(), false).count() > 0) {
+
+            long  waitMs = timeoutExpiredMs - System.currentTimeMillis();
+
+            try {
+                sleep(SLEEP_MS);
+            } catch (InterruptedException e) {
+                log.error("Exception caused during rule purging...");
+            }
+
+            if (stream(flowRuleService.getFlowEntriesById(appId)
+                                      .spliterator(), false).count() == 0) {
+                break;
+            } else {
+                flowRuleService.removeFlowRulesById(appId);
+                print("Failed to purging flow rules, retrying rule purging...");
+            }
+
+            if (waitMs <= 0) {
+                result = false;
                 break;
             }
         }
 
-        print("Successfully purged flow rules installed by OpenStack networking application.");
+        if (result) {
+            print("Successfully purged flow rules!");
+        } else {
+            error("Failed to purge flow rules.");
+        }
     }
 }
diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java
index 7834a82..bd828aa 100644
--- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java
+++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java
@@ -53,7 +53,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.stream.Stream;
 
 import static java.lang.Thread.sleep;
 import static java.util.stream.StreamSupport.stream;
@@ -78,6 +77,7 @@
     private static final String USE_SECURITY_GROUP_NAME = "useSecurityGroup";
 
     private static final long SLEEP_MS = 3000; // we wait 3s for init each node
+    private static final long TIMEOUT_MS = 10000; // we wait 10s
 
     private static final String DEVICE_OWNER_IFACE = "network:router_interface";
 
@@ -223,8 +223,11 @@
     @Path("purge/rules")
     public Response purgeRules() {
 
-        purgeRulesBase();
-        return ok(mapper().createObjectNode()).build();
+        if (purgeRulesBase()) {
+            return ok(mapper().createObjectNode()).build();
+        } else {
+            return Response.serverError().build();
+        }
     }
 
     /**
@@ -355,20 +358,50 @@
         }
     }
 
-    private void purgeRulesBase() {
+    private boolean purgeRulesBase() {
         ApplicationId appId = coreService.getAppId(Constants.OPENSTACK_NETWORKING_APP_ID);
         if (appId == null) {
             throw new ItemNotFoundException("application not found");
         }
+
         flowRuleService.removeFlowRulesById(appId);
 
+        boolean result = true;
+        long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
+
         // we make sure all flow rules are removed from the store
-        while (true) {
-            Stream stream = stream(flowRuleService.getFlowEntriesById(appId).spliterator(), false);
-            if (stream.count() == 0) {
+        while (stream(flowRuleService.getFlowEntriesById(appId)
+                                     .spliterator(), false).count() > 0) {
+
+            long  waitMs = timeoutExpiredMs - System.currentTimeMillis();
+
+            try {
+                sleep(SLEEP_MS);
+            } catch (InterruptedException e) {
+                log.error("Exception caused during rule purging...");
+            }
+
+            if (stream(flowRuleService.getFlowEntriesById(appId)
+                                      .spliterator(), false).count() == 0) {
+                break;
+            } else {
+                flowRuleService.removeFlowRulesById(appId);
+                log.info("Failed to purging flow rules, retrying rule purging...");
+            }
+
+            if (waitMs <= 0) {
+                result = false;
                 break;
             }
         }
+
+        if (result) {
+            log.info("Successfully purged flow rules!");
+        } else {
+            log.warn("Failed to purge flow rules.");
+        }
+
+        return result;
     }
 
     private void configArpModeBase(String arpMode) {