Fix: guarantee to referring to the correct openstack node state

Change-Id: I73265ba744ea5ec8e5e0e3e8f36735ae1d1ac952
diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackSyncRulesCommand.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackSyncRulesCommand.java
index 33f1a03..f6ba119 100644
--- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackSyncRulesCommand.java
+++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/cli/OpenstackSyncRulesCommand.java
@@ -22,6 +22,8 @@
 import org.onosproject.openstacknode.api.OpenstackNodeAdminService;
 import org.onosproject.openstacknode.api.OpenstackNodeService;
 
+import static java.lang.Thread.sleep;
+
 /**
  * Re-installs flow rules for OpenStack networking.
  */
@@ -29,7 +31,7 @@
         description = "Re-installs flow rules for OpenStack networking")
 public class OpenstackSyncRulesCommand extends AbstractShellCommand {
 
-    private static final long TIMEOUT_MS = 10000; // we wait 10s for init each node
+    private static final long SLEEP_MS = 3000; // we wait 3s for init each node
 
     @Override
     protected void execute() {
@@ -45,19 +47,16 @@
             OpenstackNode updated = osNode.updateState(NodeState.INIT);
             osNodeAdminService.updateNode(updated);
 
-            long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
-            while (updated.state() != NodeState.COMPLETE) {
-                long  waitMs = timeoutExpiredMs - System.currentTimeMillis();
+            try {
+                sleep(SLEEP_MS);
+            } catch (InterruptedException e) {
+                log.error("Exception caused during node synchronization...");
+            }
 
-                if (updated.state() == NodeState.COMPLETE) {
-                    print("Finished sync rules for node {}", updated.hostname());
-                    break;
-                }
-
-                if (waitMs <= 0) {
-                    error("Failed to sync rules for node {}", updated.hostname());
-                    break;
-                }
+            if (osNodeService.node(osNode.hostname()).state() == NodeState.COMPLETE) {
+                print("Finished sync rules for node %s", osNode.hostname());
+            } else {
+                error("Failed to sync rules for node %s", osNode.hostname());
             }
         });
         print("Successfully requested re-installing flow rules.");
diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandler.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandler.java
index 4e69759..fa72289 100644
--- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandler.java
+++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandler.java
@@ -607,6 +607,12 @@
 
         @Override
         public boolean isRelevant(OpenstackNodeEvent event) {
+
+            // add subnet gateway to local storage in all cluster nodes
+            // TODO: need to persistent the gateway collection into eventually
+            // consistent map sooner or later
+            addAllSubnetGateways();
+
             // do not allow to proceed without leadership
             NodeId leader = leadershipService.getLeader(appId.name());
             return Objects.equals(localNodeId, leader) && event.subject().type() == COMPUTE;
@@ -619,7 +625,6 @@
                 case OPENSTACK_NODE_COMPLETE:
                     setDefaultArpRule(osNode, true);
                     setAllArpRules(osNode, true);
-                    addAllSubnetGateways();
                     break;
                 case OPENSTACK_NODE_INCOMPLETE:
                     setDefaultArpRule(osNode, false);
diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java
index ddb130f..3af0eca 100644
--- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java
+++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/web/OpenstackManagementWebResource.java
@@ -52,6 +52,7 @@
 import java.util.Objects;
 import java.util.Optional;
 
+import static java.lang.Thread.sleep;
 import static org.onlab.util.Tools.nullIsIllegal;
 import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.addRouterIface;
 import static org.onosproject.openstacknetworking.util.OpenstackNetworkingUtil.checkArpMode;
@@ -68,7 +69,7 @@
     private static final String FLOATINGIPS = "floatingips";
     private static final String ARP_MODE_NAME = "arpMode";
 
-    private static final long TIMEOUT_MS = 10000; // we wait 10s for init each node
+    private static final long SLEEP_MS = 3000; // we wait 3s for init each node
 
     private static final String DEVICE_OWNER_IFACE = "network:router_interface";
 
@@ -299,19 +300,16 @@
             OpenstackNode updated = osNode.updateState(NodeState.INIT);
             osNodeAdminService.updateNode(updated);
 
-            long timeoutExpiredMs = System.currentTimeMillis() + TIMEOUT_MS;
-            while (updated.state() != NodeState.COMPLETE) {
-                long  waitMs = timeoutExpiredMs - System.currentTimeMillis();
+            try {
+                sleep(SLEEP_MS);
+            } catch (InterruptedException e) {
+                log.error("Exception caused during node synchronization...");
+            }
 
-                if (updated.state() == NodeState.COMPLETE) {
-                    log.info("Finished sync rules for node {}", updated.hostname());
-                    break;
-                }
-
-                if (waitMs <= 0) {
-                    log.warn("Failed to sync rules for node {}", updated.hostname());
-                    break;
-                }
+            if (osNodeService.node(osNode.hostname()).state() == NodeState.COMPLETE) {
+                log.info("Finished sync rules for node {}", osNode.hostname());
+            } else {
+                log.info("Failed to sync rules for node {}", osNode.hostname());
             }
         });
     }