Fix race condition between AtomixManager and metadata providers startup

Change-Id: I4799c079455e0e5c79a800ba3108b4c9eedbe1b2
diff --git a/core/net/BUILD b/core/net/BUILD
index 8cd9b48..571c45a 100644
--- a/core/net/BUILD
+++ b/core/net/BUILD
@@ -5,6 +5,7 @@
     "//incubator/net:onos-incubator-net",
     "//incubator/store:onos-incubator-store",
     "//core/store/serializers:onos-core-serializers",
+    "//core/store/primitives:onos-core-primitives",
     "@org_osgi_service_cm//jar",
 ]
 
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/ClusterMetadataManager.java b/core/net/src/main/java/org/onosproject/cluster/impl/ClusterMetadataManager.java
index 3cf47ba..df77d09 100644
--- a/core/net/src/main/java/org/onosproject/cluster/impl/ClusterMetadataManager.java
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/ClusterMetadataManager.java
@@ -15,8 +15,8 @@
  */
 package org.onosproject.cluster.impl;
 
+import com.google.common.collect.ImmutableList;
 import org.onlab.packet.IpAddress;
-import org.onlab.util.Tools;
 import org.onosproject.cluster.ClusterMetadata;
 import org.onosproject.cluster.ClusterMetadataAdminService;
 import org.onosproject.cluster.ClusterMetadataEvent;
@@ -31,10 +31,14 @@
 import org.onosproject.cluster.PartitionId;
 import org.onosproject.net.provider.AbstractListenerProviderRegistry;
 import org.onosproject.net.provider.AbstractProviderService;
+import org.onosproject.net.provider.ProviderId;
+import org.onosproject.store.atomix.ClusterActivator;
 import org.onosproject.store.service.Versioned;
 import org.osgi.service.component.annotations.Activate;
 import org.osgi.service.component.annotations.Component;
 import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
 import org.slf4j.Logger;
 
 import java.net.Inet4Address;
@@ -45,6 +49,9 @@
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.util.Enumeration;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.onosproject.security.AppGuard.checkPermission;
@@ -67,12 +74,16 @@
     private static final int MAX_WAIT_TRIES = 600;
     private static final int MAX_WAIT_MS = 100;
 
+    private List<String> requiredProviders = ImmutableList.of("file", "default");
+
     private final Logger log = getLogger(getClass());
     private ControllerNode localNode;
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    private ClusterActivator clusterActivator;
+
     @Activate
     public void activate() {
-        // FIXME: Need to ensure all cluster metadata providers are registered before we activate
         eventDispatcher.addSink(ClusterMetadataEvent.class, listenerRegistry);
         log.info("Started");
     }
@@ -84,22 +95,22 @@
     }
 
     @Override
-    public ClusterMetadata getClusterMetadata() {
-        checkPermission(CLUSTER_READ);
-        waitForAvailableProvider();
-        Versioned<ClusterMetadata> metadata = getProvider().getClusterMetadata();
-        return metadata.value();
+    public synchronized ClusterMetadataProviderService register(ClusterMetadataProvider provider) {
+        ClusterMetadataProviderService s = super.register(provider);
+        Set<String> providerNames = getProviders().stream().map(ProviderId::scheme).collect(Collectors.toSet());
+        if (providerNames.containsAll(requiredProviders)) {
+            // Safe to release Atomix now, cluster metadata is ready
+            clusterActivator.activateCluster();
+        }
+        return s;
     }
 
-    // FIXME: Temporary hack to navigate around bootstrap timing issue
-    private void waitForAvailableProvider() {
-        for (int i = 0; i < MAX_WAIT_TRIES && getProvider() == null; i++) {
-            Tools.delay(MAX_WAIT_MS);
-        }
-        if (getProvider() == null) {
-            log.error("Unable to find cluster metadata provider");
-            throw new IllegalStateException("Unable to find cluster metadata provider");
-        }
+
+    @Override
+    public ClusterMetadata getClusterMetadata() {
+        checkPermission(CLUSTER_READ);
+        Versioned<ClusterMetadata> metadata = getProvider().getClusterMetadata();
+        return metadata.value();
     }
 
     @Override
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/atomix/ClusterActivator.java b/core/store/primitives/src/main/java/org/onosproject/store/atomix/ClusterActivator.java
new file mode 100644
index 0000000..125ceab
--- /dev/null
+++ b/core/store/primitives/src/main/java/org/onosproject/store/atomix/ClusterActivator.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.store.atomix;
+
+import org.onosproject.component.ComponentService;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+@Component(immediate = true, service = ClusterActivator.class)
+public class ClusterActivator {
+    private final Logger log = getLogger(getClass());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY)
+    private ComponentService componentService;
+
+    @Activate
+    public void activate() {
+        log.info("Started");
+    }
+
+    /**
+     * Resources needed by the cluster are now available and the Atomix
+     * cluster can be formed.
+     */
+    public void activateCluster() {
+        componentService.activate(null, "org.onosproject.store.atomix.impl.AtomixManager");
+    }
+}
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/atomix/impl/AtomixManager.java b/core/store/primitives/src/main/java/org/onosproject/store/atomix/impl/AtomixManager.java
index 8e6b9c0..1135fda 100644
--- a/core/store/primitives/src/main/java/org/onosproject/store/atomix/impl/AtomixManager.java
+++ b/core/store/primitives/src/main/java/org/onosproject/store/atomix/impl/AtomixManager.java
@@ -36,7 +36,7 @@
 /**
  * Atomix manager.
  */
-@Component(immediate = true, service = AtomixManager.class)
+@Component(immediate = true, enabled = false, service = AtomixManager.class)
 public class AtomixManager {
     private static final String LOCAL_DATA_DIR = System.getProperty("karaf.data") + "/db/partitions/";
     private final Logger log = LoggerFactory.getLogger(getClass());
diff --git a/core/store/primitives/src/main/java/org/onosproject/store/atomix/package-info.java b/core/store/primitives/src/main/java/org/onosproject/store/atomix/package-info.java
new file mode 100644
index 0000000..5613aa7
--- /dev/null
+++ b/core/store/primitives/src/main/java/org/onosproject/store/atomix/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2014-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Subsystem for tracking controller cluster nodes.
+ */
+package org.onosproject.store.atomix;