Added ability to track whether or not node has all components running fully.

Change-Id: Ib2b90c7a842976a3b3a9711367fa1eed43103b17
diff --git a/core/net/pom.xml b/core/net/pom.xml
index 9f611c7..026a34d 100644
--- a/core/net/pom.xml
+++ b/core/net/pom.xml
@@ -80,6 +80,11 @@
         </dependency>
 
         <dependency>
+            <groupId>org.osgi</groupId>
+            <artifactId>org.osgi.core</artifactId>
+        </dependency>
+
+        <dependency>
             <groupId>org.onosproject</groupId>
             <artifactId>onos-incubator-api</artifactId>
         </dependency>
@@ -93,6 +98,11 @@
             <groupId>org.apache.karaf.system</groupId>
             <artifactId>org.apache.karaf.system.core</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.felix</groupId>
+            <artifactId>org.apache.felix.scr</artifactId>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/ClusterManager.java b/core/net/src/main/java/org/onosproject/cluster/impl/ClusterManager.java
index 904fdff..7c70b8a 100644
--- a/core/net/src/main/java/org/onosproject/cluster/impl/ClusterManager.java
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/ClusterManager.java
@@ -15,6 +15,8 @@
  */
 package org.onosproject.cluster.impl;
 
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Sets;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -44,9 +46,6 @@
 import org.onosproject.event.AbstractListenerManager;
 import org.slf4j.Logger;
 
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Sets;
-
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -58,8 +57,8 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.onosproject.security.AppGuard.checkPermission;
+import static org.onosproject.security.AppPermission.Type.CLUSTER_READ;
 import static org.slf4j.LoggerFactory.getLogger;
-import static org.onosproject.security.AppPermission.Type.*;
 
 /**
  * Implementation of the cluster service.
@@ -133,6 +132,10 @@
         return store.getState(nodeId);
     }
 
+    @Override
+    public void markFullyStarted(boolean started) {
+        store.markFullyStarted(started);
+    }
 
     @Override
     public DateTime getLastUpdated(NodeId nodeId) {
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/ComponentsMonitor.java b/core/net/src/main/java/org/onosproject/cluster/impl/ComponentsMonitor.java
new file mode 100644
index 0000000..5d670a3
--- /dev/null
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/ComponentsMonitor.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2016 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.cluster.impl;
+
+import org.apache.felix.scr.Component;
+import org.apache.felix.scr.ScrService;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.karaf.features.Feature;
+import org.apache.karaf.features.FeaturesService;
+import org.onlab.util.SharedScheduledExecutors;
+import org.onosproject.cluster.ClusterAdminService;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Monitors the system to make sure that all bundles and their components
+ * are properly activated and keeps the cluster node service appropriately
+ * updated.
+ */
+@org.apache.felix.scr.annotations.Component(immediate = true)
+public class ComponentsMonitor {
+
+    private Logger log = LoggerFactory.getLogger(getClass());
+
+    private static final long PERIOD = 2500;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected FeaturesService featuresService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ScrService scrService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterAdminService clusterAdminService;
+
+    private BundleContext bundleContext;
+    private ScheduledFuture<?> poller;
+
+    @Activate
+    protected void activate(ComponentContext context) {
+        bundleContext = context.getBundleContext();
+        poller = SharedScheduledExecutors.getSingleThreadExecutor()
+                .scheduleAtFixedRate(this::checkStartedState, PERIOD,
+                                     PERIOD, TimeUnit.MILLISECONDS);
+        log.info("Started");
+    }
+
+    @Deactivate
+    protected void deactivate() {
+        poller.cancel(false);
+        log.info("Stopped");
+    }
+
+    private void checkStartedState() {
+        clusterAdminService.markFullyStarted(isFullyStarted());
+    }
+
+    /**
+     * Scans the system to make sure that all bundles and their components
+     * are fully started.
+     *
+     * @return true if all bundles and their components are active
+     */
+    private boolean isFullyStarted() {
+        for (Feature feature : featuresService.listInstalledFeatures()) {
+            if (!isFullyStarted(feature)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean isFullyStarted(Feature feature) {
+        return feature.getBundles().stream()
+                .map(info -> bundleContext.getBundle(info.getLocation()))
+                .allMatch(this::isFullyStarted);
+    }
+
+    private boolean isFullyStarted(Bundle bundle) {
+        Component[] components = scrService.getComponents(bundle);
+        if (components != null) {
+            for (Component component : components) {
+                if (!isFullyStarted(component)) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    private boolean isFullyStarted(Component component) {
+        int state = component.getState();
+        return state == Component.STATE_ACTIVE || state == Component.STATE_DISABLED ||
+                (state == Component.STATE_REGISTERED && !component.isImmediate());
+    }
+
+}
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/LeadershipManager.java b/core/net/src/main/java/org/onosproject/cluster/impl/LeadershipManager.java
index e9a78d6..9ceffee 100644
--- a/core/net/src/main/java/org/onosproject/cluster/impl/LeadershipManager.java
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/LeadershipManager.java
@@ -76,7 +76,7 @@
         deadlockDetector.scheduleWithFixedDelay(() -> clusterService.getNodes()
                 .stream()
                 .map(ControllerNode::id)
-                .filter(id -> clusterService.getState(id) != ControllerNode.State.ACTIVE)
+                .filter(id -> !clusterService.getState(id).isActive())
                 .forEach(this::unregister), 0, 2, TimeUnit.SECONDS);
         log.info("Started");
     }
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/MastershipManager.java b/core/net/src/main/java/org/onosproject/cluster/impl/MastershipManager.java
index 50a3350..e746d92 100644
--- a/core/net/src/main/java/org/onosproject/cluster/impl/MastershipManager.java
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/MastershipManager.java
@@ -30,8 +30,8 @@
 import org.onosproject.cluster.ControllerNode;
 import org.onosproject.cluster.NodeId;
 import org.onosproject.cluster.RoleInfo;
-import org.onosproject.event.AbstractListenerManager;
 import org.onosproject.core.MetricsHelper;
+import org.onosproject.event.AbstractListenerManager;
 import org.onosproject.mastership.MastershipAdminService;
 import org.onosproject.mastership.MastershipEvent;
 import org.onosproject.mastership.MastershipListener;
@@ -57,11 +57,11 @@
 import static com.google.common.collect.Lists.newArrayList;
 import static org.onlab.metrics.MetricsUtil.startTimer;
 import static org.onlab.metrics.MetricsUtil.stopTimer;
-import static org.onosproject.cluster.ControllerNode.State.ACTIVE;
 import static org.onosproject.net.MastershipRole.MASTER;
 import static org.onosproject.security.AppGuard.checkPermission;
+import static org.onosproject.security.AppPermission.Type.CLUSTER_READ;
+import static org.onosproject.security.AppPermission.Type.CLUSTER_WRITE;
 import static org.slf4j.LoggerFactory.getLogger;
-import static org.onosproject.security.AppPermission.Type.*;
 
 
 
@@ -204,7 +204,7 @@
 
         // Create buckets reflecting current ownership.
         for (ControllerNode node : nodes) {
-            if (clusterService.getState(node.id()) == ACTIVE) {
+            if (clusterService.getState(node.id()).isActive()) {
                 Set<DeviceId> devicesOf = new HashSet<>(getDevicesOf(node.id()));
                 deviceCount += devicesOf.size();
                 controllerDevices.put(node, devicesOf);