[VOL-2950][VOL-3110][AETHER-457] Fix a bug in component readiness check

The existing code checks for onos-core-net only due to the bug

In addition, this patch introduces a new REST API to check health of given app.
Active in app status only indicates that the app has been activated.
This new API will look deeper and make sure all OSGi features, bundles and components of given app are all ready.

Change-Id: If91326ba9cffdbe25821eeaaa092ec9d2ab952ea
(cherry picked from commit 5bdaf106e4b30208d6acee6ad5bf1d58c9057d66)
diff --git a/core/api/src/main/java/org/onosproject/cluster/ComponentsMonitorService.java b/core/api/src/main/java/org/onosproject/cluster/ComponentsMonitorService.java
new file mode 100644
index 0000000..50aad17
--- /dev/null
+++ b/core/api/src/main/java/org/onosproject/cluster/ComponentsMonitorService.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2020-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cluster;
+
+import java.util.List;
+
+/**
+ * Monitors the status of OSGi components.
+ */
+public interface ComponentsMonitorService {
+    /**
+     * Checks if all given OSGi features are ready.
+     *
+     * @param features list of feature name in string
+     * @return true if all features are ready, false otherwise.
+     */
+    boolean isFullyStarted(List<String> features);
+}
diff --git a/core/net/src/main/java/org/onosproject/cluster/impl/ComponentsMonitor.java b/core/net/src/main/java/org/onosproject/cluster/impl/ComponentsMonitorManager.java
similarity index 81%
rename from core/net/src/main/java/org/onosproject/cluster/impl/ComponentsMonitor.java
rename to core/net/src/main/java/org/onosproject/cluster/impl/ComponentsMonitorManager.java
index 86636f7..551c646 100644
--- a/core/net/src/main/java/org/onosproject/cluster/impl/ComponentsMonitor.java
+++ b/core/net/src/main/java/org/onosproject/cluster/impl/ComponentsMonitorManager.java
@@ -16,6 +16,7 @@
 
 package org.onosproject.cluster.impl;
 
+import com.google.common.collect.Lists;
 import org.osgi.service.component.runtime.ServiceComponentRuntime;
 import org.osgi.service.component.annotations.Activate;
 import org.osgi.service.component.annotations.Component;
@@ -25,6 +26,7 @@
 import org.apache.karaf.features.Feature;
 import org.apache.karaf.features.FeaturesService;
 import org.onosproject.cluster.ClusterAdminService;
+import org.onosproject.cluster.ComponentsMonitorService;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleContext;
 import org.osgi.service.component.ComponentContext;
@@ -33,6 +35,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
@@ -45,8 +48,8 @@
  * are properly activated and keeps the cluster node service appropriately
  * updated.
  */
-@Component(immediate = true)
-public class ComponentsMonitor {
+@Component(immediate = true, service = { ComponentsMonitorService.class })
+public class ComponentsMonitorManager implements ComponentsMonitorService {
 
     private Logger log = LoggerFactory.getLogger(getClass());
 
@@ -133,7 +136,7 @@
     private boolean isFullyStarted() {
         try {
             for (Feature feature : featuresService.listInstalledFeatures()) {
-                if (!isFullyStarted(feature)) {
+                if (needToCheck(feature) && !isFullyStarted(feature)) {
                     return false;
                 }
             }
@@ -149,20 +152,30 @@
                !feature.getId().contains("thirdparty");
     }
 
-    private boolean isFullyStarted(Feature feature) {
-        if (needToCheck(feature)) {
+    @Override
+    public boolean isFullyStarted(List<String> featureStrings) {
+        List<Feature> features = Lists.newArrayList();
+        for (String featureString : featureStrings) {
             try {
-                return feature.getBundles().stream()
-                    .map(info -> bundleContext.getBundle())
-                    .allMatch(this::isFullyStarted);
-            } catch (NullPointerException npe) {
-                // FIXME: Remove this catch block when Felix fixes the bug
-                // Due to a bug in the Felix implementation, this can throw an NPE.
-                // Catch the error and do something sensible with it.
+                features.add(featuresService.getFeature(featureString));
+            } catch (Exception e) {
+                log.debug("Feature {} not found", featureString);
                 return false;
             }
-        } else {
-            return true;
+        }
+        return features.stream().allMatch(this::isFullyStarted);
+    }
+
+    private boolean isFullyStarted(Feature feature) {
+        try {
+            return feature.getBundles().stream()
+                .map(info -> bundleContext.getBundle(info.getLocation()))
+                .allMatch(bundle -> bundle != null && isFullyStarted(bundle));
+        } catch (NullPointerException npe) {
+            // FIXME: Remove this catch block when Felix fixes the bug
+            // Due to a bug in the Felix implementation, this can throw an NPE.
+            // Catch the error and do something sensible with it.
+            return false;
         }
     }
 
diff --git a/web/api/src/main/java/org/onosproject/rest/resources/ApplicationsWebResource.java b/web/api/src/main/java/org/onosproject/rest/resources/ApplicationsWebResource.java
index c73c612..b2ee977 100644
--- a/web/api/src/main/java/org/onosproject/rest/resources/ApplicationsWebResource.java
+++ b/web/api/src/main/java/org/onosproject/rest/resources/ApplicationsWebResource.java
@@ -18,6 +18,7 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.onosproject.app.ApplicationAdminService;
 import org.onosproject.app.ApplicationException;
+import org.onosproject.cluster.ComponentsMonitorService;
 import org.onosproject.core.Application;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.CoreService;
@@ -50,6 +51,8 @@
 
     private static final String APP_ID_NOT_FOUND = "Application ID is not found";
     private static final String APP_NOT_FOUND = "Application is not found";
+    private static final String APP_READY = "ready";
+    private static final String APP_PENDING = "pending";
 
     private static final String URL = "url";
     private static final String ACTIVATE = "activate";
@@ -85,6 +88,23 @@
     }
 
     /**
+     * Get application health.
+     *
+     * @param name application name
+     * @return 200 OK with app health in the body; 404 if app is not found
+     */
+    @GET
+    @Path("{name}/health")
+    public Response health(@PathParam("name") String name) {
+        ApplicationAdminService service = get(ApplicationAdminService.class);
+        ApplicationId appId = nullIsNotFound(service.getId(name), APP_NOT_FOUND);
+
+        ComponentsMonitorService componentsMonitorService = get(ComponentsMonitorService.class);
+        boolean ready = componentsMonitorService.isFullyStarted(service.getApplication(appId).features());
+        return Response.ok(mapper().createObjectNode().put("message", ready ? APP_READY : APP_PENDING)).build();
+    }
+
+    /**
      * Install a new application.
      * Uploads application archive stream and optionally activates the
      * application.