Check nodes and apps in ONOSSetup

Change-Id: I81fee2c1873ef3099ff6c568715d322cea6e5ba9
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
index 6bfbb3f..50b0ca0 100644
--- a/TestON/tests/dependencies/Cluster.py
+++ b/TestON/tests/dependencies/Cluster.py
@@ -161,15 +161,11 @@
             * ips - ip( s ) of the node( s ).
         Returns:
         """
-        try:
-            apps = main.apps
-        except ( NameError, AttributeError ):
-            apps = cellApps
         self.command( "createCellFile",
                       args=[ main.ONOSbench.ip_address,
                              cellName,
                              mininetIp,
-                             apps,
+                             cellApps,
                              ips,
                              main.ONOScell.karafUser,
                              useSSH ],
@@ -420,6 +416,13 @@
         return result
 
     def nodesCheck( self ):
+        """
+        Description:
+            Checking if all the onos nodes are in READY state
+        Required:
+        Returns:
+            Returns True if it successfully checked
+        """
         results = True
         nodesOutput = self.command( "nodes", specificDriver=2 )
         ips = sorted( self.getIps( activeOnly=True ) )
@@ -441,6 +444,30 @@
             results = results and currentResult
         return results
 
+    def appsCheck( self, apps ):
+        """
+        Description:
+            Checking if all the applications are activated
+        Required:
+            apps: list of applications that are expected to be activated
+        Returns:
+            Returns True if it successfully checked
+        """
+        results = True
+        for app in apps:
+            states = self.command( "appStatus",
+                                   args=[ app ],
+                                   specificDriver=2 )
+            for i in range( len( states ) ):
+                ctrl = self.controllers[ i ]
+                if states[ i ] == "ACTIVE":
+                    results = results and True
+                    main.log.info( "{}: {} is activated".format( ctrl.name, app ) )
+                else:
+                    results = False
+                    main.log.warn( "{}: {} is in {} state".format( ctrl.name, app, states[ i ] ) )
+        return results
+
     def printResult( self, results, activeList, logLevel="debug" ):
         """
         Description:
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index 63a5b9c..440e616 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -404,6 +404,57 @@
                                  onfail="Failed to start ONOS cli" )
         return startCliResult
 
+    def checkOnosNodes( self, cluster ):
+        """
+        Description:
+            Checking if the onos nodes are in READY state
+        Required:
+            * cluster - the cluster driver that will be used.
+        Returns:
+            Returns main.TRUE if it successfully checked
+        """
+        main.step( "Checking ONOS nodes" )
+        stepResult = utilities.retry( main.Cluster.nodesCheck,
+                                      False,
+                                      attempts=9 )
+
+        utilities.assert_equals( expect=True,
+                                 actual=stepResult,
+                                 onpass="All ONOS nodes are in READY state",
+                                 onfail="Not all ONOS nodes are in READY state" )
+
+        if not stepResult:
+            for ctrl in main.Cluster.active():
+                main.log.debug( "{} components not ACTIVE: \n{}".format(
+                    ctrl.name,
+                    ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
+            main.log.error( "Failed to start ONOS, stopping test" )
+        return main.TRUE if stepResult else main.FALSE
+
+    def checkOnosApps( self, cluster, apps ):
+        """
+        Description:
+            Checking if the onos applications are activated
+        Required:
+            * cluster - the cluster driver that will be used.
+            * apps: list of applications that are expected to be activated
+        Returns:
+            Returns main.TRUE if it successfully checked
+        """
+        main.step( "Checking ONOS applications" )
+        stepResult = utilities.retry( main.Cluster.appsCheck,
+                                      False,
+                                      args = [ apps ],
+                                      sleep=5,
+                                      attempts=9 )
+
+        utilities.assert_equals( expect=True,
+                                 actual=stepResult,
+                                 onpass="All ONOS apps are activated",
+                                 onfail="Not all ONOS apps are activated" )
+
+        return main.TRUE if stepResult else main.FALSE
+
     def processList( self, functions, args ):
         if functions is not None:
             if isinstance( functions, list ):
@@ -440,7 +491,7 @@
             * startOnos - True if wish to start onos.
             * newCell - True for making a new cell and False for not making it.
             * cellName - Name of the cell that will be used.
-            * cellApps - The cell apps string.
+            * cellApps - The cell apps string. Will be overwritten by main.apps if it exists
             * mininetIp - Mininet IP address.
             * removeLog - True if wish to remove raft logs
             * extraApply - Function( s ) that will be called before building ONOS. Default to None.
@@ -473,6 +524,11 @@
         packageResult = main.TRUE
         onosCliResult = main.TRUE
         if cellApply:
+            try:
+                apps = main.apps
+            except ( NameError, AttributeError ):
+                apps = cellApps
+            main.log.debug( "Apps: " + str( apps ) )
             tempOnosIp = []
             for ctrl in cluster.runningNodes:
                 tempOnosIp.append( ctrl.ipAddress )
@@ -483,7 +539,7 @@
                         mininetIp = getattr( main, key ).ip_address
                         break
             cellResult = self.createApplyCell( cluster, newCell,
-                                               cellName, cellApps,
+                                               cellName, apps,
                                                mininetIp, useSSH,
                                                tempOnosIp, installMax )
 
@@ -510,5 +566,16 @@
         if startOnos:
             onosCliResult = self.startOnosClis( cluster )
 
+        onosNodesResult = self.checkOnosNodes( cluster )
+
+        onosAppsResult = main.TRUE
+        if cellApply:
+            if apps:
+                apps = apps.split( ',' )
+                onosAppsResult = self.checkOnosApps( cluster, apps )
+            else:
+                main.log.warn( "No apps were specified to be checked after startup" )
+
         return killResult and cellResult and packageResult and uninstallResult and \
-               installResult and secureSshResult and onosServiceResult and onosCliResult
+               installResult and secureSshResult and onosServiceResult and onosCliResult and \
+               onosNodesResult and onosAppsResult