Unify onos startup checks

Change-Id: Ia82f696839608031f67aec8f944156d6231b0177
diff --git a/TestON/tests/CHO/CHOtest/CHOtest.py b/TestON/tests/CHO/CHOtest/CHOtest.py
index 411035d..41f1693 100644
--- a/TestON/tests/CHO/CHOtest/CHOtest.py
+++ b/TestON/tests/CHO/CHOtest/CHOtest.py
@@ -134,16 +134,6 @@
                                      onfail="Test step FAIL" )
             installResult = ( installResult and i_result )
 
-        main.step( "Verify ONOS nodes UP status" )
-        statusResult = main.TRUE
-        for i in range( int( main.numCtrls ) ):
-            main.log.info( "ONOS Node " + main.onosIPs[i] + " status:" )
-            onos_status = main.ONOSbench.onosStatus( node=main.onosIPs[i] )
-            utilities.assert_equals( expect=main.TRUE, actual=onos_status,
-                                     onpass="Test step PASS",
-                                     onfail="Test step FAIL" )
-            statusResult = ( statusResult and onos_status )
-
         main.step( "Set up ONOS secure SSH" )
         secureSshResult = main.TRUE
         for i in range( int( main.numCtrls ) ):
@@ -152,6 +142,30 @@
                                  onpass="Test step PASS",
                                  onfail="Test step FAIL" )
 
+        time.sleep( 5 )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+
         main.step( "Start ONOS CLI on all nodes" )
         cliResult = main.TRUE
         main.step(" Start ONOS cli using thread ")
diff --git a/TestON/tests/CHOTestMonkey/CHOTestMonkey.py b/TestON/tests/CHOTestMonkey/CHOTestMonkey.py
index ec64601..78bed55 100644
--- a/TestON/tests/CHOTestMonkey/CHOTestMonkey.py
+++ b/TestON/tests/CHOTestMonkey/CHOTestMonkey.py
@@ -147,16 +147,6 @@
                                      onfail="Test step FAIL" )
             installResult = ( installResult and iResult )
 
-        main.step( "Verify ONOS nodes UP status" )
-        statusResult = main.TRUE
-        for i in range( int( main.numCtrls ) ):
-            main.log.info( "ONOS Node " + main.onosIPs[i] + " status:" )
-            onos_status = main.ONOSbench.onosStatus( node=main.onosIPs[i] )
-            utilities.assert_equals( expect=main.TRUE, actual=onos_status,
-                                     onpass="Test step PASS",
-                                     onfail="Test step FAIL" )
-            statusResult = ( statusResult and onos_status )
-
         main.step( "Set up ONOS secure SSH" )
         secureSshResult = main.TRUE
         for i in range( int( main.numCtrls ) ):
@@ -165,6 +155,30 @@
                                  onpass="Test step PASS",
                                  onfail="Test step FAIL" )
 
+        time.sleep( 5 )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+
         main.step( "Start ONOS CLI on all nodes" )
         cliResult = main.TRUE
         startCliResult  = main.TRUE
diff --git a/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.py b/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.py
index 42159b4..6c382e4 100644
--- a/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.py
+++ b/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.py
@@ -167,16 +167,6 @@
                                      onpass="Test step PASS",
                                      onfail="Test step FAIL" )
             installResult = installResult and i_result
-        time.sleep( main.startUpSleep )
-        main.step( "Verify ONOS nodes UP status" )
-        statusResult = main.TRUE
-        for i in range( int( main.numCtrls ) ):
-            main.log.info( "ONOS Node " + main.ONOSip[i] + " status:" )
-            onos_status = main.ONOSbench.onosStatus( node=main.ONOSip[i] )
-            utilities.assert_equals( expect=main.TRUE, actual=onos_status,
-                                     onpass="Test step PASS",
-                                     onfail="Test step FAIL" )
-            statusResult = ( statusResult and onos_status )
 
         main.step( "Set up ONOS secure SSH" )
         secureSshResult = main.TRUE
@@ -186,6 +176,31 @@
                                  onpass="Test step PASS",
                                  onfail="Test step FAIL" )
 
+        time.sleep( main.startUpSleep )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+
         main.step( "Start ONOS CLI on all nodes" )
         cliResult = main.TRUE
         main.step(" Start ONOS cli using thread ")
diff --git a/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.py b/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.py
index 42c3522..3e221a5 100644
--- a/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.py
+++ b/TestON/tests/SCPF/SCPFintentEventTp/SCPFintentEventTp.py
@@ -166,17 +166,6 @@
                                     onfail="Test step FAIL")
             installResult = installResult and i_result
 
-        main.step( "Verify ONOS nodes UP status" )
-        statusResult = main.TRUE
-        for i in range( int( main.numCtrls ) ):
-            main.log.info( "ONOS Node " + main.ONOSip[i] + " status:" )
-            onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
-            utilities.assert_equals(expect=main.TRUE, actual=onos_status,
-                                    onpass="Test step PASS",
-                                    onfail="Test step FAIL")
-            statusResult = (statusResult and onos_status)
-        time.sleep(2)
-
         main.step( "Set up ONOS secure SSH" )
         secureSshResult = main.TRUE
         for i in range( int( main.numCtrls ) ):
@@ -185,6 +174,30 @@
                                  onpass="Test step PASS",
                                  onfail="Test step FAIL" )
 
+        time.sleep( main.startUpSleep )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+
         main.step( "Start ONOS cli using thread" )
         startCliResult = main.TRUE
         pool = []
diff --git a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py
index 1c04fd1..412ef29 100644
--- a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py
+++ b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.py
@@ -164,16 +164,6 @@
                                     onfail="Test step FAIL")
             installResult = installResult and i_result
 
-        main.step("Verify ONOS nodes UP status")
-        statusResult = main.TRUE
-        for i in range(int(main.numCtrls)):
-            main.log.info("ONOS Node " + main.ONOSip[i] + " status:")
-            onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
-            utilities.assert_equals(expect=main.TRUE, actual=onos_status,
-                                    onpass="Test step PASS",
-                                    onfail="Test step FAIL")
-            statusResult = (statusResult and onos_status)
-
         main.step( "Set up ONOS secure SSH" )
         secureSshResult = main.TRUE
         for i in range( int( main.numCtrls ) ):
@@ -182,6 +172,30 @@
                                  onpass="Test step PASS",
                                  onfail="Test step FAIL" )
 
+        time.sleep( main.startUpSleep )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+
         time.sleep(2)
         main.step("Start ONOS CLI on all nodes")
         cliResult = main.TRUE
diff --git a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.py b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.py
index 59dc3a1..ae38911 100644
--- a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.py
+++ b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.py
@@ -168,17 +168,6 @@
                                     onfail="Test step FAIL")
             installResult = installResult and i_result
 
-        main.step( "Verify ONOS nodes UP status" )
-        statusResult = main.TRUE
-        for i in range( int( main.numCtrls ) ):
-            main.log.info( "ONOS Node " + main.ONOSip[i] + " status:" )
-            onos_status = main.ONOSbench.onosStatus( node=main.ONOSip[i] )
-            utilities.assert_equals(expect=main.TRUE, actual=onos_status,
-                                    onpass="Test step PASS",
-                                    onfail="Test step FAIL")
-            statusResult = statusResult and onos_status
-        time.sleep(2)
-
         main.step( "Set up ONOS secure SSH" )
         secureSshResult = main.TRUE
         for i in range( int( main.numCtrls ) ):
@@ -187,6 +176,30 @@
                                  onpass="Test step PASS",
                                  onfail="Test step FAIL" )
 
+        time.sleep( main.startUpSleep )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+
         main.step( "Start ONOS CLI on all nodes" )
         cliResult = main.TRUE
         main.step( " Start ONOS cli using thread " )
diff --git a/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.py b/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.py
index a7a7bc5..46714ac 100644
--- a/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.py
+++ b/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.py
@@ -192,16 +192,6 @@
                                      onfail="Test step FAIL" )
             installResult = installResult and i_result
 
-        main.step( "Verify ONOS nodes UP status" )
-        statusResult = main.TRUE
-        for i in range( int( main.numCtrls ) ):
-            main.log.info( "ONOS Node " + main.ONOSip[i] + " status:" )
-            onos_status = main.ONOSbench.onosStatus( node=main.ONOSip[i] )
-            utilities.assert_equals( expect=main.TRUE, actual=onos_status,
-                                     onpass="Test step PASS",
-                                     onfail="Test step FAIL" )
-            statusResult = ( statusResult and onos_status )
-
         main.step( "Set up ONOS secure SSH" )
         secureSshResult = main.TRUE
         for i in range( int( main.numCtrls ) ):
@@ -210,6 +200,30 @@
                                  onpass="Test step PASS",
                                  onfail="Test step FAIL" )
 
+        time.sleep( main.startUpSleep )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+
         main.step( "Start ONOS CLI on all nodes" )
         cliResult = main.TRUE
         main.step(" Start ONOS cli using thread ")
diff --git a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
index 3fef087..46a5f47 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
+++ b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
@@ -171,16 +171,6 @@
                                     onfail="Test step FAIL")
             installResult = installResult and i_result
 
-        main.step("Verify ONOS nodes UP status")
-        statusResult = main.TRUE
-        for i in range(int(main.numCtrls)):
-            main.log.info("ONOS Node " + main.ONOSip[i] + " status:")
-            onos_status = main.ONOSbench.onosStatus(node=main.ONOSip[i])
-            utilities.assert_equals(expect=main.TRUE, actual=onos_status,
-                                    onpass="Test step PASS",
-                                    onfail="Test step FAIL")
-            statusResult = (statusResult and onos_status)
-
         main.step( "Set up ONOS secure SSH" )
         secureSshResult = main.TRUE
         for i in range( int( main.numCtrls ) ):
@@ -189,6 +179,30 @@
                                  onpass="Test step PASS",
                                  onfail="Test step FAIL" )
 
+        time.sleep( main.startUpSleep )
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+
         time.sleep(2)
         main.step("Start ONOS CLI on all nodes")
         cliResult = main.TRUE