Merge "Refactor tests to use ONOSSetup.checkOnosNodes function"
diff --git a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
index d770e83..16ca659 100644
--- a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
+++ b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
@@ -210,23 +210,7 @@
                                  onfail="ONOS nodes NOT successfully stopped" )
         main.Cluster.reset()
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
         main.killCount += 1
 
diff --git a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
index 9020edc..0aba15f 100644
--- a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
+++ b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
@@ -346,23 +346,7 @@
             main.Cluster.runningNodes[ node ].active = True
         main.Cluster.reset()
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=50 )
-
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE7( self, main ):
         """
diff --git a/TestON/tests/HA/HAkillNodes/HAkillNodes.py b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
index 1be271f..f826e91 100644
--- a/TestON/tests/HA/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HA/HAkillNodes/HAkillNodes.py
@@ -203,23 +203,7 @@
                                  onpass="ONOS nodes killed successfully",
                                  onfail="ONOS nodes NOT successfully killed" )
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE62( self, main ):
         """
diff --git a/TestON/tests/HA/HApowerFailure/HApowerFailure.py b/TestON/tests/HA/HApowerFailure/HApowerFailure.py
index aa04792..e395b9f 100644
--- a/TestON/tests/HA/HApowerFailure/HApowerFailure.py
+++ b/TestON/tests/HA/HApowerFailure/HApowerFailure.py
@@ -209,15 +209,7 @@
                                  onpass="ONOS nodes killed successfully",
                                  onfail="ONOS nodes NOT successfully killed" )
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
         if not nodeResults:
             for ctrl in main.Cluster.active():
diff --git a/TestON/tests/HA/HAscaling/HAscaling.py b/TestON/tests/HA/HAscaling/HAscaling.py
index 19b0ec3..4870d59 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.py
+++ b/TestON/tests/HA/HAscaling/HAscaling.py
@@ -245,13 +245,7 @@
 
         main.Cluster.startCLIs()
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       attempts=90 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
         for i in range( 10 ):
             ready = True
diff --git a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index c23284b..c6ef4d4 100644
--- a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -427,20 +427,7 @@
         utilities.assert_equals( expect=main.TRUE, actual=topoResult,
                                  onpass="Topology Check Test successful",
                                  onfail="Topology Check Test NOT successful" )
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       attempts=5 )
-
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-
+        main.testSetUp.checkOnosNodes( main.Cluster )
         if not topoResult:
             main.cleanAndExit()
 
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.py b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
index 0c6c7d0..6998896 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
@@ -201,34 +201,14 @@
                                  onpass="ONOS nodes stopped successfully",
                                  onfail="ONOS nodes NOT successfully stopped" )
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=50 )
-
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE62( self, main ):
         """
         The bring up stopped nodes
         """
         main.HA.bringUpStoppedNodes( main )
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=50 )
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE7( self, main ):
         """
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.py b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
index e2efe92..4542002 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.py
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
@@ -232,13 +232,7 @@
 
         main.testSetUp.startOnosClis( main.Cluster )
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       attempts=15 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
         ready = utilities.retry( main.Cluster.command,
                                  False,
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.py b/TestON/tests/HA/HAupgrade/HAupgrade.py
index cea03c8..355d87b 100644
--- a/TestON/tests/HA/HAupgrade/HAupgrade.py
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.py
@@ -202,22 +202,7 @@
             main.kill.append( main.Cluster.runningNodes[ i ] )  # ONOS node to kill, listed by index in main.nodes
         main.HA.upgradeNodes( main )
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE62( self, main ):
         """
@@ -240,14 +225,7 @@
         main.log.debug( status )
         # TODO: check things here?
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE63( self, main ):
         """
@@ -261,15 +239,7 @@
                 main.kill.append( node )
 
         main.HA.upgradeNodes( main )
-        main.step( "Checking ONOS nodes" )
-
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE64( self, main ):
         """
diff --git a/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
index f5ae894..0f10515 100644
--- a/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
+++ b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
@@ -202,22 +202,7 @@
             main.kill.append( main.Cluster.runningNodes[ i ] )  # ONOS node to kill, listed by index in main.nodes
         main.HA.upgradeNodes( main )
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE62( self, main ):
         """
@@ -240,14 +225,7 @@
         main.log.debug( status )
         # TODO: check things here?
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE63( self, main ):
         """
@@ -274,22 +252,7 @@
             ctrl.onosInstall( options="-f", node=ctrl.ipAddress )
             ctrl.onosSecureSSH( node=ctrl.ipAddress )
             ctrl.startOnosCli( ctrl.ipAddress, waitForStart=True )
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def CASE64( self, main ):
         """
diff --git a/TestON/tests/HA/dependencies/HA.py b/TestON/tests/HA/dependencies/HA.py
index c809ffe..adc4d5f 100644
--- a/TestON/tests/HA/dependencies/HA.py
+++ b/TestON/tests/HA/dependencies/HA.py
@@ -29,6 +29,8 @@
     def __init__( self ):
         self.default = ''
         main.topoMappings = {}
+        from tests.dependencies.ONOSSetup import ONOSSetup
+        main.testSetUp = ONOSSetup()
 
     def removeKarafConsoleLogging( self ):
         main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
@@ -259,22 +261,7 @@
             main.ONOSbench.handle.sendline( "git checkout -- tools/package/init/onos.service" )
             main.ONOSbench.handle.expect( "\$" )
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       attempts=90 )
-
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
         main.step( "Activate apps defined in the params file" )
         # get data from the params
@@ -2708,23 +2695,7 @@
         main.restartTime = time.time() - restartTime
         main.log.debug( "Restart time: " + str( main.restartTime ) )
         # TODO: MAke this configurable. Also, we are breaking the above timer
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
         self.commonChecks()
 
@@ -2797,23 +2768,7 @@
         main.restartTime = time.time() - restartTime
         main.log.debug( "Restart time: " + str( main.restartTime ) )
         # TODO: Make this configurable.
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-            main.log.error( "Failed to start ONOS, stopping test" )
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
         self.commonChecks()
 
@@ -3387,21 +3342,7 @@
             onfail="Hosts are incorrect" )
 
         # FIXME: move this to an ONOS state case
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       attempts=5 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
-        if not nodeResults:
-            for ctrl in main.Cluster.active():
-                main.log.debug( "{} components not ACTIVE: \n{}".format(
-                    ctrl.name,
-                    ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
-
-        if not topoResult:
-            main.cleanAndExit()
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def linkDown( self, main, src="s3", dst="s28" ):
         """
@@ -3753,14 +3694,7 @@
         main.log.debug( status )
         # TODO: check things here?
 
-        main.step( "Checking ONOS nodes" )
-        nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                       False,
-                                       sleep=15,
-                                       attempts=5 )
-        utilities.assert_equals( expect=True, actual=nodeResults,
-                                 onpass="Nodes check successful",
-                                 onfail="Nodes check NOT successful" )
+        main.testSetUp.checkOnosNodes( main.Cluster )
 
     def backupData( self, main, location ):
         """
diff --git a/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.py b/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.py
index c8dd52b..4a0fa3e 100644
--- a/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.py
+++ b/TestON/tests/USECASE/VPLS/VPLSfailsafe/VPLSfailsafe.py
@@ -420,15 +420,7 @@
             # Run some basic checks to see if ONOS node truly has succesfully restarted:
 
             # Checking if all nodes appear with status READY using 'nodes' command
-            main.step( "Checking ONOS nodes." )
-            nodeResults = utilities.retry( main.Cluster.nodesCheck,
-                                           False,
-                                           sleep=main.timeSleep,
-                                           attempts=main.numAttempts )
-
-            utilities.assert_equals( expect=True, actual=nodeResults,
-                                     onpass="Nodes check successful.",
-                                     onfail="Nodes check NOT successful." )
+            main.testSetUp.checkOnosNodes( main.Cluster )
 
             # All apps that are present are active
             main.log.info( "Checking if apps are active." )