[ONOS-6593]Review and Refactor ONOS startup procedures in TestON

Change-Id: I509a8ee7a26c198957bebf59da5c85a0edb8b995
diff --git a/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.py b/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.py
index 5a81c23..1106f83 100644
--- a/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.py
+++ b/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.py
@@ -21,105 +21,70 @@
         import time
         import os
         import imp
-
-        main.case( "Constructing test variables and building ONOS package" )
-        main.step( "Constructing test variables" )
-        stepResult = main.FALSE
-
-        # Test variables
-        main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
-        main.dependencyPath = main.testOnDirectory + \
-                main.params['DEPENDENCY']['path']
-        main.cellName = main.params[ 'ENV' ][ 'cellName' ]
-        main.apps = main.params[ 'ENV' ][ 'cellApps' ]
-        main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
-        main.scale = ( main.params[ 'SCALE' ] ).split( "," )
-        main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
-        main.timeout = int(main.params['SLEEP']['timeout'])
-        main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
-        main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] )
-        main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] )
-        main.rerouteSleep = int ( main.params['SLEEP']['reroute'] )
-        main.verifyAttempts = int( main.params['ATTEMPTS']['verify'] )
-        main.ingress = main.params['LINK']['ingress']
-        main.egress = main.params['LINK']['egress']
-        main.cellData = {} # for creating cell file
-        main.reroute = main.params['reroute']
-        main.flowObj = main.params['TEST']['flowObj']
-        if main.flowObj == "True":
-            main.flowObj = True
-            main.dbFileName = main.params['DATABASE']['dbFlowObj']
-        else:
-            main.flowObj = False
-            main.dbFileName = main.params['DATABASE']['dbName']
-        main.threadID = 0
-
-        if main.reroute == "True":
-            main.reroute = True
-        else:
-            main.reroute = False
-
-        main.CLIs = []
-        main.setupSkipped = False
-
-        wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
-        gitBranch = main.params[ 'GIT' ][ 'branch' ]
-        gitPull = main.params[ 'GIT' ][ 'pull' ]
-        nic = main.params['DATABASE']['nic']
-        node = main.params['DATABASE']['node']
-        nic = main.params['DATABASE']['nic']
-        node = main.params['DATABASE']['node']
-        stepResult = main.TRUE
-
-        main.log.info("Cresting DB file")
-        with open(main.dbFileName, "w+") as dbFile:
-            dbFile.write("")
-
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=stepResult,
-                                 onpass="environment set up successfull",
-                                 onfail="environment set up Failed" )
-
-    def CASE1( self ):
-        # main.scale[ 0 ] determines the current number of ONOS controller
-        main.CLIs = []
-        main.numCtrls = int( main.scale[ 0 ] )
-        main.ONOSip = []
-        main.maxNumBatch = 0
-        main.AllONOSip = main.ONOSbench.getOnosIps()
-        for i in range(main.numCtrls):
-            main.ONOSip.append(main.AllONOSip[i])
-        main.log.info(main.ONOSip)
-
-        main.log.info( "Creating list of ONOS cli handles" )
-        for i in range(main.numCtrls):
-            main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
-
-        main.log.info(main.CLIs)
-        if not main.CLIs:
-            main.log.error( "Failed to create the list of ONOS cli handles" )
-            main.cleanup()
+        try:
+            from tests.dependencies.ONOSSetup import ONOSSetup
+            main.testSetUp = ONOSSetup()
+        except ImportError:
+            main.log.error( "ONOSSetup not found. exiting the test" )
             main.exit()
+        main.testSetUp.envSetupDescription()
+        stepResult = main.FALSE
+        try:
+            # Test variables
+            main.dependencyPath = main.testOnDirectory + \
+                    main.params[ 'DEPENDENCY' ][ 'path' ]
+            main.cellName = main.params[ 'ENV' ][ 'cellName' ]
+            main.apps = main.params[ 'ENV' ][ 'cellApps' ]
+            main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
+            main.scale = ( main.params[ 'SCALE' ] ).split( "," )
+            main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
+            main.timeout = int( main.params[ 'SLEEP' ][ 'timeout' ] )
+            main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
+            main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] )
+            main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] )
+            main.rerouteSleep = int ( main.params[ 'SLEEP' ][ 'reroute' ] )
+            main.verifyAttempts = int( main.params[ 'ATTEMPTS' ][ 'verify' ] )
+            main.ingress = main.params[ 'LINK' ][ 'ingress' ]
+            main.egress = main.params[ 'LINK' ][ 'egress' ]
+            main.reroute = main.params[ 'reroute' ]
+            main.flowObj = main.params[ 'TEST' ][ 'flowObj' ]
+            if main.flowObj == "True":
+                main.flowObj = True
+                main.dbFileName = main.params[ 'DATABASE' ][ 'dbFlowObj' ]
+            else:
+                main.flowObj = False
+                main.dbFileName = main.params[ 'DATABASE' ][ 'dbName' ]
+            main.threadID = 0
 
-        main.log.info( "Loading wrapper files" )
-        main.startUp = imp.load_source( wrapperFile1,
-                                        main.dependencyPath +
-                                        wrapperFile1 +
-                                        ".py" )
+            if main.reroute == "True":
+                main.reroute = True
+            else:
+                main.reroute = False
+            main.setupSkipped = False
 
+            wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
+            nic = main.params[ 'DATABASE' ][ 'nic' ]
+            node = main.params[ 'DATABASE' ][ 'node' ]
+            stepResult = main.testSetUp.gitPulling()
+            main.log.info( "Cresting DB file" )
+            with open( main.dbFileName, "w+" ) as dbFile:
+                dbFile.write("")
+        except Exception as e:
+            main.testSetUp.envSetupException( e )
+        main.testSetUp.evnSetupConclusion( stepResult )
+        main.commit = main.commit.split( " " )[ 1 ]
+        with open( main.dbFileName, "a" ) as dbFile:
+            temp = "'" + main.commit + "',"
+            temp += "'" + nic + "',"
+            dbFile.write( temp )
+    def CASE1( self ):
+        main.testSetUp.getNumCtrls( True )
+        main.testSetUp.envSetup( includeGitPull=False, makeMaxNodes=False )
         copyResult = main.ONOSbench.copyMininetFile( main.topology,
                                                      main.dependencyPath,
                                                      main.Mininet1.user_name,
                                                      main.Mininet1.ip_address )
 
-        commit = main.ONOSbench.getVersion(report=True)
-        commit = commit.split(" ")[1]
-
-        with open(main.dbFileName, "a") as dbFile:
-            temp = "'" + commit + "',"
-            temp += "'" + nic + "',"
-            dbFile.write(temp)
-
     def CASE2( self, main ):
         """
         - Uninstall ONOS cluster
@@ -127,115 +92,8 @@
         - Install ONOS cluster
         - Connect to cli
         """
-        main.log.info( "Starting up %s node(s) ONOS cluster" % main.numCtrls)
-        main.log.info( "Safety check, killing all ONOS processes" +
-                       " before initiating environment setup" )
-
-        for i in range( main.numCtrls ):
-            main.ONOSbench.onosStop( main.ONOSip[ i ] )
-            main.ONOSbench.onosKill( main.ONOSip[ i ] )
-
-        main.log.info( "NODE COUNT = %s" % main.numCtrls)
-
-        tempOnosIp = []
-        for i in range( main.numCtrls ):
-            tempOnosIp.append( main.ONOSip[i] )
-
-        main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
-                                       "temp",
-                                       main.Mininet1.ip_address,
-                                       main.apps,
-                                       tempOnosIp, main.ONOScli1.karafUser )
-
-        main.step( "Apply cell to environment" )
-        cellResult = main.ONOSbench.setCell( "temp" )
-        verifyResult = main.ONOSbench.verifyCell()
-        stepResult = cellResult and verifyResult
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=stepResult,
-                                 onpass="Successfully applied cell to " + \
-                                        "environment",
-                                 onfail="Failed to apply cell to environment " )
-
-        main.step( "Creating ONOS package" )
-        packageResult = main.ONOSbench.buckBuild()
-        stepResult = packageResult
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=stepResult,
-                                 onpass="Successfully created ONOS package",
-                                 onfail="Failed to create ONOS package" )
-
-        main.step( "Uninstall ONOS package on all Nodes" )
-        uninstallResult = main.TRUE
-        for i in range( int( main.numCtrls ) ):
-            main.log.info( "Uninstalling package on ONOS Node IP: " + main.ONOSip[i] )
-            u_result = main.ONOSbench.onosUninstall( main.ONOSip[i] )
-            utilities.assert_equals( expect=main.TRUE, actual=u_result,
-                                     onpass="Test step PASS",
-                                     onfail="Test step FAIL" )
-            uninstallResult = ( uninstallResult and u_result )
-
-        main.step( "Install ONOS package on all Nodes" )
-        installResult = main.TRUE
-        for i in range( int( main.numCtrls ) ):
-            main.log.info( "Installing package on ONOS Node IP: " + main.ONOSip[i] )
-            i_result = main.ONOSbench.onosInstall( node=main.ONOSip[i] )
-            utilities.assert_equals( expect=main.TRUE, actual=i_result,
-                                     onpass="Test step PASS",
-                                     onfail="Test step FAIL" )
-            installResult = installResult and i_result
-
-        main.step( "Set up ONOS secure SSH" )
-        secureSshResult = main.TRUE
-        for i in range( int( main.numCtrls ) ):
-            secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=main.ONOSip[i] )
-        utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
-                                 onpass="Test step PASS",
-                                 onfail="Test step FAIL" )
-
-        time.sleep( main.startUpSleep )
-        main.step( "Starting ONOS service" )
-        stopResult = main.TRUE
-        startResult = main.TRUE
-        onosIsUp = main.TRUE
-        for i in range( main.numCtrls ):
-            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
-        if onosIsUp == main.TRUE:
-            main.log.report( "ONOS instance is up and ready" )
-        else:
-            main.log.report( "ONOS instance may not be up, stop and " +
-                             "start ONOS again " )
-            for i in range( main.numCtrls ):
-                stopResult = stopResult and \
-                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
-            for i in range( main.numCtrls ):
-                startResult = startResult and \
-                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
-        stepResult = onosIsUp and stopResult and startResult
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=stepResult,
-                                 onpass="ONOS service is ready",
-                                 onfail="ONOS service did not start properly" )
-
-        main.step( "Start ONOS CLI on all nodes" )
-        cliResult = main.TRUE
-        main.step(" Start ONOS cli using thread ")
-        startCliResult  = main.TRUE
-        pool = []
-
-        for i in range( int( main.numCtrls) ):
-            t = main.Thread( target=main.CLIs[i].startOnosCli,
-                             threadID=main.threadID,
-                             name="startOnosCli",
-                             args=[ main.ONOSip[i] ],
-                             kwargs = {"onosStartTimeout":main.timeout} )
-            pool.append(t)
-            t.start()
-            main.threadID = main.threadID + 1
-        for t in pool:
-            t.join()
-            startCliResult = startCliResult and t.result
-        time.sleep( main.startUpSleep )
+        main.testSetUp.ONOSSetUp( main.Mininet1, True,
+                                  killRemoveMax=False, CtrlsSet=False )
 
     def CASE10( self, main ):
         """
@@ -246,7 +104,7 @@
         main.step("Activating null-provider")
         appStatus = utilities.retry( main.CLIs[0].activateApp,
                                      main.FALSE,
-                                     ['org.onosproject.null'],
+                                     [ 'org.onosproject.null' ],
                                      sleep=main.verifySleep,
                                      attempts=main.verifyAttempts )
         utilities.assert_equals( expect=main.TRUE,
@@ -259,20 +117,20 @@
         cfgStatus = utilities.retry( main.ONOSbench.onosCfgSet,
                                     main.FALSE,
                                     [ main.ONOSip[0],
-                                      'org.onosproject.provider.nil.NullProviders', 'deviceCount 8'],
+                                      'org.onosproject.provider.nil.NullProviders', 'deviceCount 8' ],
                                     sleep=main.verifySleep,
                                     attempts = main.verifyAttempts )
         cfgStatus = cfgStatus and utilities.retry( main.ONOSbench.onosCfgSet,
                                                    main.FALSE,
                                                    [ main.ONOSip[0],
-                                                     'org.onosproject.provider.nil.NullProviders', 'topoShape reroute'],
+                                                     'org.onosproject.provider.nil.NullProviders', 'topoShape reroute' ],
                                                    sleep=main.verifySleep,
                                                    attempts = main.verifyAttempts )
 
         cfgStatus = cfgStatus and utilities.retry( main.ONOSbench.onosCfgSet,
                                                    main.FALSE,
                                                    [ main.ONOSip[0],
-                                                     'org.onosproject.provider.nil.NullProviders', 'enabled true'],
+                                                     'org.onosproject.provider.nil.NullProviders', 'enabled true' ],
                                                    sleep=main.verifySleep,
                                                    attempts = main.verifyAttempts )
 
@@ -405,16 +263,25 @@
 
 
     def CASE20( self, main ):
+        try:
+            from tests.dependencies.utils import Utils
+        except ImportError:
+            main.log.error( "Utils not found exiting the test" )
+            main.exit()
+        try:
+            main.Utils
+        except ( NameError, AttributeError ):
+            main.Utils = Utils()
         if main.reroute:
-            main.minIntents = int(main.params['NULL']['REROUTE']['min_intents'])
-            main.maxIntents = int(main.params['NULL']['REROUTE']['max_intents'])
-            main.checkInterval = int(main.params['NULL']['REROUTE']['check_interval'])
-            main.batchSize = int(main.params['NULL']['REROUTE']['batch_size'])
+            main.minIntents = int(main.params[ 'NULL' ][ 'REROUTE' ][ 'min_intents' ] )
+            main.maxIntents = int(main.params[ 'NULL' ][ 'REROUTE' ][ 'max_intents' ] )
+            main.checkInterval = int(main.params[ 'NULL' ][ 'REROUTE' ][ 'check_interval' ] )
+            main.batchSize = int(main.params[ 'NULL' ][ 'REROUTE' ][ 'batch_size' ] )
         else:
-            main.minIntents = int(main.params['NULL']['PUSH']['min_intents'])
-            main.maxIntents = int(main.params['NULL']['PUSH']['max_intents'])
-            main.checkInterval = int(main.params['NULL']['PUSH']['check_interval'])
-            main.batchSize = int(main.params['NULL']['PUSH']['batch_size'])
+            main.minIntents = int(main.params[ 'NULL' ][ 'PUSH' ][ 'min_intents' ] )
+            main.maxIntents = int(main.params[ 'NULL' ][ 'PUSH' ][ 'max_intents' ] )
+            main.checkInterval = int(main.params[ 'NULL' ][ 'PUSH' ][ 'check_interval' ] )
+            main.batchSize = int(main.params[ 'NULL' ][ 'PUSH' ][ 'batch_size' ] )
 
         # check if the case needs to be skipped
         if main.setupSkipped:
@@ -536,7 +403,6 @@
                     main.log.info("Total Intents: {}".format( verifyTotalIntents) )
                     break
 
-        del main.scale[0]
         utilities.assert_equals( expect = main.TRUE,
                                  actual = intentsState,
                                  onpass = "Successfully pushed and verified intents",
@@ -545,8 +411,8 @@
         main.log.info( "Total Intents Installed before crash: {}".format( totalIntents ) )
         main.log.info( "Total Flows ADDED before crash: {}".format( totalFlows ) )
 
-        main.step('clean up Mininet')
-        main.Mininet1.stopNet()
+        main.Utils.mininetCleanup( main.Mininet1 )
+
         main.log.info("Writing results to DS file")
         with open(main.dbFileName, "a") as dbFile:
             # Scale number