[ONOS-6593]Review and Refactor ONOS startup procedures in TestON
Change-Id: I509a8ee7a26c198957bebf59da5c85a0edb8b995
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
new file mode 100644
index 0000000..70fe3c2
--- /dev/null
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -0,0 +1,321 @@
+
+import time
+import re
+import imp
+
+class ONOSSetup:
+ main = None
+ def __init__( self ):
+ self.default = ''
+ def envSetupDescription ( self ):
+ main.case( "Constructing test variables and building ONOS package" )
+ main.step( "Constructing test variables" )
+ main.caseExplanation = "For loading from params file, and pull" + \
+ " and build the latest ONOS package"
+ main.testOnDirectory = re.sub( "(/tests)$", "", main.testDir )
+
+ def gitPulling( self ):
+ main.case( "Pull onos branch and build onos on Teststation." )
+ gitPull = main.params[ 'GIT' ][ 'pull' ]
+ gitBranch = main.params[ 'GIT' ][ 'branch' ]
+ if gitPull == 'True':
+ main.step( "Git Checkout ONOS branch: " + gitBranch )
+ stepResult = main.ONOSbench.gitCheckout( branch=gitBranch )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully checkout onos branch.",
+ onfail="Failed to checkout onos branch. Exiting test..." )
+ if not stepResult: main.exit()
+
+ main.step( "Git Pull on ONOS branch:" + gitBranch )
+ stepResult = main.ONOSbench.gitPull()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully pull onos. ",
+ onfail="Failed to pull onos. Exiting test ..." )
+ if not stepResult: main.exit()
+
+ else:
+ main.log.info( "Skipped git checkout and pull as they are disabled in params file" )
+
+ return main.TRUE
+
+ def setRest( self, hasRest, i ):
+ if hasRest:
+ main.RESTs.append( getattr( main, "ONOSrest" + str( i ) ) )
+
+ def setNode( self, hasNode, i ):
+ if hasNode:
+ main.nodes.append( getattr( main, 'ONOS' + str(i)) )
+
+ def setCli( self, hasCli, i ):
+ if hasCli:
+ main.CLIs.append( getattr ( main, "ONOScli" + str( i ) ) )
+
+ def getNumNode( self, hasCli, hasNode, hasRest ):
+ if hasCli:
+ return len( main.CLIs )
+ if hasNode:
+ return len( main.nodes )
+ if hasRest:
+ return len( main.RESTs )
+
+ def envSetup ( self, hasMultiNodeRounds=False, hasRest=False, hasNode=False,
+ hasCli=True, specificIp="", includeGitPull=True, makeMaxNodes=True ):
+ if includeGitPull :
+ self.gitPulling()
+ if main.ONOSbench.maxNodes:
+ main.maxNodes = int( main.ONOSbench.maxNodes )
+ else:
+ main.maxNodes = 0
+ main.cellData = {} # For creating cell file
+ if hasCli:
+ main.CLIs = []
+ if hasRest:
+ main.RESTs = []
+ if hasNode:
+ main.nodes = []
+ main.ONOSip = [] # List of IPs of active ONOS nodes. CASE 2
+
+ if specificIp == "":
+ if makeMaxNodes:
+ main.ONOSip = main.ONOSbench.getOnosIps()
+ else:
+ main.ONOSip.append( specificIp )
+
+ # Assigning ONOS cli handles to a list
+ try:
+ for i in range( 1, ( main.maxNodes if makeMaxNodes else main.numCtrls ) + 1 ):
+ self.setCli( hasCli, i )
+ self.setRest( hasRest, i )
+ self.setNode( hasNode, i )
+ if not makeMaxNodes:
+ main.ONOSip.append( main.ONOSbench.getOnosIps()[ i - 1 ] )
+ except AttributeError:
+ numNode = self.getNumNode( hasCli, hasNode, hasRest )
+ main.log.warn( "A " + str( main.maxNodes ) + " node cluster " +
+ "was defined in env variables, but only " +
+ str( numNode ) +
+ " nodes were defined in the .topo file. " +
+ "Using " + str( numNode ) +
+ " nodes for the test." )
+ main.maxNodes = numNode
+
+ main.log.debug( "Found ONOS ips: {}".format( main.ONOSip ) )
+ if ( not hasCli or main.CLIs ) and ( not hasRest or main.RESTs )\
+ and ( not hasNode or main.nodes ):
+ return main.TRUE
+ else:
+ main.log.error( "Did not properly created list of ONOS CLI handle" )
+ return main.FALSE
+
+ def envSetupException ( self, e ):
+ main.log.exception( e )
+ main.cleanup()
+ main.exit()
+
+ def evnSetupConclusion ( self, stepResult ):
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully construct " +
+ "test variables ",
+ onfail="Failed to construct test variables" )
+
+ main.commit = main.ONOSbench.getVersion( report=True )
+
+ def getNumCtrls( self, hasMultiNodeRounds ):
+ if hasMultiNodeRounds:
+ try:
+ main.cycle
+ except Exception:
+ main.cycle = 0
+ main.cycle += 1
+ # main.scale[ 0 ] determines the current number of ONOS controller
+ main.numCtrls = int( main.scale.pop( 0 ) )
+ else:
+ main.numCtrls = main.maxNodes
+
+ def killingAllOnos( self, killRemoveMax, stopOnos ):
+ # kill off all onos processes
+ main.log.info( "Safety check, killing all ONOS processes" +
+ " before initiating environment setup" )
+
+ for i in range( main.maxNodes if killRemoveMax else main.numCtrls ):
+ if stopOnos:
+ main.ONOSbench.onosStop( main.ONOSip[ i ] )
+ main.ONOSbench.onosKill( main.ONOSip[ i ] )
+
+ def createApplyCell( self, newCell, cellName, Mininet, useSSH ):
+ if newCell:
+ tempOnosIp = []
+ for i in range( main.numCtrls ):
+ tempOnosIp.append( main.ONOSip[i] )
+
+ main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
+ cellName,
+ Mininet if isinstance( Mininet, str ) else
+ Mininet.ip_address,
+ main.apps,
+ tempOnosIp,
+ main.ONOScli1.karafUser,
+ useSSH=useSSH )
+ main.step( "Apply cell to environment" )
+ cellResult = main.ONOSbench.setCell( cellName )
+ verifyResult = main.ONOSbench.verifyCell()
+ stepResult = cellResult and verifyResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully applied cell to " +
+ "environment",
+ onfail="Failed to apply cell to environment " )
+ return stepResult
+
+ def uninstallOnos( self, killRemoveMax ):
+ main.step( "Uninstalling ONOS package" )
+ onosUninstallResult = main.TRUE
+ for i in range( main.maxNodes if killRemoveMax else main.numCtrls ):
+ onosUninstallResult = onosUninstallResult and \
+ main.ONOSbench.onosUninstall( nodeIp=main.ONOSip[ i ] )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=onosUninstallResult,
+ onpass="Successfully uninstalled ONOS package",
+ onfail="Failed to uninstall ONOS package" )
+ return onosUninstallResult
+
+ def buildOnos( self ):
+ main.step( "Creating ONOS package" )
+ packageResult = main.ONOSbench.buckBuild()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=packageResult,
+ onpass="Successfully created ONOS package",
+ onfail="Failed to create ONOS package" )
+ return packageResult
+
+ def installOnos( self, installMax ):
+ main.step( "Installing ONOS package" )
+ onosInstallResult = main.TRUE
+
+ for i in range( main.ONOSbench.maxNodes if installMax else main.numCtrls ):
+ options = "-f"
+ if installMax and i >= main.numCtrls:
+ options = "-nf"
+ onosInstallResult = onosInstallResult and \
+ main.ONOSbench.onosInstall( node=main.ONOSip[ i ], options=options )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=onosInstallResult,
+ onpass="Successfully installed ONOS package",
+ onfail="Failed to install ONOS package" )
+ if not onosInstallResult:
+ main.cleanup()
+ main.exit()
+ return onosInstallResult
+
+ def setupSsh( self ):
+ main.step( "Set up ONOS secure SSH" )
+ secureSshResult = main.TRUE
+ for i in range( main.numCtrls ):
+ secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=main.ONOSip[ i ] )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=secureSshResult,
+ onpass="Test step PASS",
+ onfail="Test step FAIL" )
+ return secureSshResult
+
+ def checkOnosService( self ):
+ stopResult = main.TRUE
+ startResult = main.TRUE
+ onosIsUp = main.TRUE
+ main.step( "Starting ONOS service" )
+ for i in range( main.numCtrls ):
+ onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+ if onosIsUp == main.TRUE:
+ main.log.report( "ONOS instance {0} is up and ready".format( i + 1 ) )
+ else:
+ main.log.report( "ONOS instance {0} may not be up, stop and ".format( i + 1 ) +
+ "start ONOS again " )
+ stopResult = stopResult and main.ONOSbench.onosStop( main.ONOSip[ i ] )
+ startResult = startResult and main.ONOSbench.onosStart( main.ONOSip[ i ] )
+ if not startResult or stopResult:
+ main.log.report( "ONOS instance {0} did not start correctly.".format( i + 1 ) )
+ stepResult = onosIsUp and stopResult and startResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="ONOS service is ready on all nodes",
+ onfail="ONOS service did not start properly on all nodes" )
+ return stepResult
+
+ def startOnosClis( self ):
+ startCliResult = main.TRUE
+ main.step( "Starting ONOS CLI sessions" )
+ pool = []
+ main.threadID = 0
+ for i in range( main.numCtrls ):
+ t = main.Thread( target=main.CLIs[ i ].startOnosCli,
+ threadID=main.threadID,
+ name="startOnosCli-" + str( i ),
+ args=[ main.ONOSip[ i ] ] )
+ pool.append( t )
+ t.start()
+ main.threadID = main.threadID + 1
+ for t in pool:
+ t.join()
+ startCliResult = startCliResult and t.result
+ if not startCliResult:
+ main.log.info( "ONOS CLI did not start up properly" )
+ main.cleanup()
+ main.exit()
+ else:
+ main.log.info( "Successful CLI startup" )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=startCliResult,
+ onpass="Successfully start ONOS cli",
+ onfail="Failed to start ONOS cli" )
+ return startCliResult
+
+ def ONOSSetUp( self, Mininet, hasMultiNodeRounds=False, hasCli=True, newCell=True,
+ cellName="temp", removeLog=False, extraApply=None, arg=None, extraClean=None,
+ skipPack=False, installMax=False, useSSH=True, killRemoveMax=True,
+ CtrlsSet=True, stopOnos=False ):
+ if CtrlsSet:
+ self.getNumCtrls( hasMultiNodeRounds )
+
+ main.case( "Starting up " + str( main.numCtrls ) +
+ " node(s) ONOS cluster" )
+ main.caseExplanation = "Set up ONOS with " + str( main.numCtrls ) + \
+ " node(s) ONOS cluster"
+ self.killingAllOnos( killRemoveMax, stopOnos )
+
+ main.log.info( "NODE COUNT = " + str( main.numCtrls ) )
+ cellResult = True
+ packageResult = True
+ onosUninstallResult = True
+ onosCliResult = True
+ if not skipPack:
+ cellResult = self.createApplyCell( newCell, cellName, Mininet, useSSH )
+
+ if removeLog:
+ main.log.info("Removing raft logs")
+ main.ONOSbench.onosRemoveRaftLogs()
+
+ onosUninstallResult = self.uninstallOnos( killRemoveMax )
+
+ if extraApply is not None:
+ extraApply( metadataMethod=arg ) if arg is not None else extraApply()
+
+ packageResult = self.buildOnos()
+
+ onosInstallResult = self.installOnos( installMax )
+
+ if extraClean is not None:
+ extraClean()
+ secureSshResult = True
+ if useSSH:
+ secureSshResult = self.setupSsh()
+
+ onosServiceResult = self.checkOnosService()
+
+ if hasCli:
+ onosCliResult = self.startOnosClis()
+
+ return cellResult and packageResult and onosUninstallResult and \
+ onosInstallResult and secureSshResult and onosServiceResult and onosCliResult
\ No newline at end of file
diff --git a/TestON/tests/dependencies/__init__.py b/TestON/tests/dependencies/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/dependencies/__init__.py
diff --git a/TestON/tests/dependencies/topology.py b/TestON/tests/dependencies/topology.py
new file mode 100644
index 0000000..f9ce3ff
--- /dev/null
+++ b/TestON/tests/dependencies/topology.py
@@ -0,0 +1,237 @@
+import time
+import re
+import imp
+import json
+class Topology:
+
+ def __init__( self ):
+ self.default = ''
+ """
+ These functions can be used for topology comparisons
+ """
+ def getAllDevices( self, numNode, needRetry, kwargs={} ):
+ """
+ Return a list containing the devices output from each ONOS node
+ """
+ devices = []
+ threads = []
+ for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
+ t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].devices,
+ name="devices-" + str( i ),
+ args=[main.CLIs[ i ].devices, [ None ] ] if needRetry else [],
+ kwargs=kwargs )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ devices.append( t.result )
+ return devices
+
+ def getAllHosts( self, numNode, needRetry, kwargs={}, inJson=False ):
+ """
+ Return a list containing the hosts output from each ONOS node
+ """
+ hosts = []
+ ipResult = main.TRUE
+ threads = []
+ for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
+ t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].hosts,
+ name="hosts-" + str( i ),
+ args=[main.CLIs[ i ].hosts, [ None ] ] if needRetry else [],
+ kwargs=kwargs )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ if inJson:
+ try:
+ hosts.append( json.loads( t.result ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing hosts results" )
+ main.log.error( repr( t.result ) )
+ hosts.append( None )
+ else:
+ hosts.append( t.result )
+ return hosts
+
+ def getAllPorts( self, numNode, needRetry, kwargs={} ):
+ """
+ Return a list containing the ports output from each ONOS node
+ """
+ ports = []
+ threads = []
+ for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
+ t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].ports,
+ name="ports-" + str( i ),
+ args=[ main.CLIs[ i ].ports, [ None ] ] if needRetry else [],
+ kwargs=kwargs )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ports.append( t.result )
+ return ports
+
+ def getAllLinks( self, numNode, needRetry, kwargs={} ):
+ """
+ Return a list containing the links output from each ONOS node
+ """
+ links = []
+ threads = []
+ print numNode
+ for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
+ t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].links,
+ name="links-" + str( i ),
+ args=[main.CLIs[ i ].links, [ None ] ] if needRetry else [],
+ kwargs=kwargs )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ links.append( t.result )
+ print links
+ return links
+
+ def getAllClusters( self, numNode, needRetry, kwargs={} ):
+ """
+ Return a list containing the clusters output from each ONOS node
+ """
+ clusters = []
+ threads = []
+ for i in ( range ( numNode ) if isinstance( numNode, int ) else numNode ):
+ t = main.Thread( target=utilities.retry if needRetry else main.CLIs[ i ].clusters,
+ name="clusters-" + str( i ),
+ args=[main.CLIs[ i ].clusters, [ None ] ] if needRetry else [],
+ kwargs=kwargs )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ clusters.append( t.result )
+ return clusters
+
+ def compareDevicePort( self, Mininet, controller, mnSwitches, devices, ports ):
+ if devices[ controller ] and ports[ controller ] and \
+ "Error" not in devices[ controller ] and \
+ "Error" not in ports[ controller ]:
+ try:
+ currentDevicesResult = Mininet.compareSwitches(
+ mnSwitches,
+ json.loads( devices[ controller ] ),
+ json.loads( ports[ controller ] ) )
+ except(TypeError, ValueError):
+ main.log.error(
+ "Could not load json: {0} or {1}".format( str( devices[ controller ] )
+ , str( ports[ controller ] ) ) )
+ currentDevicesResult = main.FALSE
+ else:
+ currentDevicesResult = main.FALSE
+ return currentDevicesResult
+
+ def compareBase( self, compareElem, controller, compareF, compareArg ):
+ if compareElem[ controller ] and "Error" not in compareElem[ controller ]:
+ try:
+ if isinstance( compareArg, list ):
+ compareArg.append( json.loads( compareElem[ controller ] ) )
+ else:
+ compareArg = [compareArg, json.loads( compareElem[ controller ] ) ]
+
+ currentCompareResult = compareF( *compareArg )
+ except(TypeError, ValueError):
+ main.log.error(
+ "Could not load json: {0} or {1}".format( str( compareElem[ controller ] ) ) )
+ currentCompareResult = main.FALSE
+ else:
+ currentCompareResult = main.FALSE
+
+ return currentCompareResult
+
+ def compareTopos( self, Mininet, attempts=1 ):
+
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplanation = "Compare topology elements between Mininet" +\
+ " and ONOS"
+ main.log.info( "Gathering topology information from Mininet" )
+ devicesResults = main.FALSE # Overall Boolean for device correctness
+ linksResults = main.FALSE # Overall Boolean for link correctness
+ hostsResults = main.FALSE # Overall Boolean for host correctness
+ deviceFails = [] # Nodes where devices are incorrect
+ linkFails = [] # Nodes where links are incorrect
+ hostFails = [] # Nodes where hosts are incorrect
+
+ mnSwitches = Mininet.getSwitches()
+ mnLinks = Mininet.getLinks()
+ mnHosts = Mininet.getHosts()
+
+ main.step( "Comparing Mininet topology to ONOS topology" )
+
+ while ( attempts >= 0 ) and\
+ ( not devicesResults or not linksResults or not hostsResults ):
+ main.log.info( "Sleeping {} seconds".format( 2 ) )
+ time.sleep( 2 )
+ if not devicesResults:
+ devices = self.getAllDevices( main.numCtrls, False )
+ ports = self.getAllPorts( main.numCtrls, False )
+ devicesResults = main.TRUE
+ deviceFails = [] # Reset for each failed attempt
+ if not linksResults:
+ links = self.getAllLinks( main.numCtrls, False )
+ linksResults = main.TRUE
+ linkFails = [] # Reset for each failed attempt
+ if not hostsResults:
+ hosts = self.getAllHosts( main.numCtrls, False )
+ hostsResults = main.TRUE
+ hostFails = [] # Reset for each failed attempt
+
+ # Check for matching topology on each node
+ for controller in range( main.numCtrls ):
+ controllerStr = str( controller + 1 ) # ONOS node number
+ # Compare Devices
+ currentDevicesResult = self.compareDevicePort( Mininet, controller,
+ mnSwitches,
+ devices, ports )
+ if not currentDevicesResult:
+ deviceFails.append( controllerStr )
+ devicesResults = devicesResults and currentDevicesResult
+ # Compare Links
+ currentLinksResult = self.compareBase( links, controller,
+ Mininet.compareLinks,
+ [ mnSwitches, mnLinks ] )
+ if not currentLinksResult:
+ linkFails.append( controllerStr )
+ linksResults = linksResults and currentLinksResult
+ # Compare Hosts
+ currentHostsResult = self.compareBase( hosts, controller,
+ Mininet.compareHosts,
+ mnHosts )
+ if not currentHostsResult:
+ hostFails.append( controllerStr )
+ hostsResults = hostsResults and currentHostsResult
+ # Decrement Attempts Remaining
+ attempts -= 1
+
+ utilities.assert_equals( expect=[],
+ actual=deviceFails,
+ onpass="ONOS correctly discovered all devices",
+ onfail="ONOS incorrectly discovered devices on nodes: " +
+ str( deviceFails ) )
+ utilities.assert_equals( expect=[],
+ actual=linkFails,
+ onpass="ONOS correctly discovered all links",
+ onfail="ONOS incorrectly discovered links on nodes: " +
+ str( linkFails ) )
+ utilities.assert_equals( expect=[],
+ actual=hostFails,
+ onpass="ONOS correctly discovered all hosts",
+ onfail="ONOS incorrectly discovered hosts on nodes: " +
+ str( hostFails ) )
+ topoResults = hostsResults and linksResults and devicesResults
+ utilities.assert_equals( expect=main.TRUE,
+ actual=topoResults,
+ onpass="ONOS correctly discovered the topology",
+ onfail="ONOS incorrectly discovered the topology" )
diff --git a/TestON/tests/dependencies/utils.py b/TestON/tests/dependencies/utils.py
new file mode 100644
index 0000000..d82ae04
--- /dev/null
+++ b/TestON/tests/dependencies/utils.py
@@ -0,0 +1,50 @@
+class Utils:
+ def __init__( self ):
+ self.default = ''
+
+ def mininetCleanIntro( self ):
+ main.log.report( "Stop Mininet" )
+
+ main.case( "Stop Mininet" )
+ main.caseExplanation = "Stopping the current mininet to start up fresh"
+
+ def mininetCleanup( self, Mininet, timeout=5 ):
+ main.step( "Stopping Mininet" )
+ topoResult = Mininet.stopNet( timeout=timeout )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=topoResult,
+ onpass="Successfully stopped mininet",
+ onfail="Failed to stopped mininet" )
+ return topoResult
+
+ def copyKarafLog( self ):
+ """
+ Copy the karaf.log files after each testcase cycle
+ """
+ main.log.report( "Copy karaf logs" )
+ main.case( "Copy karaf logs" )
+ main.caseExplanation = "Copying the karaf logs to preserve them through" +\
+ "reinstalling ONOS"
+ main.step( "Copying karaf logs" )
+ stepResult = main.TRUE
+ scpResult = main.TRUE
+ copyResult = main.TRUE
+ for i in range( main.numCtrls ):
+ main.node = main.CLIs[ i ]
+ ip = main.ONOSip[ i ]
+ main.node.ip_address = ip
+ scpResult = scpResult and main.ONOSbench.scp( main.node,
+ "/opt/onos/log/karaf.log",
+ "/tmp/karaf.log",
+ direction="from" )
+ copyResult = copyResult and main.ONOSbench.cpLogsToDir( "/tmp/karaf.log", main.logdir,
+ copyFileName=( "karaf.log.node{0}.cycle{1}".format(
+ str( i + 1 ), str( main.cycle ) ) ) )
+ if scpResult and copyResult:
+ stepResult = main.TRUE and stepResult
+ else:
+ stepResult = main.FALSE and stepResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully copied remote ONOS logs",
+ onfail="Failed to copy remote ONOS logs" )
\ No newline at end of file