[ONOS-3181] Refactored test and added try catches
Change-Id: I51e4693b28e4a0662037373ec35945ae9d940a0e
diff --git a/TestON/tests/SCPFscaleTopo/Dependency/scaleTopoFunction.py b/TestON/tests/SCPFscaleTopo/Dependency/scaleTopoFunction.py
index 4965331..075f4f2 100644
--- a/TestON/tests/SCPFscaleTopo/Dependency/scaleTopoFunction.py
+++ b/TestON/tests/SCPFscaleTopo/Dependency/scaleTopoFunction.py
@@ -198,7 +198,7 @@
switchList = main.Mininet1.getSwitch()
assignResult = main.Mininet1.assignSwController( sw=switchList,
ip=main.ONOSip[ 0 ],
- port=6653 )
+ port=6633 )
for sw in switchList:
response = main.Mininet1.getSwController( sw )
diff --git a/TestON/tests/SCPFscaleTopo/Dependency/topo.py b/TestON/tests/SCPFscaleTopo/Dependency/topo.py
index b44e3fc..0dbb02d 100644
--- a/TestON/tests/SCPFscaleTopo/Dependency/topo.py
+++ b/TestON/tests/SCPFscaleTopo/Dependency/topo.py
@@ -12,7 +12,7 @@
"""
devices = []
threads = []
- for i in range( main.numCtrls ):
+ for i in main.activeNodes:
t = main.Thread( target=main.CLIs[i].devices,
name="devices-" + str( i ),
args=[ ] )
@@ -31,7 +31,7 @@
hosts = []
ipResult = main.TRUE
threads = []
- for i in range( main.numCtrls ):
+ for i in main.activeNodes:
t = main.Thread( target=main.CLIs[i].hosts,
name="hosts-" + str( i ),
args=[ ] )
@@ -49,7 +49,7 @@
"""
ports = []
threads = []
- for i in range( main.numCtrls ):
+ for i in main.activeNodes:
t = main.Thread( target=main.CLIs[i].ports,
name="ports-" + str( i ),
args=[ ] )
@@ -67,7 +67,7 @@
"""
links = []
threads = []
- for i in range( main.numCtrls ):
+ for i in main.activeNodes:
t = main.Thread( target=main.CLIs[i].links,
name="links-" + str( i ),
args=[ ] )
@@ -85,7 +85,7 @@
"""
clusters = []
threads = []
- for i in range( main.numCtrls ):
+ for i in main.activeNodes:
t = main.Thread( target=main.CLIs[i].clusters,
name="clusters-" + str( i ),
args=[ ] )
diff --git a/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.params b/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.params
index 62150b3..5d1c7e1 100755
--- a/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.params
+++ b/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.params
@@ -1,21 +1,22 @@
<PARAMS>
- <testcases>1,2,1001,8,1002,8</testcases>
+ # 1 - Optional pull and build ONOS package
+ # 2 - Set cell and install ONOS
+ # 10 - start mininet and assign switches to controller
+ # 11 - pingall
+ # 12 - compare topology
+ # 100 - balance master
+ # 200 - bring down onos node
+ # 300 - bring up onos node
- <SCALE>
- <size>3</size>
- <max>3</max>
- </SCALE>
+ <testcases>1,2,[10,100,12,200,100,12,300,100,12,1000]*3</testcases>
<DEPENDENCY>
<path>/tests/SCPFscaleTopo/Dependency/</path>
<wrapper1>startUp</wrapper1>
<wrapper2>scaleTopoFunction</wrapper2>
<wrapper3>topo</wrapper3>
- <topology>spine.py</topology>
<multiovs>multiovs.py</multiovs>
- <spine>spine,15,30</spine>
- <torus>torus,28,28</torus>
</DEPENDENCY>
<ENV>
@@ -29,11 +30,25 @@
<CTRL>
<port>6653</port>
+ <numCtrls>3</numCtrls>
</CTRL>
<SLEEP>
<startup>15</startup>
<fwd>30</fwd>
+ <topoAttempts>1</topoAttempts>
+ <balance>10</balance>
+ <nodeDown>10</nodeDown>
+ <nodeUp>10</nodeUp>
</SLEEP>
+ <TIMEOUT>
+ <pingall>60</pingall>
+ </TIMEOUT>
+
+ <TOPOLOGY>
+ <topology>torus</topology>
+ <scale>5,10,20</scale>
+ </TOPOLOGY>
+
</PARAMS>
diff --git a/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.py b/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.py
index 24e8842..5094f4d 100644
--- a/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.py
+++ b/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.py
@@ -33,73 +33,77 @@
gitBranch = main.params[ 'GIT' ][ 'branch' ]
main.dependencyPath = main.testOnDirectory + \
main.params[ 'DEPENDENCY' ][ 'path' ]
- main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
- main.torus = main.params[ 'DEPENDENCY' ][ 'torus' ]
- main.spine = main.params[ 'DEPENDENCY' ][ 'spine' ]
- main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
- if main.ONOSbench.maxNodes:
- main.maxNodes = int( main.ONOSbench.maxNodes )
- else:
- main.maxNodes = 0
+ main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
+ main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] )
+ main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
+ main.topoScaleSize = len( main.topoScale )
wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
+ main.checkTopoAttempts = int( main.params['SLEEP']['topoAttempts'])
main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
main.fwdSleep = int( main.params[ 'SLEEP' ][ 'fwd' ] )
+ main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
+ main.nodeDownSleep = int( main.params[ 'SLEEP' ][ 'nodeDown' ] )
+ main.nodeUpSleep = int( main.params[ 'SLEEP' ][ 'nodeUp' ] )
+ main.pingallTimeout = int( main.params[ 'TIMEOUT' ][ 'pingall' ] )
gitPull = main.params[ 'GIT' ][ 'pull' ]
+ main.homeDir = os.path.expanduser('~')
main.cellData = {} # for creating cell file
main.hostsData = {}
main.CLIs = []
main.ONOSip = []
-
+ main.activeNodes = []
main.ONOSip = main.ONOSbench.getOnosIps()
- print main.ONOSip
- # Assigning ONOS cli handles to a list
- for i in range( 1, main.maxNodes + 1 ):
- main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+ except Exception:
+ main.log.exception( "Exception: constructing test variables" )
+ main.cleanup()
+ main.exit()
- # -- INIT SECTION, ONLY RUNS ONCE -- #
+ try:
+ for i in range(main.numCtrls):
+ main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
+
+ except Exception:
+ main.log.exception( "Exception: assinging ONOS cli handles to a list" )
+ main.cleanup()
+ main.exit()
+
+ try:
main.startUp = imp.load_source( wrapperFile1,
main.dependencyPath +
wrapperFile1 +
".py" )
main.scaleTopoFunction = imp.load_source( wrapperFile2,
- main.dependencyPath +
- wrapperFile2 +
- ".py" )
+ main.dependencyPath +
+ wrapperFile2 +
+ ".py" )
main.topo = imp.load_source( wrapperFile3,
main.dependencyPath +
wrapperFile3 +
".py" )
-
- copyResult1 = main.ONOSbench.scp( main.Mininet1,
- main.dependencyPath +
- main.topology,
- main.Mininet1.home,
- direction="to" )
- time.sleep(3)
- copyResult2 = main.ONOSbench.scp( main.Mininet1,
- main.dependencyPath +
- main.multiovs,
- main.Mininet1.home,
- direction="to" )
- time.sleep(3)
- if main.CLIs:
- stepResult = main.TRUE
- else:
- main.log.error( "Did not properly created list of " +
- "ONOS CLI handle" )
- stepResult = main.FALSE
-
- except Exception as e:
- main.log.exception(e)
+ except Exception:
+ main.log.exception( "Exception: importing wrapper files" )
main.cleanup()
main.exit()
+ main.ONOSbench.scp( main.Mininet1,
+ main.dependencyPath +
+ main.multiovs,
+ main.Mininet1.home,
+ direction="to" )
+
+ if main.CLIs:
+ stepResult = main.TRUE
+ else:
+ main.log.error( "Did not properly created list of " +
+ "ONOS CLI handle" )
+ stepResult = main.FALSE
+
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
onpass="Successfully construct " +
@@ -119,22 +123,9 @@
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
- main.ONOSbench.getVersion( report=True )
- if gitPull == 'True':
- main.step( "Building ONOS in " + gitBranch + " branch" )
- onosBuildResult = main.startUp.onosBuild( main, gitBranch )
- stepResult = onosBuildResult
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="Successfully compiled " +
- "latest ONOS",
- onfail="Failed to compile " +
- "latest ONOS" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
- def CASE2( self, main ):
+
+ def CASE2( self, main):
"""
- Set up cell
- Create cell file
@@ -147,29 +138,27 @@
- Connect to cli
"""
- # main.scale[ 0 ] determines the current number of ONOS controller
- main.numCtrls = int( main.scale[ 0 ] )
-
main.case( "Starting up " + str( main.numCtrls ) +
" node(s) ONOS cluster" )
+ main.caseExplanation = "Set up ONOS with " + str( main.numCtrls ) +\
+ " node(s) ONOS cluster"
+
+
#kill off all onos processes
main.log.info( "Safety check, killing all ONOS processes" +
" before initiating enviornment setup" )
- for i in range( main.maxNodes ):
+ for i in range( main.numCtrls ):
main.ONOSbench.onosDie( main.ONOSip[ i ] )
- print "NODE COUNT = ", main.numCtrls
-
tempOnosIp = []
for i in range( main.numCtrls ):
tempOnosIp.append( main.ONOSip[i] )
main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
"temp", main.Mininet1.ip_address,
- main.apps,
- tempOnosIp )
+ main.apps, tempOnosIp )
main.step( "Apply cell to environment" )
cellResult = main.ONOSbench.setCell( "temp" )
@@ -189,11 +178,124 @@
onpass="Successfully created ONOS package",
onfail="Failed to create ONOS package" )
+ time.sleep( main.startUpSleep )
+ main.step( "Uninstalling ONOS package" )
+ onosUninstallResult = main.TRUE
+ for ip in main.ONOSip:
+ onosUninstallResult = onosUninstallResult and \
+ main.ONOSbench.onosUninstall( nodeIp=ip )
+ stepResult = onosUninstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully uninstalled ONOS package",
+ onfail="Failed to uninstall ONOS package" )
- # Remove the first element in main.scale list
- #main.scale.remove( main.scale[ 0 ] )
+ time.sleep( main.startUpSleep )
+ main.step( "Installing ONOS package" )
+ onosInstallResult = main.TRUE
+ for i in range( main.numCtrls ):
+ onosInstallResult = onosInstallResult and \
+ main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+ stepResult = onosInstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully installed ONOS package",
+ onfail="Failed to install ONOS package" )
- def CASE8( self, main ):
+ time.sleep( main.startUpSleep )
+ main.step( "Starting ONOS service" )
+ stopResult = main.TRUE
+ startResult = main.TRUE
+ onosIsUp = main.TRUE
+
+ for i in range( main.numCtrls ):
+ onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+ if onosIsUp == main.TRUE:
+ main.log.report( "ONOS instance is up and ready" )
+ else:
+ main.log.report( "ONOS instance may not be up, stop and " +
+ "start ONOS again " )
+
+ for i in range( main.numCtrls ):
+ stopResult = stopResult and \
+ main.ONOSbench.onosStop( main.ONOSip[ i ] )
+ for i in range( main.numCtrls ):
+ startResult = startResult and \
+ main.ONOSbench.onosStart( main.ONOSip[ i ] )
+ stepResult = onosIsUp and stopResult and startResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+
+ main.step( "Start ONOS cli" )
+ cliResult = main.TRUE
+ for i in range( main.numCtrls ):
+ cliResult = cliResult and \
+ main.CLIs[ i ].startOnosCli( main.ONOSip[ i ] )
+ main.activeNodes.append( i )
+ stepResult = cliResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully start ONOS cli",
+ onfail="Failed to start ONOS cli" )
+
+
+ def CASE10( self, main ):
+ """
+ Starting up torus topology
+ """
+ main.case( "Starting up torus topology" )
+ main.step( "Starting up torus topology" )
+
+ main.log.info( "Checking if mininet is already running" )
+ if len( main.topoScale ) < main.topoScaleSize:
+ main.log.info( "Mininet is already running. Stopping mininet." )
+ main.Mininet1.stopNet()
+ time.sleep(5)
+ else:
+ main.log.info( "Mininet was not running" )
+
+ try:
+ scale = main.topoScale.pop(0)
+ except Exception:
+ main.log.exception("Exception: popping from list of topology scales ")
+ main.cleanup()
+ main.exit()
+
+ mnCmd = " mn --custom=" + main.homeDir + "/mininet/custom/multiovs.py " +\
+ "--switch=ovsm --topo " + main.topoName + ","+ scale + "," + scale +\
+ " --controller=remote,ip=" + main.ONOSip[ 0 ] +\
+ " --controller=remote,ip=" + main.ONOSip[ 1 ] +\
+ " --controller=remote,ip=" + main.ONOSip[ 2 ] + " --mac"
+ stepResult = main.Mininet1.startNet(mnCmd=mnCmd)
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass=main.topoName +
+ " topology started successfully",
+ onfail=main.topoName +
+ " topology failed to start" )
+
+
+ def CASE11( self, main ):
+ '''
+ Pingall
+ '''
+ main.case( "Pingall" )
+ main.step( "Pingall" )
+ pingResult = main.Mininet1.pingall()
+ if not pingResult:
+ main.log.warn( "First pingall failed. Retrying..." )
+ time.sleep(3)
+ pingResult = main.Mininet1.pingall( timeout=main.pingallTimeout )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=pingResult,
+ onpass="Pingall successfull",
+ onfail="Pingall failed" )
+
+
+ def CASE12( self, main ):
"""
Compare Topo
"""
@@ -204,10 +306,10 @@
" and ONOS"
main.step( "Gathering topology information" )
- # TODO: add a paramaterized sleep here
devicesResults = main.TRUE
linksResults = main.TRUE
hostsResults = main.TRUE
+
devices = main.topo.getAllDevices( main )
hosts = main.topo.getAllHosts( main )
ports = main.topo.getAllPorts( main )
@@ -218,9 +320,10 @@
mnLinks = main.Mininet1.getLinks()
mnHosts = main.Mininet1.getHosts()
- main.step( "Conmparing MN topology to ONOS topology" )
- for controller in range( main.numCtrls ):
- controllerStr = str( controller + 1 )
+ main.step( "Comparing MN topology to ONOS topology" )
+
+ for controller in range(len(main.activeNodes)):
+ controllerStr = str( main.activeNodes[controller] + 1 )
if devices[ controller ] and ports[ controller ] and\
"Error" not in devices[ controller ] and\
"Error" not in ports[ controller ]:
@@ -257,14 +360,96 @@
json.loads( hosts[ controller ] ) )
else:
currentHostsResult = main.FALSE
- utilities.assert_equals( expect=main.TRUE,
- actual=currentHostsResult,
- onpass="ONOS" + controllerStr +
- " hosts exist in Mininet",
- onfail="ONOS" + controllerStr +
- " hosts don't match Mininet" )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=currentHostsResult,
+ onpass="ONOS" + controllerStr +
+ " hosts exist in Mininet",
+ onfail="ONOS" + controllerStr +
+ " hosts don't match Mininet" )
- def CASE9( self, main ):
+ def CASE100( self, main ):
+ '''
+ Balance master
+ '''
+ main.case("Balancing Masters")
+ main.step("Balancing Masters")
+ try:
+ controller = main.activeNodes[0]
+ stepResult = main.CLIs[controller].balanceMasters()
+ except Exception:
+ main.log.exception("Exception: balancing masters")
+ main.cleanup()
+ main.exit()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Balance masters was successfull",
+ onfail="Failed to balance masters")
+
+ time.sleep(main.balanceSleep)
+
+
+ def CASE200( self, main ):
+ '''
+ Bring random node down
+ '''
+ import random
+ main.case( "Randomly stopping an ONOS service" )
+ main.step( "Bringing down an onos node" )
+
+ random.seed()
+ # Get the random index of the node
+ main.deadNode = random.randrange(0,main.numCtrls)
+
+ node = main.deadNode + 1
+
+ main.log.info( "deadnode: %s" % node )
+
+ main.log.info( "Stopping node %s" % node )
+ startResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] )
+
+ try:
+ main.activeNodes.pop( main.deadNode )
+ except Exception:
+ main.log.exception( "Exception: popping from list of active nodes" )
+ main.cleanup()
+ main.exit()
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully brought down onos node %s" % node,
+ onfail="Failed to bring down onos node %s" % node )
+
+ time.sleep(main.nodeDownSleep)
+
+
+ def CASE300( self, main ):
+ '''
+ Bring up onos node
+ '''
+ main.case( "Bring the dead ONOS node back up" )
+ main.step( "Bringing up an onos node" )
+
+ node = main.deadNode + 1
+
+ main.log.info( "Starting node %s" % node )
+ startResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] )
+
+ main.log.info( "Starting onos cli" )
+ startCliResult = main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] )
+
+ main.activeNodes.append( main.deadNode )
+
+ stepResult = startResult and startCliResult
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully brought up onos node %s" % node,
+ onfail="Failed to bring up onos node %s" % node )
+
+
+ time.sleep(main.nodeUpSleep)
+
+ def CASE1000( self, main ):
'''
Report errors/warnings/exceptions
'''
@@ -277,79 +462,3 @@
"ERROR",
"Except" ],
"s" )
-
- def CASE11( self, main ):
- """
- Start mininet
- """
- main.log.report( "Start Mininet topology" )
- main.log.case( "Start Mininet topology" )
-
- main.step( "Starting Mininet Topology" )
- topology = main.dependencyPath + main.topology
- topoResult = main.Mininet1.startNet( topoFile=topology )
- stepResult = topoResult
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="Successfully loaded topology",
- onfail="Failed to load topology" )
- # Exit if topology did not load properly
- if not topoResult:
- main.cleanup()
- main.exit()
-
- def CASE1001( self, main ):
- """
- Topology test
- """
- import time
- main.topoName = "SPINE"
- main.case( "Spine topology test" )
- main.step( main.topoName + " topology" )
- mnCmd = "sudo mn --custom " + main.dependencyPath +\
- main.multiovs + " --switch=ovsm --custom " +\
- main.dependencyPath + main.topology +\
- " --topo " + main.spine + " --controller=remote,ip=" +\
- main.ONOSip[ 0 ] + " --mac"
-
- stepResult = main.scaleTopoFunction.testTopology( main,
- mnCmd=mnCmd,
- timeout=900,
- clean=False )
-
- time.sleep(3)
-
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass=main.spine + " topology successful",
- onfail=main.spine +
- "Spine topology failed" )
-
- time.sleep(60)
-
- '''
- main.ONOSbench.scp( main.Mininet1,
- "~/mininet/custom/spine.json",
- "/tmp/",
- direction="to" )
- '''
-
- def CASE1002( self, main ):
- """
- Topology test
- """
- main.topoName = "TORUS"
- main.case( "Topology discovery test" )
- stepResult = main.TRUE
- main.step( main.torus + " topology" )
- mnCmd = "sudo mn --custom=mininet/custom/multiovs.py " +\
- "--switch=ovsm --topo " + main.torus +\
- " --controller=remote,ip=" + main.ONOSip[ 0 ] +" --mac"
- stepResult = main.scaleTopoFunction.testTopology( main,
- mnCmd=mnCmd,
- timeout=900,
- clean=True )
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass=main.torus + " topology successful",
- onfail=main.torus + " topology failed" )