[ONOS-3181] Refactored test:
- Increased log output
- Combined similar cases
- Case steps are more specific
- Other minor changes
Change-Id: I45431e017c7786e8ae495f9b0fdb55d49571edaa
diff --git a/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.py b/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.py
index b44d482..e0f2f1a 100644
--- a/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.py
+++ b/TestON/tests/SCPFscaleTopo/SCPFscaleTopo.py
@@ -23,73 +23,60 @@
- Build ONOS package
"""
- main.case( "Constructing test variables and building ONOS package" )
+ main.case( "Constructing test variables" )
main.step( "Constructing test variables" )
stepResult = main.FALSE
- try:
- main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
- main.apps = main.params[ 'ENV' ][ 'cellApps' ]
- gitBranch = main.params[ 'GIT' ][ 'branch' ]
- main.dependencyPath = main.testOnDirectory + \
- main.params[ 'DEPENDENCY' ][ 'path' ]
- main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
- main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
- main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] )
- main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
- main.topoScaleSize = len( main.topoScale )
- wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
- wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
- wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
- main.checkTopoAttempts = int( main.params['SLEEP']['topoAttempts'])
- main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
- main.fwdSleep = int( main.params[ 'SLEEP' ][ 'fwd' ] )
- main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
- main.nodeDownSleep = int( main.params[ 'SLEEP' ][ 'nodeDown' ] )
- main.nodeUpSleep = int( main.params[ 'SLEEP' ][ 'nodeUp' ] )
- main.pingallTimeout = int( main.params[ 'TIMEOUT' ][ 'pingall' ] )
- gitPull = main.params[ 'GIT' ][ 'pull' ]
- main.homeDir = os.path.expanduser('~')
- main.cellData = {} # for creating cell file
- main.hostsData = {}
- main.CLIs = []
- main.ONOSip = []
- main.activeNodes = []
- main.ONOSip = main.ONOSbench.getOnosIps()
+ main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
+ main.apps = main.params[ 'ENV' ][ 'cellApps' ]
+ gitBranch = main.params[ 'GIT' ][ 'branch' ]
+ main.dependencyPath = main.testOnDirectory + \
+ main.params[ 'DEPENDENCY' ][ 'path' ]
+ main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
+ main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
+ main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] )
+ main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
+ main.topoScaleSize = len( main.topoScale )
+ wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
+ wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
+ wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
+ main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] )
+ main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] )
+ main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
+ main.fwdSleep = int( main.params[ 'SLEEP' ][ 'fwd' ] )
+ main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
+ main.nodeDownSleep = int( main.params[ 'SLEEP' ][ 'nodeDown' ] )
+ main.nodeUpSleep = int( main.params[ 'SLEEP' ][ 'nodeUp' ] )
+ main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] )
+ main.stopMNSleep = int( main.params[ 'SLEEP' ][ 'stopMN' ] )
+ main.startMNSleep = int( main.params[ 'SLEEP' ][ 'startMN' ] )
+ main.pingTimeout = int( main.params[ 'TIMEOUT' ][ 'pingall' ] )
+ gitPull = main.params[ 'GIT' ][ 'pull' ]
+ main.homeDir = os.path.expanduser('~')
+ main.cellData = {} # for creating cell file
+ main.hostsData = {}
+ main.CLIs = []
+ main.ONOSip = []
+ main.activeNodes = []
+ main.ONOSip = main.ONOSbench.getOnosIps()
- except Exception:
- main.log.exception( "Exception: constructing test variables" )
- main.cleanup()
- main.exit()
-
- try:
- for i in range(main.numCtrls):
+ for i in range(main.numCtrls):
main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
- except Exception:
- main.log.exception( "Exception: assinging ONOS cli handles to a list" )
- main.cleanup()
- main.exit()
+ main.startUp = imp.load_source( wrapperFile1,
+ main.dependencyPath +
+ wrapperFile1 +
+ ".py" )
- try:
- main.startUp = imp.load_source( wrapperFile1,
- main.dependencyPath +
- wrapperFile1 +
- ".py" )
+ main.scaleTopoFunction = imp.load_source( wrapperFile2,
+ main.dependencyPath +
+ wrapperFile2 +
+ ".py" )
- main.scaleTopoFunction = imp.load_source( wrapperFile2,
- main.dependencyPath +
- wrapperFile2 +
- ".py" )
-
- main.topo = imp.load_source( wrapperFile3,
- main.dependencyPath +
- wrapperFile3 +
- ".py" )
- except Exception:
- main.log.exception( "Exception: importing wrapper files" )
- main.cleanup()
- main.exit()
+ main.topo = imp.load_source( wrapperFile3,
+ main.dependencyPath +
+ wrapperFile3 +
+ ".py" )
main.ONOSbench.scp( main.Mininet1,
main.dependencyPath +
@@ -243,31 +230,36 @@
def CASE10( self, main ):
"""
- Starting up torus topology
+ Starting up torus topology, pingall, and compare topo
"""
- main.case( "Starting up torus topology" )
- main.step( "Starting up torus topology" )
+ import json
+
+ main.case( "Starting up Mininet and verifying topology" )
+ main.caseExplanation = "Starting Mininet with a scalling topology and " +\
+ "comparing topology elements between Mininet and ONOS"
main.log.info( "Checking if mininet is already running" )
if len( main.topoScale ) < main.topoScaleSize:
main.log.info( "Mininet is already running. Stopping mininet." )
main.Mininet1.stopNet()
- time.sleep(5)
+ time.sleep(main.stopMNSleep)
else:
main.log.info( "Mininet was not running" )
- try:
+ if main.topoScale:
scale = main.topoScale.pop(0)
- except Exception:
- main.log.exception("Exception: popping from list of topology scales ")
- main.cleanup()
- main.exit()
+ else: main.log.error( "topology scale is empty" )
- mnCmd = " mn --custom=" + main.homeDir + "/mininet/custom/multiovs.py " +\
- "--switch=ovsm --topo " + main.topoName + ","+ scale + "," + scale +\
- " --controller=remote,ip=" + main.ONOSip[ 0 ] +\
- " --controller=remote,ip=" + main.ONOSip[ 1 ] +\
- " --controller=remote,ip=" + main.ONOSip[ 2 ] + " --mac"
+
+ main.step( "Starting up TORUS %sx%s topology" % (scale, scale) )
+
+ main.log.info( "Constructing Mininet command" )
+ mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs +\
+ " --switch ovsm --topo " + main.topoName + ","+ scale + "," + scale
+
+ for i in range( main.numCtrls ):
+ mnCmd += " --controller remote,ip=" + main.ONOSip[ i ]
+
stepResult = main.Mininet1.startNet(mnCmd=mnCmd)
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
@@ -276,36 +268,23 @@
onfail=main.topoName +
" topology failed to start" )
+ time.sleep( main.startMNSleep )
- def CASE11( self, main ):
- '''
- Pingall
- '''
- main.case( "Pingall" )
- main.step( "Pingall" )
- pingResult = main.Mininet1.pingall( timeout=main.pingallTimeout )
- if not pingResult:
- main.log.warn( "First pingall failed. Retrying..." )
- time.sleep(3)
- pingResult = main.Mininet1.pingall( timeout=main.pingallTimeout )
+ main.step( "Pinging all hosts" )
+
+ for i in range(main.pingallAttempts):
+ pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
+ if not pingResult:
+ main.log.warn( "Pingall attempt: %s failed" % (i+1) )
+ time.sleep(main.pingallSleep)
+ else: break
utilities.assert_equals( expect=main.TRUE,
actual=pingResult,
onpass="Pingall successfull",
onfail="Pingall failed" )
-
- def CASE12( self, main ):
- """
- Compare Topo
- """
- import json
-
- main.case( "Compare ONOS Topology view to Mininet topology" )
- main.caseExplanation = "Compare topology elements between Mininet" +\
- " and ONOS"
-
- main.step( "Gathering topology information" )
+ main.log.info( "Gathering topology information" )
devicesResults = main.TRUE
linksResults = main.TRUE
hostsResults = main.TRUE
@@ -367,19 +346,112 @@
onfail="ONOS" + controllerStr +
" hosts don't match Mininet" )
+ def CASE11( self, main ):
+ """
+ Pingall, and compare topo
+ """
+ import json
+
+ scale = main.topoScale[0]
+
+ main.case( "Verifying topology: TORUS %sx%s" % (scale, scale) )
+ main.caseExplanation = "Pinging all hosts andcomparing topology " +\
+ "elements between Mininet and ONOS"
+ main.step( "Pinging all hosts" )
+
+ for i in range(main.pingallAttempts):
+ pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
+ if not pingResult:
+ main.log.warn( "Pingall attempt: %s failed" % (i+1) )
+ time.sleep(main.pingallSleep)
+ else: break
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=pingResult,
+ onpass="Pingall successfull",
+ onfail="Pingall failed" )
+
+ main.log.info( "Gathering topology information" )
+ devicesResults = main.TRUE
+ linksResults = main.TRUE
+ hostsResults = main.TRUE
+
+ devices = main.topo.getAllDevices( main )
+ hosts = main.topo.getAllHosts( main )
+ ports = main.topo.getAllPorts( main )
+ links = main.topo.getAllLinks( main )
+ clusters = main.topo.getAllClusters( main )
+
+ mnSwitches = main.Mininet1.getSwitches()
+ mnLinks = main.Mininet1.getLinks()
+ mnHosts = main.Mininet1.getHosts()
+
+ main.step( "Comparing MN topology to ONOS topology" )
+
+ for controller in range(len(main.activeNodes)):
+ controllerStr = str( main.activeNodes[controller] + 1 )
+ if devices[ controller ] and ports[ controller ] and\
+ "Error" not in devices[ controller ] and\
+ "Error" not in ports[ controller ]:
+
+ currentDevicesResult = main.Mininet1.compareSwitches(
+ mnSwitches,
+ json.loads( devices[ controller ] ),
+ json.loads( ports[ controller ] ) )
+ else:
+ currentDevicesResult = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=currentDevicesResult,
+ onpass="ONOS" + controllerStr +
+ " Switches view is correct",
+ onfail="ONOS" + controllerStr +
+ " Switches view is incorrect" )
+
+ if links[ controller ] and "Error" not in links[ controller ]:
+ currentLinksResult = main.Mininet1.compareLinks(
+ mnSwitches, mnLinks,
+ json.loads( links[ controller ] ) )
+ else:
+ currentLinksResult = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=currentLinksResult,
+ onpass="ONOS" + controllerStr +
+ " links view is correct",
+ onfail="ONOS" + controllerStr +
+ " links view is incorrect" )
+
+ if hosts[ controller ] or "Error" not in hosts[ controller ]:
+ currentHostsResult = main.Mininet1.compareHosts(
+ mnHosts,
+ json.loads( hosts[ controller ] ) )
+ else:
+ currentHostsResult = main.FALSE
+ utilities.assert_equals( expect=main.TRUE,
+ actual=currentHostsResult,
+ onpass="ONOS" + controllerStr +
+ " hosts exist in Mininet",
+ onfail="ONOS" + controllerStr +
+ " hosts don't match Mininet" )
+
+
def CASE100( self, main ):
'''
- Balance master
+ Balance masters, ping and bring third ONOS node down
'''
- main.case("Balancing Masters")
- main.step("Balancing Masters")
- try:
+ scale = main.topoScale[0]
+
+ main.case("Balancing Masters and bring ONOS node 3 down: TORUS %sx%s" % (scale, scale))
+ main.caseExplanation = "Balance masters to make sure " +\
+ "each controller has some devices and " +\
+ "stop ONOS node 3 service. "
+
+ main.step( "Balancing Masters" )
+ stepResult = main.FALSE
+ if main.activeNodes:
controller = main.activeNodes[0]
stepResult = main.CLIs[controller].balanceMasters()
- except Exception:
- main.log.exception("Exception: balancing masters")
- main.cleanup()
- main.exit()
+ else: main.log.error( "List of active nodes is empty" )
+
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
onpass="Balance masters was successfull",
@@ -387,12 +459,20 @@
time.sleep(main.balanceSleep)
+ main.step( "Pinging all hosts" )
- def CASE200( self, main ):
- '''
- Bring third node down
- '''
- main.case( "Stopping an ONOS service" )
+ for i in range(main.pingallAttempts):
+ pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
+ if not pingResult:
+ main.log.warn( "Pingall attempt: %s failed" % (i+1) )
+ time.sleep(main.pingallSleep)
+ else: break
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=pingResult,
+ onpass="Pingall successfull",
+ onfail="Pingall failed" )
+
main.step( "Bringing down node 3" )
# Always bring down the third node
@@ -401,17 +481,11 @@
# Printing purposes
node = main.deadNode + 1
- main.log.info( "deadnode: %s" % node )
-
main.log.info( "Stopping node %s" % node )
- startResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] )
+ stepResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] )
- try:
- main.activeNodes.pop( main.deadNode )
- except Exception:
- main.log.exception( "Exception: popping from list of active nodes" )
- main.cleanup()
- main.exit()
+ main.log.info( "Removing dead node from list of active nodes" )
+ main.activeNodes.pop( main.deadNode )
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
@@ -421,25 +495,27 @@
time.sleep(main.nodeDownSleep)
- def CASE300( self, main ):
+ def CASE200( self, main ):
'''
- Bring up onos node
+ Bring up onos node and balance masters
'''
- main.case( "Bring the dead ONOS node back up" )
- main.step( "Bringing up an onos node" )
+
+ scale = main.topoScale[0]
+
+ main.case("Bring ONOS node 3 up and balance masters: TORUS %sx%s" % (scale, scale))
+ main.caseExplanation = "Bring node 3 back up and balance the masters"
node = main.deadNode + 1
main.log.info( "Starting node %s" % node )
- startResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] )
+ stepResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] )
main.log.info( "Starting onos cli" )
- startCliResult = main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] )
+ stepResult = stepResult and main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] )
+ main.log.info( "Adding previously dead node to list of active nodes" )
main.activeNodes.append( main.deadNode )
- stepResult = startResult and startCliResult
-
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
onpass="Successfully brought up onos node %s" % node,
@@ -448,16 +524,38 @@
time.sleep(main.nodeUpSleep)
+ main.step( "Balancing Masters" )
+ stepResult = main.FALSE
+ if main.activeNodes:
+ controller = main.activeNodes[0]
+ stepResult = main.CLIs[controller].balanceMasters()
+ else: main.log.error( "List of active nodes is empty" )
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Balance masters was successfull",
+ onfail="Failed to balance masters")
+
+ time.sleep(main.balanceSleep)
+
+ for i in range(main.pingallAttempts):
+ pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
+ if not pingResult:
+ main.log.warn( "Pingall attempt: %s failed" % (i+1) )
+ time.sleep(main.pingallSleep)
+ else: break
+
def CASE1000( self, main ):
'''
Report errors/warnings/exceptions
'''
+ main.case( "Checking logs for errors, warnings, and exceptions" )
main.log.info("Error report: \n" )
main.ONOSbench.logReport( main.ONOSip[ 0 ],
- [ "INFO",
- "FOLLOWER",
- "WARN",
- "flow",
- "ERROR",
- "Except" ],
- "s" )
+ [ "INFO",
+ "FOLLOWER",
+ "WARN",
+ "flow",
+ "ERROR",
+ "Except" ],
+ "s" )