Move current to SR Sanity tests
Change-Id: I8727fb969c8bfb3806c3ffa78ad47c43762b5621
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
new file mode 100755
index 0000000..59c1214
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
@@ -0,0 +1,16 @@
+This test verifies basic connectivity using SegmentRouting via pingall,
+it should not fail.
+
+It consists of
+
+1) Configure and Install ONOS
+2) Start Mininet and check flow state
+3) Pingall
+
+Requirements
+
+ - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+ 1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+The topology is a 2x2 Leaf-spine
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.params b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.params
similarity index 66%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.params
rename to TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.params
index 15ce439..24403e2 100755
--- a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.params
@@ -1,6 +1,6 @@
<PARAMS>
- <testcases>1,[2,3,4,4,10]*2</testcases>
+ <testcases>1,2,3,4,10</testcases>
<SCALE>
<size>1</size>
@@ -8,7 +8,6 @@
</SCALE>
<DEPENDENCY>
- <path>/tests/USECASE_SegmentRouting/dependencies/</path>
<wrapper1>startUp</wrapper1>
<topology>cord_fabric.py</topology>
</DEPENDENCY>
@@ -16,7 +15,7 @@
<ENV>
<cellName>productionCell</cellName>
<cellApps>drivers,segmentrouting</cellApps>
- <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider;openflow</diffApps>
+ <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
<cellUser>sdn</cellUser>
</ENV>
@@ -29,6 +28,17 @@
<port>6653</port>
</CTRL>
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ </timers>
+
+ <kill>
+ <switch> spine101 </switch>
+ <dpid> 000000000101 </dpid>
+ <links> leaf1 leaf2 </links>
+ </kill>
+
<SLEEP>
<startup>10</startup>
</SLEEP>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py
new file mode 100755
index 0000000..9a0d91c
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py
@@ -0,0 +1,619 @@
+# This test should always succeed. it runs cases 1,2,3,4
+#CASE1: Get and Build ONOS
+#CASE2: Package and Install ONOS
+#CASE3: Start Mininet and check flows
+#CASE4: Ping all
+#CASE5: Link Failure
+#CASE6: Switch Failure
+#CASE7: ONOS Failure
+#CASE8: CLUSTER Failure
+#CASE10: Logging
+
+class SRSanity:
+
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ import time
+ import os
+ import imp
+ import re
+
+ """
+ - Construct tests variables
+ - GIT ( optional )
+ - Checkout ONOS master branch
+ - Pull latest ONOS code
+ - Building ONOS ( optional )
+ - Install ONOS package
+ - Build ONOS package
+ """
+
+ main.case( "Constructing test variables and building ONOS" )
+ main.step( "Constructing test variables" )
+ stepResult = main.FALSE
+
+ # Test variables
+ main.cellName = main.params[ 'ENV' ][ 'cellName' ]
+ main.apps = main.params[ 'ENV' ][ 'cellApps' ]
+ main.diff = []
+ main.diff.extend(( main.params[ 'ENV' ][ 'diffApps' ] ).split(";"))
+ main.diff=main.diff*2
+ gitBranch = main.params[ 'GIT' ][ 'branch' ]
+ main.path = os.path.dirname( main.testFile )
+ main.dependencyPath = main.path +"/../dependencies/"
+ main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
+ main.json = ["2x2"]*4
+ main.args = [" "]*4
+ main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
+ main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
+ main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
+ wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
+ main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
+ gitPull = main.params[ 'GIT' ][ 'pull' ]
+ main.cellData = {} # for creating cell file
+ main.CLIs = []
+ main.ONOSip = []
+
+ # Assigning ONOS cli handles to a list
+ for i in range( 1, main.maxNodes + 1 ):
+ main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+ main.ONOSip.append( main.CLIs[i-1].ip_address )
+ # -- INIT SECTION, ONLY RUNS ONCE -- #
+ main.startUp = imp.load_source( wrapperFile1,
+ main.dependencyPath +
+ wrapperFile1 +
+ ".py" )
+
+ copyResult1 = main.ONOSbench.scp( main.Mininet1,
+ main.dependencyPath +
+ main.topology,
+ main.Mininet1.home,
+ direction="to" )
+ if main.CLIs:
+ stepResult = main.TRUE
+ else:
+ main.log.error( "Did not properly created list of ONOS CLI handle" )
+ stepResult = main.FALSE
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully construct " +
+ "test variables ",
+ onfail="Failed to construct test variables" )
+
+ if gitPull == 'True':
+ main.step( "Building ONOS in " + gitBranch + " branch" )
+ onosBuildResult = main.startUp.onosBuild( main, gitBranch )
+ stepResult = onosBuildResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully compiled " +
+ "latest ONOS",
+ onfail="Failed to compile " +
+ "latest ONOS" )
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
+ main.numCtrls = int( main.scale[ 0 ] )
+
+ def CASE2( self, main ):
+ """
+ - Set up cell
+ - Create cell file
+ - Set cell file
+ - Verify cell file
+ - Kill ONOS process
+ - Uninstall ONOS cluster
+ - Verify ONOS start up
+ - Install ONOS cluster
+ - Connect to cli
+ """
+
+ # main.scale[ 0 ] determines the current number of ONOS controller
+ main.numCtrls = int( main.scale[ 0 ] )
+ main.scale.remove( main.scale[ 0 ] )
+ apps=main.apps
+ if main.diff:
+ apps = main.apps+","+main.diff.pop(0)
+ else: main.log.error( "App list is empty" )
+ main.case( "Package and start ONOS using apps:" + apps)
+
+ print "NODE COUNT = ", main.numCtrls
+ print main.ONOSip
+ tempOnosIp = []
+ for i in range( main.numCtrls ):
+ tempOnosIp.append( main.ONOSip[i] )
+
+ onosUser = main.params[ 'ENV' ][ 'cellUser' ]
+ main.step("Create and Apply cell file")
+ main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
+ "temp",
+ main.Mininet1.ip_address,
+ apps,
+ tempOnosIp,
+ onosUser )
+
+ cellResult = main.ONOSbench.setCell( "temp" )
+ verifyResult = main.ONOSbench.verifyCell()
+ stepResult = cellResult and verifyResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully applied cell to " + \
+ "environment",
+ onfail="Failed to apply cell to environment " )
+ #kill off all onos processes
+ main.log.info( "Safety check, killing all ONOS processes" +
+ " before initiating environment setup" )
+
+ for i in range( main.maxNodes ):
+ main.ONOSbench.onosDie( main.ONOSip[ i ] )
+
+ main.step( "Create and Install ONOS package" )
+ main.jsonFile=main.json.pop(0)
+ main.ONOSbench.handle.sendline( "cp "+main.dependencyPath+"/"+main.jsonFile+".json ~/onos/tools/package/config/network-cfg.json")
+ packageResult = main.ONOSbench.onosPackage()
+
+ onosInstallResult = main.TRUE
+ for i in range( main.numCtrls ):
+ onosInstallResult = onosInstallResult and \
+ main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+ stepResult = onosInstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully installed ONOS package",
+ onfail="Failed to install ONOS package" )
+
+ main.step( "Starting ONOS service" )
+ stopResult = main.TRUE
+ startResult = main.TRUE
+ onosIsUp = main.TRUE
+
+ for i in range( main.numCtrls ):
+ onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+ if onosIsUp == main.TRUE:
+ main.log.report( "ONOS instance is up and ready" )
+ else:
+ main.log.report( "ONOS instance may not be up, stop and " +
+ "start ONOS again " )
+ for i in range( main.numCtrls ):
+ stopResult = stopResult and \
+ main.ONOSbench.onosStop( main.ONOSip[ i ] )
+ for i in range( main.numCtrls ):
+ startResult = startResult and \
+ main.ONOSbench.onosStart( main.ONOSip[ i ] )
+ stepResult = onosIsUp and stopResult and startResult
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+ main.step( "Checking if ONOS CLI is ready" )
+ main.CLIs[0].startCellCli()
+ cliResult = main.CLIs[0].startOnosCli( main.ONOSip[ 0 ],
+ commandlineTimeout=60, onosStartTimeout=100 )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cliResult,
+ onpass="ONOS CLI is ready",
+ onfail="ONOS CLI is not ready" )
+ for i in range( 10 ):
+ ready = True
+ output = main.CLIs[0].summary()
+ if not output:
+ ready = False
+ if ready:
+ break
+ time.sleep( 10 )
+ utilities.assert_equals( expect=True, actual=ready,
+ onpass="ONOS summary command succeded",
+ onfail="ONOS summary command failed" )
+
+ if not ready:
+ main.log.error( "ONOS startup failed!" )
+ main.cleanup()
+ main.exit()
+
+ def CASE3( self, main ):
+ """
+ Start mininet
+ """
+ main.case( "Start Leaf-Spine "+main.jsonFile+" Mininet Topology" )
+ main.log.report( "Start Mininet topology" )
+
+ main.step( "Starting Mininet Topology" )
+ args,topo=" "," "
+ if main.args:
+ args = "--onos %d %s" % (main.numCtrls, main.args.pop(0))
+ else: main.log.error( "Argument list is empty" )
+
+ topoResult = main.Mininet1.startNet( topoFile= main.dependencyPath + main.topology, args=args )
+ stepResult = topoResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully loaded topology",
+ onfail="Failed to load topology" )
+ # Exit if topology did not load properly
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+ main.step(" Check whether the flow count is bigger than 116" )
+ count = utilities.retry( main.CLIs[0].checkFlowCount,
+ main.FALSE,
+ kwargs={'min':116},
+ attempts=10,
+ sleep=10 )
+ utilities.assertEquals( \
+ expect=True,
+ actual=(count>0),
+ onpass="Flow count looks correct: "+str(count),
+ onfail="Flow count looks wrong: "+str(count) )
+
+ main.step( "Check whether all flow status are ADDED" )
+ flowCheck = utilities.retry( main.CLIs[0].checkFlowsState,
+ main.FALSE,
+ kwargs={'isPENDING':False},
+ attempts=10,
+ sleep=10)
+ utilities.assertEquals( \
+ expect=main.TRUE,
+ actual=flowCheck,
+ onpass="Flow status is correct!",
+ onfail="Flow status is wrong!" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir, "flowsBefore" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir, "groupsBefore" + main.jsonFile)
+ main.count=1
+
+ def CASE4( self, main ):
+ main.case( "Check full connectivity" )
+ main.log.report( "Check full connectivity" )
+
+ main.step("Check full connectivity"+str(main.count))
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Full connectivity successfully tested",
+ onfail="Full connectivity failed" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir, "flowsAfter" + str(main.count) + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir, "groupsAfter" + str(main.count) + main.jsonFile)
+
+ def CASE5( self, main ):
+ """
+ Link spine101-leaf2 down
+ Pingall
+ Link spine101-leaf2 up
+ Pingall
+ """
+ import time
+ assert main.numCtrls, "main.numCtrls not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert main.CLIs, "main.CLIs not defined"
+
+ linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
+
+ description = "Turn off a link to ensure that Segment Routing reroutes traffic " + \
+ "properly"
+ main.case( description )
+
+ main.step( "Kill Link between spine101 and leaf2" )
+ LinkDown = main.Mininet1.link( END1="spine101", END2="leaf2", OPTION="down" )
+ main.log.info( "Waiting %s seconds for link up to be discovered" % (linkSleep) )
+ # TODO Maybe parameterize number of expected links
+ time.sleep( linkSleep )
+ topology = utilities.retry( main.CLIs[0].checkStatus,
+ main.FALSE,
+ kwargs={'numoswitch':'4', 'numolink':'6'},
+ attempts=10,
+ sleep=linkSleep)
+ result = topology & LinkDown
+ utilities.assert_equals( expect=main.TRUE, actual=result,
+ onpass="Link down successful",
+ onfail="Failed to turn off link?" )
+
+ main.step( "Check connectivity after link failure" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived link failure successfully",
+ onfail="Did not survive link failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir,
+ "flowsAfterLinkFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir,
+ "groupsAfterLinkFailure" + main.jsonFile)
+
+ main.step( "Restore link between spine101 and leaf2" )
+
+ result = False
+ count=0
+ while True:
+ count+=0
+ main.Mininet1.link( END1="spine101", END2="leaf2", OPTION="up" )
+ main.Mininet1.link( END2="spine101", END1="leaf2", OPTION="up" )
+ main.log.info( "Waiting %s seconds for link up to be discovered" % (linkSleep) )
+ time.sleep( linkSleep )
+
+ main.CLIs[0].portstate(dpid='of:0000000000000002', port='1')
+ main.CLIs[0].portstate(dpid='of:0000000000000101', port='2')
+ time.sleep( linkSleep )
+
+ result = main.CLIs[0].checkStatus(numoswitch='4', numolink='8')
+ if count>10 or result:
+ break
+ utilities.assert_equals( expect=main.TRUE, actual=result,
+ onpass="Link up successful",
+ onfail="Failed to bring link up" )
+
+ main.step( "Check connectivity after link recovery" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived link recovery successfully",
+ onfail="Did not survive link recovery" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir,
+ "flowsAfterLinkRecovery" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir,
+ "groupsAfterLinkRecovery" + main.jsonFile)
+
+ def CASE6( self, main ):
+ """
+ Switch Down
+ Pingall
+ Switch up
+ Pingall
+ """
+ import time
+ assert main.numCtrls, "main.numCtrls not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert main.CLIs, "main.CLIs not defined"
+
+ switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
+
+ description = "Killing a switch to ensure it is discovered correctly"
+ onosCli = main.CLIs[0]
+ main.case( description )
+ switch = main.params[ 'kill' ][ 'switch' ]
+ switchDPID = main.params[ 'kill' ][ 'dpid' ]
+
+ main.step( "Kill " + switch )
+ main.log.info( "Stopping" + switch )
+ main.Mininet1.switch( SW=switch, OPTION="stop")
+ main.log.info( "Waiting %s seconds for switch down to be discovered" %(switchSleep))
+ time.sleep( switchSleep )
+ topology = utilities.retry( main.CLIs[0].checkStatus,
+ main.FALSE,
+ kwargs={'numoswitch':'3', 'numolink':'4'},
+ attempts=10,
+ sleep=switchSleep)
+ utilities.assert_equals( expect=main.TRUE, actual=topology,
+ onpass="Kill switch successful",
+ onfail="Failed to kill switch?" )
+
+ main.step( "Check connectivity after switch failure" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived switch failure successfully",
+ onfail="Did not survive switch failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir,
+ "flowsAfterSwitchFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir,
+ "groupsAfterSwitchFailure" + main.jsonFile)
+
+ main.step( "Recovering " + switch )
+ main.log.info( "Starting" + switch )
+ main.Mininet1.switch( SW=switch, OPTION="start")
+ main.log.info( "Waiting %s seconds for switch up to be discovered" %(switchSleep))
+ time.sleep( switchSleep )
+ topology = utilities.retry( main.CLIs[0].checkStatus,
+ main.FALSE,
+ kwargs={'numoswitch':'4', 'numolink':'8'},
+ attempts=10,
+ sleep=switchSleep)
+ utilities.assert_equals( expect=main.TRUE, actual=topology,
+ onpass="Switch recovery successful",
+ onfail="Failed to recover switch?" )
+
+ main.step( "Check connectivity after switch recovery" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived switch recovery successfully",
+ onfail="Did not survive switch recovery" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir,
+ "flowsAfterSwitchRecovery" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir,
+ "groupsAfterSwitchRecovery" + main.jsonFile)
+
+ def CASE7( self, main ):
+ """
+ OnosInstance1 Down
+ Pingall
+ OnosInstance1 up
+ Pingall
+ """
+
+ description = "Killing single instance to test controlplane resilience of Segment Routing"
+ main.case( description )
+ main.step( "Killing ONOS instance" )
+ killResult = main.ONOSbench.onosDie( main.CLIs[0].ip_address )
+ utilities.assert_equals( expect=main.TRUE, actual=killResult,
+ onpass="ONOS instance Killed",
+ onfail="Error killing ONOS instance" )
+ time.sleep( 12 )
+ topology = utilities.retry( main.CLIs[1].checkStatus,
+ main.FALSE,
+ kwargs={'numoswitch':'4', 'numolink':'8', 'numoctrl':'2'},
+ attempts=10,
+ sleep=12)
+ utilities.assert_equals( expect=main.TRUE, actual=topology,
+ onpass="ONOS Instance down successful",
+ onfail="Failed to turn off ONOS Instance" )
+ main.step( "Check connectivity after ONOS instance failure" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived instance failure successfully",
+ onfail="Did not survive instance failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[1], main.logdir,
+ "flowsAfterSwitchFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[1], main.logdir,
+ "groupsAfterSwitchFailure" + main.jsonFile)
+ main.step( "Recovering ONOS instance" )
+ startResult = main.ONOSbench.onosStart( main.CLIs[0].ip_address )
+ isUp = main.ONOSbench.isup( main.ONOSip[ 0 ] )
+ utilities.assert_equals( expect=main.TRUE, actual=isUp,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+ main.step( "Checking if ONOS CLI is ready" )
+ main.CLIs[0].startCellCli()
+ cliResult = main.CLIs[0].startOnosCli( main.ONOSip[ 0 ],
+ commandlineTimeout=60, onosStartTimeout=100 )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cliResult,
+ onpass="ONOS CLI is ready",
+ onfail="ONOS CLI is not ready" )
+ for i in range( 10 ):
+ ready = True
+ output = main.CLIs[0].summary()
+ if not output:
+ ready = False
+ if ready:
+ break
+ time.sleep( 10 )
+ utilities.assert_equals( expect=True, actual=ready,
+ onpass="ONOS summary command succeded",
+ onfail="ONOS summary command failed" )
+ if not ready:
+ main.log.error( "ONOS startup failed!" )
+ main.cleanup()
+ main.exit()
+ main.step(" Check whether the flow count is bigger than 116" )
+ count = utilities.retry( main.CLIs[0].checkFlowCount,
+ main.FALSE,
+ kwargs={'min':116},
+ attempts=10,
+ sleep=10 )
+ utilities.assertEquals( expect=True,
+ actual=(count>0),
+ onpass="Flow count looks correct: "+str(count),
+ onfail="Flow count looks wrong: "+str(count) )
+
+ main.step( "Check connectivity after ONOS instance recovery" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived instance recovery successfully",
+ onfail="Did not survive recovery failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0], main.logdir,
+ "flowsAfterCtrlFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0], main.logdir,
+ "groupsAfterCtrlFailure" + main.jsonFile)
+
+ def CASE8( self, main ):
+ """
+ Cluster Down
+ Pingall
+ Cluster up
+ Pingall
+ """
+ description = "Killing all instances to test controlplane resilience of Segment Routing"
+ main.case( description )
+ main.step( "Shutting down ONOS Cluster" )
+ for i in range( main.numCtrls ):
+ killResult = main.ONOSbench.onosDie( main.CLIs[i].ip_address )
+ utilities.assert_equals( expect=main.TRUE, actual=killResult,
+ onpass="ONOS instance Killed",
+ onfail="Error killing ONOS instance" )
+ time.sleep(12)
+ main.step( "Check connectivity after ONOS instance failure" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived instance failure successfully",
+ onfail="Did not survive instance failure" )
+ main.step( "Recovering ONOS Cluster" )
+ for i in range( main.numCtrls ):
+ startResult = main.ONOSbench.onosStart( main.CLIs[i].ip_address )
+ isUp = main.ONOSbench.isup( main.ONOSip[ i ] )
+ utilities.assert_equals( expect=main.TRUE, actual=isUp,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+ main.step( "Checking if ONOS CLI is ready" )
+ for i in range( main.numCtrls ):
+ main.CLIs[i].startCellCli()
+ cliResult = main.CLIs[i].startOnosCli( main.ONOSip[ i ],
+ commandlineTimeout=60, onosStartTimeout=100 )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cliResult,
+ onpass="ONOS CLI is ready",
+ onfail="ONOS CLI is not ready" )
+ for i in range( 10 ):
+ ready = True
+ output = main.CLIs[0].summary()
+ if not output:
+ ready = False
+ if ready:
+ break
+ time.sleep( 10 )
+ utilities.assert_equals( expect=True, actual=ready,
+ onpass="ONOS summary command succeded",
+ onfail="ONOS summary command failed" )
+ if not ready:
+ main.log.error( "ONOS startup failed!" )
+ main.cleanup()
+ main.exit()
+
+ main.step(" Check whether the flow count is bigger than 116" )
+ count = utilities.retry( main.CLIs[0].checkFlowCount,
+ main.FALSE,
+ kwargs={'min':116},
+ attempts=10,
+ sleep=10 )
+ utilities.assertEquals( expect=True,
+ actual=(count>0),
+ onpass="Flow count looks correct: "+str(count),
+ onfail="Flow count looks wrong: "+str(count) )
+
+ main.step( "Check connectivity after CLUSTER recovery" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived instance recovery successfully",
+ onfail="Did not survive recovery failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0], main.logdir,
+ "flowsAfterCtrlFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0], main.logdir,
+ "groupsAfterCtrlFailure" + main.jsonFile)
+
+ def CASE10( self, main ):
+ '''
+ Report errors/warnings/exceptions
+ '''
+ main.case( "Logging test for " + main.jsonFile )
+ for i in range(main.numCtrls):
+ main.ONOSbench.onosStop( main.ONOSip[i] )
+ main.Mininet1.stopNet()
+
+ main.ONOSbench.scp( main.ONOScli1 ,
+ "/opt/onos/log/karaf.log",
+ "/tmp/karaf.log",
+ direction="from" )
+ main.ONOSbench.cpLogsToDir("/tmp/karaf.log",main.logdir,
+ copyFileName="karaf.log."+main.jsonFile+str(len(main.json)))
+ #main.ONOSbench.logReport( main.ONOSip[ 0 ],
+ # [ "INFO" ],
+ # "a" )
+ #main.log.info("Error report: \n" )
+ main.ONOSbench.logReport( main.ONOSip[ 0 ],
+ [ "INFO",
+ "FOLLOWER",
+ "WARN",
+ "flow",
+ "ERROR",
+ "Except" ],
+ "s" )
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.topo b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
similarity index 61%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.topo
rename to TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
index a5266a0..546a023 100755
--- a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
@@ -13,7 +13,7 @@
</ONOSbench>
<ONOScli1>
- <host>localhost</host>
+ <host>OC1</host>
<user>sdn</user>
<password>rocks</password>
<type>OnosCliDriver</type>
@@ -21,13 +21,34 @@
<COMPONENTS>
</COMPONENTS>
</ONOScli1>
+
+ <ONOScli2>
+ <host>OC2</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>OC3</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli3>
+
<Mininet1>
<host>OCN</host>
<user>sdn</user>
<password>rocks</password>
<type>MininetCliDriver</type>
- <connect_order>3</connect_order>
+ <connect_order>5</connect_order>
<COMPONENTS>
<home>~/mininet/custom/</home>
</COMPONENTS>
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRSanity/__init__.py
old mode 100644
new mode 100755
similarity index 100%
copy from TestON/tests/USECASE/USECASE_SegmentRouting/__init__.py
copy to TestON/tests/USECASE/SegmentRouting/SRSanity/__init__.py
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/__init__.py b/TestON/tests/USECASE/SegmentRouting/__init__.py
old mode 100644
new mode 100755
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/__init__.py
rename to TestON/tests/USECASE/SegmentRouting/__init__.py
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/2x2.json b/TestON/tests/USECASE/SegmentRouting/dependencies/2x2.json
old mode 100644
new mode 100755
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/2x2.json
rename to TestON/tests/USECASE/SegmentRouting/dependencies/2x2.json
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/4x4.json b/TestON/tests/USECASE/SegmentRouting/dependencies/4x4.json
old mode 100644
new mode 100755
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/4x4.json
rename to TestON/tests/USECASE/SegmentRouting/dependencies/4x4.json
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/dependencies/cord_fabric.py b/TestON/tests/USECASE/SegmentRouting/dependencies/cord_fabric.py
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/dependencies/cord_fabric.py
rename to TestON/tests/USECASE/SegmentRouting/dependencies/cord_fabric.py
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/dependencies/startUp.py b/TestON/tests/USECASE/SegmentRouting/dependencies/startUp.py
old mode 100644
new mode 100755
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/dependencies/startUp.py
rename to TestON/tests/USECASE/SegmentRouting/dependencies/startUp.py
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/README.md b/TestON/tests/USECASE/USECASE_SegmentRouting/README.md
deleted file mode 100644
index 8240068..0000000
--- a/TestON/tests/USECASE/USECASE_SegmentRouting/README.md
+++ /dev/null
@@ -1,33 +0,0 @@
-This test is designed to verify basic connectivity the SegmentRouting application via pingaall.
-
-It consists of
-
-1) Configure and Install ONOS
-2) Start Mininet and check flow state
-2) Test connectivity
-
-Requirements
-
- - A single ONOS instance is required for the test, the application is currently not stable in a cluster as of today.
- - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
-
-Step 1:
-In this step, we copy config file to ONOS, next we package ONOS and install it in the target machine.
-
-Step 2:
-
-In this step we start the topology, connect to ONOS
-
-Step:3
-
-Here, we send several pings between hosts to test connectivity.
-
-Then Steps are repeated for different configurations and topologies.
-
- Configurations:
- 1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
- 2) APPS=openflow
-
- Topologies:
- 1) 2x2 Leaf-Spine
- 2) 4x4 Leaf-Spine
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.py b/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.py
deleted file mode 100755
index 1a4834b..0000000
--- a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.py
+++ /dev/null
@@ -1,320 +0,0 @@
-
-# This test is to determine if the Segment Routing application is working properly
-
-class USECASE_SegmentRouting:
-
- def __init__( self ):
- self.default = ''
-
- def CASE1( self, main ):
- import time
- import os
- import imp
- import re
-
- """
- - Construct tests variables
- - GIT ( optional )
- - Checkout ONOS master branch
- - Pull latest ONOS code
- - Building ONOS ( optional )
- - Install ONOS package
- - Build ONOS package
- """
-
- main.case( "Constructing test variables and building ONOS" )
- main.step( "Constructing test variables" )
- stepResult = main.FALSE
-
- # Test variables
- main.cellName = main.params[ 'ENV' ][ 'cellName' ]
- main.apps = main.params[ 'ENV' ][ 'cellApps' ]
- main.diff = []
- main.diff.extend(( main.params[ 'ENV' ][ 'diffApps' ] ).split(";"))
- main.diff.extend(( main.params[ 'ENV' ][ 'diffApps' ] ).split(";"))
- gitBranch = main.params[ 'GIT' ][ 'branch' ]
- main.path = os.path.dirname( main.testFile )
- main.dependencyPath = main.path + "/dependencies/"
- main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
- #main.json = ["0x1","0x1"]
- main.json = ["2x2", "2x2","4x4","4x4"]
- main.args = [" ", " ", " --spine 4 --leaf 4 ", " --spine 4 --leaf 4 "]
- #main.args = [" --spine 0 --leaf 1 "," --spine 0 --leaf 1 "]
- main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
- main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
- main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
- wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
- main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
- gitPull = main.params[ 'GIT' ][ 'pull' ]
- main.cellData = {} # for creating cell file
- main.CLIs = []
- main.ONOSip = []
-
- main.ONOSip = main.ONOSbench.getOnosIps()
-
- # Assigning ONOS cli handles to a list
- for i in range( 1, main.maxNodes + 1 ):
- main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
-
- # -- INIT SECTION, ONLY RUNS ONCE -- #
- main.startUp = imp.load_source( wrapperFile1,
- main.dependencyPath +
- wrapperFile1 +
- ".py" )
-
- copyResult1 = main.ONOSbench.scp( main.Mininet1,
- main.dependencyPath +
- main.topology,
- main.Mininet1.home,
- direction="to" )
- if main.CLIs:
- stepResult = main.TRUE
- else:
- main.log.error( "Did not properly created list of ONOS CLI handle" )
- stepResult = main.FALSE
-
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="Successfully construct " +
- "test variables ",
- onfail="Failed to construct test variables" )
-
- if gitPull == 'True':
- main.step( "Building ONOS in " + gitBranch + " branch" )
- onosBuildResult = main.startUp.onosBuild( main, gitBranch )
- stepResult = onosBuildResult
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="Successfully compiled " +
- "latest ONOS",
- onfail="Failed to compile " +
- "latest ONOS" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
-
- def CASE2( self, main ):
- """
- - Set up cell
- - Create cell file
- - Set cell file
- - Verify cell file
- - Kill ONOS process
- - Uninstall ONOS cluster
- - Verify ONOS start up
- - Install ONOS cluster
- - Connect to cli
- """
-
- # main.scale[ 0 ] determines the current number of ONOS controller
- main.numCtrls = int( main.scale[ 0 ] )
- apps=main.apps
- if main.diff:
- apps = main.apps+","+main.diff.pop(0)
- else: main.log.error( "App list is empty" )
- main.case( "Package and start ONOS using apps:" + apps)
-
- print "NODE COUNT = ", main.numCtrls
-
- tempOnosIp = []
- for i in range( main.numCtrls ):
- tempOnosIp.append( main.ONOSip[i] )
-
- onosUser = main.params[ 'ENV' ][ 'cellUser' ]
- main.step("Create and Apply cell file")
- main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
- "temp",
- main.Mininet1.ip_address,
- apps,
- tempOnosIp,
- onosUser )
-
- cellResult = main.ONOSbench.setCell( "temp" )
- verifyResult = main.ONOSbench.verifyCell()
- stepResult = cellResult and verifyResult
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="Successfully applied cell to " + \
- "environment",
- onfail="Failed to apply cell to environment " )
- #kill off all onos processes
- main.log.info( "Safety check, killing all ONOS processes" +
- " before initiating environment setup" )
-
- for i in range( main.maxNodes ):
- main.ONOSbench.onosDie( main.ONOSip[ i ] )
-
- main.step( "Create and Install ONOS package" )
- main.jsonFile=main.json.pop(0)
- main.ONOSbench.handle.sendline( "cp "+main.path+"/"+main.jsonFile+".json ~/onos/tools/package/config/network-cfg.json")
- packageResult = main.ONOSbench.onosPackage()
- #stepResult = packageResult
- #utilities.assert_equals( expect=main.TRUE,
- # actual=stepResult,
- # onpass="Successfully created ONOS package",
- # onfail="Failed to create ONOS package" )
-
- #time.sleep( main.startUpSleep )
-
- onosInstallResult = main.TRUE
- for i in range( main.numCtrls ):
- onosInstallResult = onosInstallResult and \
- main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
- stepResult = onosInstallResult
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="Successfully installed ONOS package",
- onfail="Failed to install ONOS package" )
-
- main.step( "Starting ONOS service" )
- stopResult = main.TRUE
- startResult = main.TRUE
- onosIsUp = main.TRUE
-
- for i in range( main.numCtrls ):
- onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
- if onosIsUp == main.TRUE:
- main.log.report( "ONOS instance is up and ready" )
- else:
- main.log.report( "ONOS instance may not be up, stop and " +
- "start ONOS again " )
- for i in range( main.numCtrls ):
- stopResult = stopResult and \
- main.ONOSbench.onosStop( main.ONOSip[ i ] )
- for i in range( main.numCtrls ):
- startResult = startResult and \
- main.ONOSbench.onosStart( main.ONOSip[ i ] )
- stepResult = onosIsUp and stopResult and startResult
-
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="ONOS service is ready",
- onfail="ONOS service did not start properly" )
- #time.sleep( 2*main.startUpSleep )
- #main.ONOSbench.handle.sendline( "onos-secure-ssh")
- main.step( "Checking if ONOS CLI is ready" )
- cellResult = main.CLIs[0].setCell( "temp" )
-
- cliResult = main.CLIs[0].startOnosCli( main.ONOSip[ 0 ],
- commandlineTimeout=60, onosStartTimeout=100 )
- utilities.assert_equals( expect=main.TRUE,
- actual=cliResult,
- onpass="ONOS CLI is ready",
- onfail="ONOS CLI is not ready" )
- for i in range( 10 ):
- ready = True
- output = main.CLIs[0].summary()
- if not output:
- ready = False
- if ready:
- break
- time.sleep( 10 )
- utilities.assert_equals( expect=True, actual=ready,
- onpass="ONOS summary command succeded",
- onfail="ONOS summary command failed" )
-
- if not ready:
- main.log.error( "ONOS startup failed!" )
- main.cleanup()
- main.exit()
-
- def CASE3( self, main ):
- """
- Start mininet
- """
- main.case( "Start Leaf-Spine "+main.jsonFile+" Mininet Topology" )
- main.log.report( "Start Mininet topology" )
-
- main.step( "Starting Mininet Topology" )
- args,topo=" "," "
- #if main.topology:
- # topo = main.topology.pop(0)
- #else: main.log.error( "Topo list is empty" )
- if main.args:
- args = "--onos 1 " + main.args.pop(0)
- else: main.log.error( "Argument list is empty" )
-
- topoResult = main.Mininet1.startNet( topoFile= main.dependencyPath + main.topology, args=args )
- stepResult = topoResult
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="Successfully loaded topology",
- onfail="Failed to load topology" )
- # Exit if topology did not load properly
- if not topoResult:
- main.cleanup()
- main.exit()
- #main.step("Waiting for switch initialization and configuration")
- main.step(" Check whether the flow count is bigger than 116" )
- count = utilities.retry( main.CLIs[0].checkFlowCount,
- main.FALSE,
- kwargs={'min':116},
- attempts=10 )
- utilities.assertEquals( \
- expect=True,
- actual=(count>0),
- onpass="Flow count looks correct: "+str(count),
- onfail="Flow count looks wrong: "+str(count) )
-
- main.step( "Check whether all flow status are ADDED" )
- flowCheck = utilities.retry( main.CLIs[0].checkFlowsState,
- main.FALSE,
- kwargs={'isPENDING':False},
- attempts=10 )
- utilities.assertEquals( \
- expect=main.TRUE,
- actual=flowCheck,
- onpass="Flow status is correct!",
- onfail="Flow status is wrong!" )
- main.ONOSbench.dumpFlows( main.ONOSip[0],
- main.logdir, "flowsBefore" + main.jsonFile)
- main.ONOSbench.dumpGroups( main.ONOSip[0],
- main.logdir, "groupsBefore" + main.jsonFile)
- #time.sleep( 3*main.startUpSleep)
- main.count=1
-
- def CASE4( self, main ):
- main.case( "Check full connectivity" )
- main.log.report( "Check full connectivity" )
-
- main.step("Check full connectivity"+str(main.count))
- pa = main.Mininet1.pingall()
- utilities.assert_equals( expect=main.TRUE, actual=pa,
- onpass="Full connectivity successfully tested",
- onfail="Full connectivity failed" )
- # cleanup mininet
- main.ONOSbench.dumpFlows( main.ONOSip[0],
- main.logdir, "flowsAfter" + str(main.count) + main.jsonFile)
- main.ONOSbench.dumpGroups( main.ONOSip[0],
- main.logdir, "groupsAfter" + str(main.count) + main.jsonFile)
-
- def CASE10( self, main ):
- '''
- Report errors/warnings/exceptions
- '''
- main.case( "Logging test for " + main.jsonFile )
- #if len(main.json) > 0 :
- main.ONOSbench.onosStop( main.ONOSip[0] )
- main.Mininet1.stopNet()
-
- main.ONOSbench.scp( main.ONOScli1 ,
- "/opt/onos/log/karaf.log",
- "/tmp/karaf.log",
- direction="from" )
- main.ONOSbench.cpLogsToDir("/tmp/karaf.log",main.logdir,
- copyFileName="karaf.log."+main.jsonFile+str(len(main.json)))
- #main.ONOSbench.logReport( main.ONOSip[ 0 ],
- # [ "INFO" ],
- # "a" )
- #main.log.info("Error report: \n" )
- main.ONOSbench.logReport( main.ONOSip[ 0 ],
- [ "INFO",
- "FOLLOWER",
- "WARN",
- "flow",
- "ERROR",
- "Except" ],
- "s" )
-
-
-