Move current to SR Sanity tests
Change-Id: I8727fb969c8bfb3806c3ffa78ad47c43762b5621
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
new file mode 100755
index 0000000..59c1214
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
@@ -0,0 +1,16 @@
+This test verifies basic connectivity using SegmentRouting via pingall,
+it should not fail.
+
+It consists of
+
+1) Configure and Install ONOS
+2) Start Mininet and check flow state
+3) Pingall
+
+Requirements
+
+ - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+ 1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+The topology is a 2x2 Leaf-spine
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.params b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.params
new file mode 100755
index 0000000..24403e2
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.params
@@ -0,0 +1,46 @@
+<PARAMS>
+
+ <testcases>1,2,3,4,10</testcases>
+
+ <SCALE>
+ <size>1</size>
+ <max>1</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <wrapper1>startUp</wrapper1>
+ <topology>cord_fabric.py</topology>
+ </DEPENDENCY>
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,segmentrouting</cellApps>
+ <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
+ <cellUser>sdn</cellUser>
+ </ENV>
+
+ <GIT>
+ <pull>False</pull>
+ <branch>master</branch>
+ </GIT>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ </timers>
+
+ <kill>
+ <switch> spine101 </switch>
+ <dpid> 000000000101 </dpid>
+ <links> leaf1 leaf2 </links>
+ </kill>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py
new file mode 100755
index 0000000..9a0d91c
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py
@@ -0,0 +1,619 @@
+# This test should always succeed. it runs cases 1,2,3,4
+#CASE1: Get and Build ONOS
+#CASE2: Package and Install ONOS
+#CASE3: Start Mininet and check flows
+#CASE4: Ping all
+#CASE5: Link Failure
+#CASE6: Switch Failure
+#CASE7: ONOS Failure
+#CASE8: CLUSTER Failure
+#CASE10: Logging
+
+class SRSanity:
+
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ import time
+ import os
+ import imp
+ import re
+
+ """
+ - Construct tests variables
+ - GIT ( optional )
+ - Checkout ONOS master branch
+ - Pull latest ONOS code
+ - Building ONOS ( optional )
+ - Install ONOS package
+ - Build ONOS package
+ """
+
+ main.case( "Constructing test variables and building ONOS" )
+ main.step( "Constructing test variables" )
+ stepResult = main.FALSE
+
+ # Test variables
+ main.cellName = main.params[ 'ENV' ][ 'cellName' ]
+ main.apps = main.params[ 'ENV' ][ 'cellApps' ]
+ main.diff = []
+ main.diff.extend(( main.params[ 'ENV' ][ 'diffApps' ] ).split(";"))
+ main.diff=main.diff*2
+ gitBranch = main.params[ 'GIT' ][ 'branch' ]
+ main.path = os.path.dirname( main.testFile )
+ main.dependencyPath = main.path +"/../dependencies/"
+ main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
+ main.json = ["2x2"]*4
+ main.args = [" "]*4
+ main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
+ main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
+ main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
+ wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
+ main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
+ gitPull = main.params[ 'GIT' ][ 'pull' ]
+ main.cellData = {} # for creating cell file
+ main.CLIs = []
+ main.ONOSip = []
+
+ # Assigning ONOS cli handles to a list
+ for i in range( 1, main.maxNodes + 1 ):
+ main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+ main.ONOSip.append( main.CLIs[i-1].ip_address )
+ # -- INIT SECTION, ONLY RUNS ONCE -- #
+ main.startUp = imp.load_source( wrapperFile1,
+ main.dependencyPath +
+ wrapperFile1 +
+ ".py" )
+
+ copyResult1 = main.ONOSbench.scp( main.Mininet1,
+ main.dependencyPath +
+ main.topology,
+ main.Mininet1.home,
+ direction="to" )
+ if main.CLIs:
+ stepResult = main.TRUE
+ else:
+ main.log.error( "Did not properly created list of ONOS CLI handle" )
+ stepResult = main.FALSE
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully construct " +
+ "test variables ",
+ onfail="Failed to construct test variables" )
+
+ if gitPull == 'True':
+ main.step( "Building ONOS in " + gitBranch + " branch" )
+ onosBuildResult = main.startUp.onosBuild( main, gitBranch )
+ stepResult = onosBuildResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully compiled " +
+ "latest ONOS",
+ onfail="Failed to compile " +
+ "latest ONOS" )
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
+ main.numCtrls = int( main.scale[ 0 ] )
+
+ def CASE2( self, main ):
+ """
+ - Set up cell
+ - Create cell file
+ - Set cell file
+ - Verify cell file
+ - Kill ONOS process
+ - Uninstall ONOS cluster
+ - Verify ONOS start up
+ - Install ONOS cluster
+ - Connect to cli
+ """
+
+ # main.scale[ 0 ] determines the current number of ONOS controller
+ main.numCtrls = int( main.scale[ 0 ] )
+ main.scale.remove( main.scale[ 0 ] )
+ apps=main.apps
+ if main.diff:
+ apps = main.apps+","+main.diff.pop(0)
+ else: main.log.error( "App list is empty" )
+ main.case( "Package and start ONOS using apps:" + apps)
+
+ print "NODE COUNT = ", main.numCtrls
+ print main.ONOSip
+ tempOnosIp = []
+ for i in range( main.numCtrls ):
+ tempOnosIp.append( main.ONOSip[i] )
+
+ onosUser = main.params[ 'ENV' ][ 'cellUser' ]
+ main.step("Create and Apply cell file")
+ main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
+ "temp",
+ main.Mininet1.ip_address,
+ apps,
+ tempOnosIp,
+ onosUser )
+
+ cellResult = main.ONOSbench.setCell( "temp" )
+ verifyResult = main.ONOSbench.verifyCell()
+ stepResult = cellResult and verifyResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully applied cell to " + \
+ "environment",
+ onfail="Failed to apply cell to environment " )
+ #kill off all onos processes
+ main.log.info( "Safety check, killing all ONOS processes" +
+ " before initiating environment setup" )
+
+ for i in range( main.maxNodes ):
+ main.ONOSbench.onosDie( main.ONOSip[ i ] )
+
+ main.step( "Create and Install ONOS package" )
+ main.jsonFile=main.json.pop(0)
+ main.ONOSbench.handle.sendline( "cp "+main.dependencyPath+"/"+main.jsonFile+".json ~/onos/tools/package/config/network-cfg.json")
+ packageResult = main.ONOSbench.onosPackage()
+
+ onosInstallResult = main.TRUE
+ for i in range( main.numCtrls ):
+ onosInstallResult = onosInstallResult and \
+ main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+ stepResult = onosInstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully installed ONOS package",
+ onfail="Failed to install ONOS package" )
+
+ main.step( "Starting ONOS service" )
+ stopResult = main.TRUE
+ startResult = main.TRUE
+ onosIsUp = main.TRUE
+
+ for i in range( main.numCtrls ):
+ onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+ if onosIsUp == main.TRUE:
+ main.log.report( "ONOS instance is up and ready" )
+ else:
+ main.log.report( "ONOS instance may not be up, stop and " +
+ "start ONOS again " )
+ for i in range( main.numCtrls ):
+ stopResult = stopResult and \
+ main.ONOSbench.onosStop( main.ONOSip[ i ] )
+ for i in range( main.numCtrls ):
+ startResult = startResult and \
+ main.ONOSbench.onosStart( main.ONOSip[ i ] )
+ stepResult = onosIsUp and stopResult and startResult
+
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+ main.step( "Checking if ONOS CLI is ready" )
+ main.CLIs[0].startCellCli()
+ cliResult = main.CLIs[0].startOnosCli( main.ONOSip[ 0 ],
+ commandlineTimeout=60, onosStartTimeout=100 )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cliResult,
+ onpass="ONOS CLI is ready",
+ onfail="ONOS CLI is not ready" )
+ for i in range( 10 ):
+ ready = True
+ output = main.CLIs[0].summary()
+ if not output:
+ ready = False
+ if ready:
+ break
+ time.sleep( 10 )
+ utilities.assert_equals( expect=True, actual=ready,
+ onpass="ONOS summary command succeded",
+ onfail="ONOS summary command failed" )
+
+ if not ready:
+ main.log.error( "ONOS startup failed!" )
+ main.cleanup()
+ main.exit()
+
+ def CASE3( self, main ):
+ """
+ Start mininet
+ """
+ main.case( "Start Leaf-Spine "+main.jsonFile+" Mininet Topology" )
+ main.log.report( "Start Mininet topology" )
+
+ main.step( "Starting Mininet Topology" )
+ args,topo=" "," "
+ if main.args:
+ args = "--onos %d %s" % (main.numCtrls, main.args.pop(0))
+ else: main.log.error( "Argument list is empty" )
+
+ topoResult = main.Mininet1.startNet( topoFile= main.dependencyPath + main.topology, args=args )
+ stepResult = topoResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully loaded topology",
+ onfail="Failed to load topology" )
+ # Exit if topology did not load properly
+ if not topoResult:
+ main.cleanup()
+ main.exit()
+ main.step(" Check whether the flow count is bigger than 116" )
+ count = utilities.retry( main.CLIs[0].checkFlowCount,
+ main.FALSE,
+ kwargs={'min':116},
+ attempts=10,
+ sleep=10 )
+ utilities.assertEquals( \
+ expect=True,
+ actual=(count>0),
+ onpass="Flow count looks correct: "+str(count),
+ onfail="Flow count looks wrong: "+str(count) )
+
+ main.step( "Check whether all flow status are ADDED" )
+ flowCheck = utilities.retry( main.CLIs[0].checkFlowsState,
+ main.FALSE,
+ kwargs={'isPENDING':False},
+ attempts=10,
+ sleep=10)
+ utilities.assertEquals( \
+ expect=main.TRUE,
+ actual=flowCheck,
+ onpass="Flow status is correct!",
+ onfail="Flow status is wrong!" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir, "flowsBefore" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir, "groupsBefore" + main.jsonFile)
+ main.count=1
+
+ def CASE4( self, main ):
+ main.case( "Check full connectivity" )
+ main.log.report( "Check full connectivity" )
+
+ main.step("Check full connectivity"+str(main.count))
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Full connectivity successfully tested",
+ onfail="Full connectivity failed" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir, "flowsAfter" + str(main.count) + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir, "groupsAfter" + str(main.count) + main.jsonFile)
+
+ def CASE5( self, main ):
+ """
+ Link spine101-leaf2 down
+ Pingall
+ Link spine101-leaf2 up
+ Pingall
+ """
+ import time
+ assert main.numCtrls, "main.numCtrls not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert main.CLIs, "main.CLIs not defined"
+
+ linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
+
+ description = "Turn off a link to ensure that Segment Routing reroutes traffic " + \
+ "properly"
+ main.case( description )
+
+ main.step( "Kill Link between spine101 and leaf2" )
+ LinkDown = main.Mininet1.link( END1="spine101", END2="leaf2", OPTION="down" )
+ main.log.info( "Waiting %s seconds for link up to be discovered" % (linkSleep) )
+ # TODO Maybe parameterize number of expected links
+ time.sleep( linkSleep )
+ topology = utilities.retry( main.CLIs[0].checkStatus,
+ main.FALSE,
+ kwargs={'numoswitch':'4', 'numolink':'6'},
+ attempts=10,
+ sleep=linkSleep)
+ result = topology & LinkDown
+ utilities.assert_equals( expect=main.TRUE, actual=result,
+ onpass="Link down successful",
+ onfail="Failed to turn off link?" )
+
+ main.step( "Check connectivity after link failure" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived link failure successfully",
+ onfail="Did not survive link failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir,
+ "flowsAfterLinkFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir,
+ "groupsAfterLinkFailure" + main.jsonFile)
+
+ main.step( "Restore link between spine101 and leaf2" )
+
+ result = False
+ count=0
+ while True:
+ count+=0
+ main.Mininet1.link( END1="spine101", END2="leaf2", OPTION="up" )
+ main.Mininet1.link( END2="spine101", END1="leaf2", OPTION="up" )
+ main.log.info( "Waiting %s seconds for link up to be discovered" % (linkSleep) )
+ time.sleep( linkSleep )
+
+ main.CLIs[0].portstate(dpid='of:0000000000000002', port='1')
+ main.CLIs[0].portstate(dpid='of:0000000000000101', port='2')
+ time.sleep( linkSleep )
+
+ result = main.CLIs[0].checkStatus(numoswitch='4', numolink='8')
+ if count>10 or result:
+ break
+ utilities.assert_equals( expect=main.TRUE, actual=result,
+ onpass="Link up successful",
+ onfail="Failed to bring link up" )
+
+ main.step( "Check connectivity after link recovery" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived link recovery successfully",
+ onfail="Did not survive link recovery" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir,
+ "flowsAfterLinkRecovery" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir,
+ "groupsAfterLinkRecovery" + main.jsonFile)
+
+ def CASE6( self, main ):
+ """
+ Switch Down
+ Pingall
+ Switch up
+ Pingall
+ """
+ import time
+ assert main.numCtrls, "main.numCtrls not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert main.CLIs, "main.CLIs not defined"
+
+ switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
+
+ description = "Killing a switch to ensure it is discovered correctly"
+ onosCli = main.CLIs[0]
+ main.case( description )
+ switch = main.params[ 'kill' ][ 'switch' ]
+ switchDPID = main.params[ 'kill' ][ 'dpid' ]
+
+ main.step( "Kill " + switch )
+ main.log.info( "Stopping" + switch )
+ main.Mininet1.switch( SW=switch, OPTION="stop")
+ main.log.info( "Waiting %s seconds for switch down to be discovered" %(switchSleep))
+ time.sleep( switchSleep )
+ topology = utilities.retry( main.CLIs[0].checkStatus,
+ main.FALSE,
+ kwargs={'numoswitch':'3', 'numolink':'4'},
+ attempts=10,
+ sleep=switchSleep)
+ utilities.assert_equals( expect=main.TRUE, actual=topology,
+ onpass="Kill switch successful",
+ onfail="Failed to kill switch?" )
+
+ main.step( "Check connectivity after switch failure" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived switch failure successfully",
+ onfail="Did not survive switch failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir,
+ "flowsAfterSwitchFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir,
+ "groupsAfterSwitchFailure" + main.jsonFile)
+
+ main.step( "Recovering " + switch )
+ main.log.info( "Starting" + switch )
+ main.Mininet1.switch( SW=switch, OPTION="start")
+ main.log.info( "Waiting %s seconds for switch up to be discovered" %(switchSleep))
+ time.sleep( switchSleep )
+ topology = utilities.retry( main.CLIs[0].checkStatus,
+ main.FALSE,
+ kwargs={'numoswitch':'4', 'numolink':'8'},
+ attempts=10,
+ sleep=switchSleep)
+ utilities.assert_equals( expect=main.TRUE, actual=topology,
+ onpass="Switch recovery successful",
+ onfail="Failed to recover switch?" )
+
+ main.step( "Check connectivity after switch recovery" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived switch recovery successfully",
+ onfail="Did not survive switch recovery" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0],
+ main.logdir,
+ "flowsAfterSwitchRecovery" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0],
+ main.logdir,
+ "groupsAfterSwitchRecovery" + main.jsonFile)
+
+ def CASE7( self, main ):
+ """
+ OnosInstance1 Down
+ Pingall
+ OnosInstance1 up
+ Pingall
+ """
+
+ description = "Killing single instance to test controlplane resilience of Segment Routing"
+ main.case( description )
+ main.step( "Killing ONOS instance" )
+ killResult = main.ONOSbench.onosDie( main.CLIs[0].ip_address )
+ utilities.assert_equals( expect=main.TRUE, actual=killResult,
+ onpass="ONOS instance Killed",
+ onfail="Error killing ONOS instance" )
+ time.sleep( 12 )
+ topology = utilities.retry( main.CLIs[1].checkStatus,
+ main.FALSE,
+ kwargs={'numoswitch':'4', 'numolink':'8', 'numoctrl':'2'},
+ attempts=10,
+ sleep=12)
+ utilities.assert_equals( expect=main.TRUE, actual=topology,
+ onpass="ONOS Instance down successful",
+ onfail="Failed to turn off ONOS Instance" )
+ main.step( "Check connectivity after ONOS instance failure" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived instance failure successfully",
+ onfail="Did not survive instance failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[1], main.logdir,
+ "flowsAfterSwitchFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[1], main.logdir,
+ "groupsAfterSwitchFailure" + main.jsonFile)
+ main.step( "Recovering ONOS instance" )
+ startResult = main.ONOSbench.onosStart( main.CLIs[0].ip_address )
+ isUp = main.ONOSbench.isup( main.ONOSip[ 0 ] )
+ utilities.assert_equals( expect=main.TRUE, actual=isUp,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+ main.step( "Checking if ONOS CLI is ready" )
+ main.CLIs[0].startCellCli()
+ cliResult = main.CLIs[0].startOnosCli( main.ONOSip[ 0 ],
+ commandlineTimeout=60, onosStartTimeout=100 )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cliResult,
+ onpass="ONOS CLI is ready",
+ onfail="ONOS CLI is not ready" )
+ for i in range( 10 ):
+ ready = True
+ output = main.CLIs[0].summary()
+ if not output:
+ ready = False
+ if ready:
+ break
+ time.sleep( 10 )
+ utilities.assert_equals( expect=True, actual=ready,
+ onpass="ONOS summary command succeded",
+ onfail="ONOS summary command failed" )
+ if not ready:
+ main.log.error( "ONOS startup failed!" )
+ main.cleanup()
+ main.exit()
+ main.step(" Check whether the flow count is bigger than 116" )
+ count = utilities.retry( main.CLIs[0].checkFlowCount,
+ main.FALSE,
+ kwargs={'min':116},
+ attempts=10,
+ sleep=10 )
+ utilities.assertEquals( expect=True,
+ actual=(count>0),
+ onpass="Flow count looks correct: "+str(count),
+ onfail="Flow count looks wrong: "+str(count) )
+
+ main.step( "Check connectivity after ONOS instance recovery" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived instance recovery successfully",
+ onfail="Did not survive recovery failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0], main.logdir,
+ "flowsAfterCtrlFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0], main.logdir,
+ "groupsAfterCtrlFailure" + main.jsonFile)
+
+ def CASE8( self, main ):
+ """
+ Cluster Down
+ Pingall
+ Cluster up
+ Pingall
+ """
+ description = "Killing all instances to test controlplane resilience of Segment Routing"
+ main.case( description )
+ main.step( "Shutting down ONOS Cluster" )
+ for i in range( main.numCtrls ):
+ killResult = main.ONOSbench.onosDie( main.CLIs[i].ip_address )
+ utilities.assert_equals( expect=main.TRUE, actual=killResult,
+ onpass="ONOS instance Killed",
+ onfail="Error killing ONOS instance" )
+ time.sleep(12)
+ main.step( "Check connectivity after ONOS instance failure" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived instance failure successfully",
+ onfail="Did not survive instance failure" )
+ main.step( "Recovering ONOS Cluster" )
+ for i in range( main.numCtrls ):
+ startResult = main.ONOSbench.onosStart( main.CLIs[i].ip_address )
+ isUp = main.ONOSbench.isup( main.ONOSip[ i ] )
+ utilities.assert_equals( expect=main.TRUE, actual=isUp,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+ main.step( "Checking if ONOS CLI is ready" )
+ for i in range( main.numCtrls ):
+ main.CLIs[i].startCellCli()
+ cliResult = main.CLIs[i].startOnosCli( main.ONOSip[ i ],
+ commandlineTimeout=60, onosStartTimeout=100 )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cliResult,
+ onpass="ONOS CLI is ready",
+ onfail="ONOS CLI is not ready" )
+ for i in range( 10 ):
+ ready = True
+ output = main.CLIs[0].summary()
+ if not output:
+ ready = False
+ if ready:
+ break
+ time.sleep( 10 )
+ utilities.assert_equals( expect=True, actual=ready,
+ onpass="ONOS summary command succeded",
+ onfail="ONOS summary command failed" )
+ if not ready:
+ main.log.error( "ONOS startup failed!" )
+ main.cleanup()
+ main.exit()
+
+ main.step(" Check whether the flow count is bigger than 116" )
+ count = utilities.retry( main.CLIs[0].checkFlowCount,
+ main.FALSE,
+ kwargs={'min':116},
+ attempts=10,
+ sleep=10 )
+ utilities.assertEquals( expect=True,
+ actual=(count>0),
+ onpass="Flow count looks correct: "+str(count),
+ onfail="Flow count looks wrong: "+str(count) )
+
+ main.step( "Check connectivity after CLUSTER recovery" )
+ pa = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pa,
+ onpass="Survived instance recovery successfully",
+ onfail="Did not survive recovery failure" )
+ main.ONOSbench.dumpFlows( main.ONOSip[0], main.logdir,
+ "flowsAfterCtrlFailure" + main.jsonFile)
+ main.ONOSbench.dumpGroups( main.ONOSip[0], main.logdir,
+ "groupsAfterCtrlFailure" + main.jsonFile)
+
+ def CASE10( self, main ):
+ '''
+ Report errors/warnings/exceptions
+ '''
+ main.case( "Logging test for " + main.jsonFile )
+ for i in range(main.numCtrls):
+ main.ONOSbench.onosStop( main.ONOSip[i] )
+ main.Mininet1.stopNet()
+
+ main.ONOSbench.scp( main.ONOScli1 ,
+ "/opt/onos/log/karaf.log",
+ "/tmp/karaf.log",
+ direction="from" )
+ main.ONOSbench.cpLogsToDir("/tmp/karaf.log",main.logdir,
+ copyFileName="karaf.log."+main.jsonFile+str(len(main.json)))
+ #main.ONOSbench.logReport( main.ONOSip[ 0 ],
+ # [ "INFO" ],
+ # "a" )
+ #main.log.info("Error report: \n" )
+ main.ONOSbench.logReport( main.ONOSip[ 0 ],
+ [ "INFO",
+ "FOLLOWER",
+ "WARN",
+ "flow",
+ "ERROR",
+ "Except" ],
+ "s" )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
new file mode 100755
index 0000000..546a023
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
@@ -0,0 +1,58 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>localhost</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ <nodes>1</nodes>
+ </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>OC1</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>OC2</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>OC3</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli3>
+
+
+ <Mininet1>
+ <host>OCN</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>MininetCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ <home>~/mininet/custom/</home>
+ </COMPONENTS>
+ </Mininet1>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRSanity/__init__.py
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/__init__.py