Merge "Move current to SR Sanity tests"
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
new file mode 100755
index 0000000..59c1214
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
@@ -0,0 +1,16 @@
+This test verifies basic connectivity using SegmentRouting via pingall,
+it should not fail.
+
+It consists of 
+
+1) Configure and Install ONOS
+2) Start Mininet and check flow state
+3) Pingall
+
+Requirements
+
+ - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+    1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+The topology is a 2x2 Leaf-spine
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.params b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.params
similarity index 66%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.params
rename to TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.params
index 15ce439..24403e2 100755
--- a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.params
@@ -1,6 +1,6 @@
 <PARAMS>
 
-    <testcases>1,[2,3,4,4,10]*2</testcases>
+    <testcases>1,2,3,4,10</testcases>
 
     <SCALE>
         <size>1</size>
@@ -8,7 +8,6 @@
     </SCALE>
 
     <DEPENDENCY>
-        <path>/tests/USECASE_SegmentRouting/dependencies/</path>
         <wrapper1>startUp</wrapper1>
         <topology>cord_fabric.py</topology>
     </DEPENDENCY>
@@ -16,7 +15,7 @@
     <ENV>
         <cellName>productionCell</cellName>
         <cellApps>drivers,segmentrouting</cellApps>
-        <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider;openflow</diffApps>
+        <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
         <cellUser>sdn</cellUser>
     </ENV>
 
@@ -29,6 +28,17 @@
         <port>6653</port>
     </CTRL>
 
+    <timers>
+        <LinkDiscovery>12</LinkDiscovery>
+        <SwitchDiscovery>12</SwitchDiscovery>
+    </timers>
+
+    <kill>
+        <switch> spine101 </switch>
+        <dpid> 000000000101 </dpid>
+        <links> leaf1 leaf2 </links>
+    </kill>
+
     <SLEEP>
         <startup>10</startup>
     </SLEEP>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py
new file mode 100755
index 0000000..9a0d91c
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py
@@ -0,0 +1,619 @@
+# This test should always succeed. it runs cases 1,2,3,4
+#CASE1: Get and Build ONOS
+#CASE2: Package and Install ONOS
+#CASE3: Start Mininet and check flows
+#CASE4: Ping all
+#CASE5: Link Failure
+#CASE6: Switch Failure
+#CASE7: ONOS Failure
+#CASE8: CLUSTER Failure
+#CASE10: Logging
+
+class SRSanity:
+
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        import time
+        import os
+        import imp
+        import re
+
+        """
+        - Construct tests variables
+        - GIT ( optional )
+            - Checkout ONOS master branch
+            - Pull latest ONOS code
+        - Building ONOS ( optional )
+            - Install ONOS package
+            - Build ONOS package
+        """
+
+        main.case( "Constructing test variables and building ONOS" )
+        main.step( "Constructing test variables" )
+        stepResult = main.FALSE
+
+        # Test variables
+        main.cellName = main.params[ 'ENV' ][ 'cellName' ]
+        main.apps = main.params[ 'ENV' ][ 'cellApps' ]
+        main.diff = []
+        main.diff.extend(( main.params[ 'ENV' ][ 'diffApps' ] ).split(";"))
+        main.diff=main.diff*2
+        gitBranch = main.params[ 'GIT' ][ 'branch' ]
+        main.path = os.path.dirname( main.testFile )
+        main.dependencyPath = main.path +"/../dependencies/"
+        main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
+        main.json = ["2x2"]*4
+        main.args = [" "]*4
+        main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
+        main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
+        main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
+        wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
+        main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
+        gitPull = main.params[ 'GIT' ][ 'pull' ]
+        main.cellData = {} # for creating cell file
+        main.CLIs = []
+        main.ONOSip = []
+
+        # Assigning ONOS cli handles to a list
+        for i in range( 1,  main.maxNodes + 1 ):
+            main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+            main.ONOSip.append( main.CLIs[i-1].ip_address )
+        # -- INIT SECTION, ONLY RUNS ONCE -- #
+        main.startUp = imp.load_source( wrapperFile1,
+                                        main.dependencyPath +
+                                        wrapperFile1 +
+                                        ".py" )
+
+        copyResult1 = main.ONOSbench.scp( main.Mininet1,
+                                          main.dependencyPath +
+                                          main.topology,
+                                          main.Mininet1.home,
+                                          direction="to" )
+        if main.CLIs:
+            stepResult = main.TRUE
+        else:
+            main.log.error( "Did not properly created list of ONOS CLI handle" )
+            stepResult = main.FALSE
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully construct " +
+                                        "test variables ",
+                                 onfail="Failed to construct test variables" )
+
+        if gitPull == 'True':
+            main.step( "Building ONOS in " + gitBranch + " branch" )
+            onosBuildResult = main.startUp.onosBuild( main, gitBranch )
+            stepResult = onosBuildResult
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=stepResult,
+                                     onpass="Successfully compiled " +
+                                            "latest ONOS",
+                                     onfail="Failed to compile " +
+                                            "latest ONOS" )
+        else:
+            main.log.warn( "Did not pull new code so skipping mvn " +
+                           "clean install" )
+        main.numCtrls = int( main.scale[ 0 ] )
+         
+    def CASE2( self, main ):
+        """
+        - Set up cell
+            - Create cell file
+            - Set cell file
+            - Verify cell file
+        - Kill ONOS process
+        - Uninstall ONOS cluster
+        - Verify ONOS start up
+        - Install ONOS cluster
+        - Connect to cli
+        """
+
+        # main.scale[ 0 ] determines the current number of ONOS controller
+        main.numCtrls = int( main.scale[ 0 ] )
+        main.scale.remove( main.scale[ 0 ] )
+        apps=main.apps
+        if main.diff:
+            apps = main.apps+","+main.diff.pop(0)
+        else: main.log.error( "App list is empty" )
+        main.case( "Package and start ONOS using apps:" + apps)
+
+        print "NODE COUNT = ", main.numCtrls
+        print main.ONOSip
+        tempOnosIp = []
+        for i in range( main.numCtrls ):
+            tempOnosIp.append( main.ONOSip[i] )
+
+        onosUser = main.params[ 'ENV' ][ 'cellUser' ]
+        main.step("Create and Apply cell file")
+        main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
+                                       "temp",
+                                       main.Mininet1.ip_address,
+                                       apps,
+                                       tempOnosIp,
+                                       onosUser )
+
+        cellResult = main.ONOSbench.setCell( "temp" )
+        verifyResult = main.ONOSbench.verifyCell()
+        stepResult = cellResult and verifyResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully applied cell to " + \
+                                        "environment",
+                                 onfail="Failed to apply cell to environment " )
+        #kill off all onos processes
+        main.log.info( "Safety check, killing all ONOS processes" +
+                       " before initiating environment setup" )
+
+        for i in range( main.maxNodes ):
+            main.ONOSbench.onosDie( main.ONOSip[ i ] )
+
+        main.step( "Create and Install ONOS package" )
+        main.jsonFile=main.json.pop(0)
+        main.ONOSbench.handle.sendline( "cp "+main.dependencyPath+"/"+main.jsonFile+".json ~/onos/tools/package/config/network-cfg.json")
+        packageResult = main.ONOSbench.onosPackage()
+
+        onosInstallResult = main.TRUE
+        for i in range( main.numCtrls ):
+            onosInstallResult = onosInstallResult and \
+                    main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+        stepResult = onosInstallResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully installed ONOS package",
+                                 onfail="Failed to install ONOS package" )
+
+        main.step( "Starting ONOS service" )
+        stopResult = main.TRUE
+        startResult = main.TRUE
+        onosIsUp = main.TRUE
+
+        for i in range( main.numCtrls ):
+            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
+        if onosIsUp == main.TRUE:
+            main.log.report( "ONOS instance is up and ready" )
+        else:
+            main.log.report( "ONOS instance may not be up, stop and " +
+                             "start ONOS again " )
+            for i in range( main.numCtrls ):
+                stopResult = stopResult and \
+                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
+            for i in range( main.numCtrls ):
+                startResult = startResult and \
+                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
+        stepResult = onosIsUp and stopResult and startResult
+
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+        main.step( "Checking if ONOS CLI is ready" )
+        main.CLIs[0].startCellCli()
+        cliResult = main.CLIs[0].startOnosCli( main.ONOSip[ 0 ],
+                                           commandlineTimeout=60, onosStartTimeout=100 )
+        utilities.assert_equals( expect=main.TRUE,
+                             actual=cliResult,
+                             onpass="ONOS CLI is ready",
+                             onfail="ONOS CLI is not ready" )
+        for i in range( 10 ):
+            ready = True
+            output = main.CLIs[0].summary()
+            if not output:
+                ready = False
+            if ready:
+                break
+            time.sleep( 10 )
+        utilities.assert_equals( expect=True, actual=ready,
+                                 onpass="ONOS summary command succeded",
+                                 onfail="ONOS summary command failed" )
+
+        if not ready:
+            main.log.error( "ONOS startup failed!" )
+            main.cleanup()
+            main.exit()
+
+    def CASE3( self, main ):
+        """
+            Start mininet
+        """
+        main.case( "Start Leaf-Spine "+main.jsonFile+" Mininet Topology" )
+        main.log.report( "Start Mininet topology" )
+
+        main.step( "Starting Mininet Topology" )
+        args,topo=" "," "
+        if main.args:
+            args = "--onos %d %s" % (main.numCtrls, main.args.pop(0))
+        else: main.log.error( "Argument list is empty" )
+ 
+        topoResult = main.Mininet1.startNet( topoFile= main.dependencyPath + main.topology, args=args )
+        stepResult = topoResult
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=stepResult,
+                                 onpass="Successfully loaded topology",
+                                 onfail="Failed to load topology" )
+        # Exit if topology did not load properly
+        if not topoResult:
+            main.cleanup()
+            main.exit()
+        main.step(" Check whether the flow count is bigger than 116" )
+        count =  utilities.retry( main.CLIs[0].checkFlowCount,
+                                 main.FALSE,
+                                 kwargs={'min':116},
+                                 attempts=10,
+                                 sleep=10 )
+        utilities.assertEquals( \
+            expect=True,
+            actual=(count>0),
+            onpass="Flow count looks correct: "+str(count),
+            onfail="Flow count looks wrong: "+str(count) )
+
+        main.step( "Check whether all flow status are ADDED" )
+        flowCheck = utilities.retry( main.CLIs[0].checkFlowsState,
+                                 main.FALSE,
+                                 kwargs={'isPENDING':False},
+                                 attempts=10,
+                                 sleep=10)
+        utilities.assertEquals( \
+            expect=main.TRUE,
+            actual=flowCheck,
+            onpass="Flow status is correct!",
+            onfail="Flow status is wrong!" )
+        main.ONOSbench.dumpFlows( main.ONOSip[0],
+                 main.logdir, "flowsBefore" + main.jsonFile)
+        main.ONOSbench.dumpGroups( main.ONOSip[0],
+                                   main.logdir, "groupsBefore" + main.jsonFile)
+        main.count=1
+
+    def CASE4( self, main ):
+        main.case( "Check full connectivity" )
+        main.log.report( "Check full connectivity" )
+
+        main.step("Check full connectivity"+str(main.count))
+        pa = main.Mininet1.pingall()
+        utilities.assert_equals( expect=main.TRUE, actual=pa,
+                                 onpass="Full connectivity successfully tested",
+                                 onfail="Full connectivity failed" )
+        main.ONOSbench.dumpFlows( main.ONOSip[0],
+                 main.logdir, "flowsAfter" + str(main.count) + main.jsonFile)
+        main.ONOSbench.dumpGroups( main.ONOSip[0],
+                           main.logdir, "groupsAfter" + str(main.count) + main.jsonFile)
+
+    def CASE5( self, main ):
+        """
+        Link spine101-leaf2 down
+        Pingall
+        Link spine101-leaf2 up
+        Pingall
+        """
+        import time
+        assert main.numCtrls, "main.numCtrls not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert main.CLIs, "main.CLIs not defined"
+
+        linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
+
+        description = "Turn off a link to ensure that Segment Routing reroutes traffic " + \
+                      "properly"
+        main.case( description )
+
+        main.step( "Kill Link between spine101 and leaf2" )
+        LinkDown = main.Mininet1.link( END1="spine101", END2="leaf2", OPTION="down" )
+        main.log.info( "Waiting %s seconds for link up to be discovered" % (linkSleep) )
+        # TODO Maybe parameterize number of expected links
+        time.sleep( linkSleep )
+        topology =  utilities.retry( main.CLIs[0].checkStatus,
+                                      main.FALSE,
+                                      kwargs={'numoswitch':'4', 'numolink':'6'},
+                                      attempts=10,
+                                      sleep=linkSleep)
+        result = topology & LinkDown
+        utilities.assert_equals( expect=main.TRUE, actual=result,
+                                 onpass="Link down successful",
+                                 onfail="Failed to turn off link?" )
+
+        main.step( "Check connectivity after link failure" )
+        pa = main.Mininet1.pingall()
+        utilities.assert_equals( expect=main.TRUE, actual=pa,
+                                 onpass="Survived link failure successfully",
+                                 onfail="Did not survive link failure" )
+        main.ONOSbench.dumpFlows( main.ONOSip[0],
+                                  main.logdir,
+                                  "flowsAfterLinkFailure" + main.jsonFile)
+        main.ONOSbench.dumpGroups( main.ONOSip[0],
+                                   main.logdir,
+                                   "groupsAfterLinkFailure" + main.jsonFile)
+
+        main.step( "Restore link between spine101 and leaf2" )
+
+        result = False
+        count=0
+        while True:
+            count+=0
+            main.Mininet1.link( END1="spine101", END2="leaf2", OPTION="up" )
+            main.Mininet1.link( END2="spine101", END1="leaf2", OPTION="up" )
+            main.log.info( "Waiting %s seconds for link up to be discovered" % (linkSleep) )
+            time.sleep( linkSleep )
+
+            main.CLIs[0].portstate(dpid='of:0000000000000002', port='1')
+            main.CLIs[0].portstate(dpid='of:0000000000000101', port='2')
+            time.sleep( linkSleep )
+
+            result = main.CLIs[0].checkStatus(numoswitch='4', numolink='8')
+            if count>10 or result:
+                break
+        utilities.assert_equals( expect=main.TRUE, actual=result,
+                                 onpass="Link up successful",
+                                 onfail="Failed to bring link up" )
+
+        main.step( "Check connectivity after link recovery" )
+        pa = main.Mininet1.pingall()
+        utilities.assert_equals( expect=main.TRUE, actual=pa,
+                                 onpass="Survived link recovery successfully",
+                                 onfail="Did not survive link recovery" )
+        main.ONOSbench.dumpFlows( main.ONOSip[0],
+                                  main.logdir,
+                                  "flowsAfterLinkRecovery" + main.jsonFile)
+        main.ONOSbench.dumpGroups( main.ONOSip[0],
+                                   main.logdir,
+                                   "groupsAfterLinkRecovery" + main.jsonFile)
+
+    def CASE6( self, main ):
+        """
+        Switch Down
+        Pingall
+        Switch up
+        Pingall
+        """
+        import time
+        assert main.numCtrls, "main.numCtrls not defined"
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        assert main.CLIs, "main.CLIs not defined"
+    
+        switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
+    
+        description = "Killing a switch to ensure it is discovered correctly"
+        onosCli = main.CLIs[0]
+        main.case( description )
+        switch = main.params[ 'kill' ][ 'switch' ]
+        switchDPID = main.params[ 'kill' ][ 'dpid' ]
+
+        main.step( "Kill " + switch )
+        main.log.info( "Stopping" + switch )
+        main.Mininet1.switch( SW=switch, OPTION="stop")
+        main.log.info( "Waiting %s seconds for switch down to be discovered" %(switchSleep))
+        time.sleep( switchSleep )
+        topology =  utilities.retry( main.CLIs[0].checkStatus,
+                                      main.FALSE,
+                                     kwargs={'numoswitch':'3', 'numolink':'4'},
+                                      attempts=10,
+                                     sleep=switchSleep)
+        utilities.assert_equals( expect=main.TRUE, actual=topology,
+                                 onpass="Kill switch successful",
+                                 onfail="Failed to kill switch?" )
+
+        main.step( "Check connectivity after switch failure" )
+        pa = main.Mininet1.pingall()
+        utilities.assert_equals( expect=main.TRUE, actual=pa,
+                                 onpass="Survived switch failure successfully",
+                                 onfail="Did not survive switch failure" )
+        main.ONOSbench.dumpFlows( main.ONOSip[0],
+                                  main.logdir,
+                                  "flowsAfterSwitchFailure" + main.jsonFile)
+        main.ONOSbench.dumpGroups( main.ONOSip[0],
+                                   main.logdir,
+                                   "groupsAfterSwitchFailure" + main.jsonFile)
+
+        main.step( "Recovering " + switch )
+        main.log.info( "Starting" + switch )
+        main.Mininet1.switch( SW=switch, OPTION="start")
+        main.log.info( "Waiting %s seconds for switch up to be discovered" %(switchSleep))
+        time.sleep( switchSleep )
+        topology =  utilities.retry( main.CLIs[0].checkStatus,
+                                     main.FALSE,
+                                     kwargs={'numoswitch':'4', 'numolink':'8'},
+                                     attempts=10,
+                                     sleep=switchSleep)
+        utilities.assert_equals( expect=main.TRUE, actual=topology,
+                                 onpass="Switch recovery successful",
+                                 onfail="Failed to recover switch?" )
+
+        main.step( "Check connectivity after switch recovery" )
+        pa = main.Mininet1.pingall()
+        utilities.assert_equals( expect=main.TRUE, actual=pa,
+                                 onpass="Survived switch recovery successfully",
+                                 onfail="Did not survive switch recovery" )
+        main.ONOSbench.dumpFlows( main.ONOSip[0],
+                                  main.logdir,
+                                  "flowsAfterSwitchRecovery" + main.jsonFile)
+        main.ONOSbench.dumpGroups( main.ONOSip[0],
+                                   main.logdir,
+                                   "groupsAfterSwitchRecovery" + main.jsonFile)
+
+    def CASE7( self, main ):
+        """
+        OnosInstance1 Down
+        Pingall
+        OnosInstance1 up
+        Pingall
+        """
+
+        description = "Killing single instance to test controlplane resilience of Segment Routing"
+        main.case( description )
+        main.step( "Killing ONOS instance" )
+        killResult = main.ONOSbench.onosDie( main.CLIs[0].ip_address )
+        utilities.assert_equals( expect=main.TRUE, actual=killResult,
+                                 onpass="ONOS instance Killed",
+                                 onfail="Error killing ONOS instance" )
+        time.sleep( 12 )
+        topology =  utilities.retry( main.CLIs[1].checkStatus,
+                                     main.FALSE,
+                                     kwargs={'numoswitch':'4', 'numolink':'8', 'numoctrl':'2'},
+                                     attempts=10,
+                                     sleep=12)
+        utilities.assert_equals( expect=main.TRUE, actual=topology,
+                                 onpass="ONOS Instance down successful",
+                                 onfail="Failed to turn off ONOS Instance" )
+        main.step( "Check connectivity after ONOS instance failure" )
+        pa = main.Mininet1.pingall()
+        utilities.assert_equals( expect=main.TRUE, actual=pa,
+                                 onpass="Survived instance failure successfully",
+                                 onfail="Did not survive instance failure" )
+        main.ONOSbench.dumpFlows( main.ONOSip[1], main.logdir,
+                                  "flowsAfterSwitchFailure" + main.jsonFile)
+        main.ONOSbench.dumpGroups( main.ONOSip[1], main.logdir,
+                                   "groupsAfterSwitchFailure" + main.jsonFile)
+        main.step( "Recovering ONOS instance" )
+        startResult = main.ONOSbench.onosStart( main.CLIs[0].ip_address )
+        isUp = main.ONOSbench.isup( main.ONOSip[ 0 ] )
+        utilities.assert_equals( expect=main.TRUE, actual=isUp,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+        main.step( "Checking if ONOS CLI is ready" )
+        main.CLIs[0].startCellCli()
+        cliResult = main.CLIs[0].startOnosCli( main.ONOSip[ 0 ],
+                                               commandlineTimeout=60, onosStartTimeout=100 )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=cliResult,
+                                 onpass="ONOS CLI is ready",
+                                 onfail="ONOS CLI is not ready" )
+        for i in range( 10 ):
+            ready = True
+            output = main.CLIs[0].summary()
+            if not output:
+                ready = False
+            if ready:
+                break
+            time.sleep( 10 )
+        utilities.assert_equals( expect=True, actual=ready,
+                                 onpass="ONOS summary command succeded",
+                                 onfail="ONOS summary command failed" )
+        if not ready:
+            main.log.error( "ONOS startup failed!" )
+            main.cleanup()
+            main.exit()
+        main.step(" Check whether the flow count is bigger than 116" )
+        count =  utilities.retry( main.CLIs[0].checkFlowCount,
+                                  main.FALSE,
+                                  kwargs={'min':116},
+                                  attempts=10,
+                                  sleep=10 )
+        utilities.assertEquals( expect=True,
+                                actual=(count>0),
+                                onpass="Flow count looks correct: "+str(count),
+                                onfail="Flow count looks wrong: "+str(count) )
+
+        main.step( "Check connectivity after ONOS instance recovery" )
+        pa = main.Mininet1.pingall()
+        utilities.assert_equals( expect=main.TRUE, actual=pa,
+                                 onpass="Survived instance recovery successfully",
+                                 onfail="Did not survive recovery failure" )
+        main.ONOSbench.dumpFlows( main.ONOSip[0], main.logdir,
+                                  "flowsAfterCtrlFailure" + main.jsonFile)
+        main.ONOSbench.dumpGroups( main.ONOSip[0], main.logdir,
+                                   "groupsAfterCtrlFailure" + main.jsonFile)
+
+    def CASE8( self, main ):
+        """
+        Cluster Down
+        Pingall
+        Cluster up
+        Pingall
+        """
+        description = "Killing all instances to test controlplane resilience of Segment Routing"
+        main.case( description )
+        main.step( "Shutting down ONOS Cluster" )
+        for i in range( main.numCtrls ):
+            killResult = main.ONOSbench.onosDie( main.CLIs[i].ip_address )
+            utilities.assert_equals( expect=main.TRUE, actual=killResult,
+                                 onpass="ONOS instance Killed",
+                                 onfail="Error killing ONOS instance" )
+        time.sleep(12)
+        main.step( "Check connectivity after ONOS instance failure" )
+        pa = main.Mininet1.pingall()
+        utilities.assert_equals( expect=main.TRUE, actual=pa,
+                                 onpass="Survived instance failure successfully",
+                                 onfail="Did not survive instance failure" )
+        main.step( "Recovering ONOS Cluster" )
+        for i in range( main.numCtrls ):
+            startResult = main.ONOSbench.onosStart( main.CLIs[i].ip_address )
+            isUp = main.ONOSbench.isup( main.ONOSip[ i ] )
+            utilities.assert_equals( expect=main.TRUE, actual=isUp,
+                                 onpass="ONOS service is ready",
+                                 onfail="ONOS service did not start properly" )
+        main.step( "Checking if ONOS CLI is ready" )
+        for i in range( main.numCtrls ):
+            main.CLIs[i].startCellCli()
+            cliResult = main.CLIs[i].startOnosCli( main.ONOSip[ i ],
+                                           commandlineTimeout=60, onosStartTimeout=100 )
+            utilities.assert_equals( expect=main.TRUE,
+                             actual=cliResult,
+                             onpass="ONOS CLI is ready",
+                             onfail="ONOS CLI is not ready" )
+        for i in range( 10 ):
+            ready = True
+            output = main.CLIs[0].summary()
+            if not output:
+                ready = False
+            if ready:
+                break
+            time.sleep( 10 )
+        utilities.assert_equals( expect=True, actual=ready,
+                                 onpass="ONOS summary command succeded",
+                                 onfail="ONOS summary command failed" )
+        if not ready:
+            main.log.error( "ONOS startup failed!" )
+            main.cleanup()
+            main.exit()
+
+        main.step(" Check whether the flow count is bigger than 116" )
+        count =  utilities.retry( main.CLIs[0].checkFlowCount,
+                                  main.FALSE,
+                                  kwargs={'min':116},
+                                  attempts=10,
+                                  sleep=10 )
+        utilities.assertEquals( expect=True,
+                                actual=(count>0),
+                                onpass="Flow count looks correct: "+str(count),
+                                onfail="Flow count looks wrong: "+str(count) )
+
+        main.step( "Check connectivity after CLUSTER recovery" )
+        pa = main.Mininet1.pingall()
+        utilities.assert_equals( expect=main.TRUE, actual=pa,
+                                 onpass="Survived instance recovery successfully",
+                                 onfail="Did not survive recovery failure" )
+        main.ONOSbench.dumpFlows( main.ONOSip[0], main.logdir,
+                                  "flowsAfterCtrlFailure" + main.jsonFile)
+        main.ONOSbench.dumpGroups( main.ONOSip[0], main.logdir,
+                                   "groupsAfterCtrlFailure" + main.jsonFile)
+
+    def CASE10( self, main ):
+        '''
+            Report errors/warnings/exceptions
+        '''
+        main.case( "Logging test for " + main.jsonFile )
+        for i in range(main.numCtrls):
+            main.ONOSbench.onosStop( main.ONOSip[i] )
+        main.Mininet1.stopNet()
+        
+        main.ONOSbench.scp( main.ONOScli1 ,
+                                          "/opt/onos/log/karaf.log",
+                                          "/tmp/karaf.log",
+                                          direction="from" )
+        main.ONOSbench.cpLogsToDir("/tmp/karaf.log",main.logdir,
+                                   copyFileName="karaf.log."+main.jsonFile+str(len(main.json)))
+        #main.ONOSbench.logReport( main.ONOSip[ 0 ],
+        #                          [ "INFO" ],
+        #                          "a" )
+        #main.log.info("Error report: \n" )
+        main.ONOSbench.logReport( main.ONOSip[ 0 ],
+                                  [ "INFO",
+                                    "FOLLOWER",
+                                    "WARN",
+                                    "flow",
+                                    "ERROR",
+                                    "Except" ],
+                                  "s" )
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.topo b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
similarity index 61%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.topo
rename to TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
index a5266a0..546a023 100755
--- a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
@@ -13,7 +13,7 @@
         </ONOSbench>
 
         <ONOScli1>
-            <host>localhost</host>
+            <host>OC1</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>OnosCliDriver</type>
@@ -21,13 +21,34 @@
             <COMPONENTS>
             </COMPONENTS>
         </ONOScli1>
+        
+        <ONOScli2>
+            <host>OC2</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosCliDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOScli2>
+
+        <ONOScli3>
+            <host>OC3</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOScli3>
+
 
         <Mininet1>
             <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
-            <connect_order>3</connect_order>
+            <connect_order>5</connect_order>
             <COMPONENTS>
                 <home>~/mininet/custom/</home>
             </COMPONENTS>
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRSanity/__init__.py
old mode 100644
new mode 100755
similarity index 100%
copy from TestON/tests/USECASE/USECASE_SegmentRouting/__init__.py
copy to TestON/tests/USECASE/SegmentRouting/SRSanity/__init__.py
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/__init__.py b/TestON/tests/USECASE/SegmentRouting/__init__.py
old mode 100644
new mode 100755
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/__init__.py
rename to TestON/tests/USECASE/SegmentRouting/__init__.py
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/2x2.json b/TestON/tests/USECASE/SegmentRouting/dependencies/2x2.json
old mode 100644
new mode 100755
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/2x2.json
rename to TestON/tests/USECASE/SegmentRouting/dependencies/2x2.json
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/4x4.json b/TestON/tests/USECASE/SegmentRouting/dependencies/4x4.json
old mode 100644
new mode 100755
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/4x4.json
rename to TestON/tests/USECASE/SegmentRouting/dependencies/4x4.json
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/dependencies/cord_fabric.py b/TestON/tests/USECASE/SegmentRouting/dependencies/cord_fabric.py
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/dependencies/cord_fabric.py
rename to TestON/tests/USECASE/SegmentRouting/dependencies/cord_fabric.py
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/dependencies/startUp.py b/TestON/tests/USECASE/SegmentRouting/dependencies/startUp.py
old mode 100644
new mode 100755
similarity index 100%
rename from TestON/tests/USECASE/USECASE_SegmentRouting/dependencies/startUp.py
rename to TestON/tests/USECASE/SegmentRouting/dependencies/startUp.py
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/README.md b/TestON/tests/USECASE/USECASE_SegmentRouting/README.md
deleted file mode 100644
index 8240068..0000000
--- a/TestON/tests/USECASE/USECASE_SegmentRouting/README.md
+++ /dev/null
@@ -1,33 +0,0 @@
-This test is designed to verify basic connectivity the SegmentRouting application via pingaall.
-
-It consists of 
-
-1) Configure and Install ONOS
-2) Start Mininet and check flow state
-2) Test connectivity
-
-Requirements
-
- - A single ONOS instance is required for the test, the application is currently not stable in a cluster as of today.
- - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
-
-Step 1:
-In this step, we copy config file to ONOS, next we package ONOS and install it in the target machine.
-
-Step 2:
-
-In this step we start the topology, connect to ONOS
-
-Step:3
-
-Here, we send several pings between hosts to test connectivity.
-
-Then Steps are repeated for different configurations and topologies.
-
-    Configurations:
-     1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
-     2) APPS=openflow
-
-    Topologies:
-     1) 2x2 Leaf-Spine
-     2) 4x4 Leaf-Spine
diff --git a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.py b/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.py
deleted file mode 100755
index 1a4834b..0000000
--- a/TestON/tests/USECASE/USECASE_SegmentRouting/USECASE_SegmentRouting.py
+++ /dev/null
@@ -1,320 +0,0 @@
-
-# This test is to determine if the Segment Routing application is working properly
-
-class USECASE_SegmentRouting:
-
-    def __init__( self ):
-        self.default = ''
-
-    def CASE1( self, main ):
-        import time
-        import os
-        import imp
-        import re
-
-        """
-        - Construct tests variables
-        - GIT ( optional )
-            - Checkout ONOS master branch
-            - Pull latest ONOS code
-        - Building ONOS ( optional )
-            - Install ONOS package
-            - Build ONOS package
-        """
-
-        main.case( "Constructing test variables and building ONOS" )
-        main.step( "Constructing test variables" )
-        stepResult = main.FALSE
-
-        # Test variables
-        main.cellName = main.params[ 'ENV' ][ 'cellName' ]
-        main.apps = main.params[ 'ENV' ][ 'cellApps' ]
-        main.diff = []
-        main.diff.extend(( main.params[ 'ENV' ][ 'diffApps' ] ).split(";"))
-        main.diff.extend(( main.params[ 'ENV' ][ 'diffApps' ] ).split(";"))
-        gitBranch = main.params[ 'GIT' ][ 'branch' ]
-        main.path = os.path.dirname( main.testFile )
-        main.dependencyPath = main.path + "/dependencies/"
-        main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
-        #main.json = ["0x1","0x1"]
-        main.json = ["2x2", "2x2","4x4","4x4"]
-        main.args = [" ", " ", " --spine 4 --leaf 4 ", " --spine 4 --leaf 4 "]
-        #main.args = [" --spine 0 --leaf 1 "," --spine 0 --leaf 1 "]
-        main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
-        main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
-        main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
-        wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
-        main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
-        gitPull = main.params[ 'GIT' ][ 'pull' ]
-        main.cellData = {} # for creating cell file
-        main.CLIs = []
-        main.ONOSip = []
-
-        main.ONOSip = main.ONOSbench.getOnosIps()
-
-        # Assigning ONOS cli handles to a list
-        for i in range( 1,  main.maxNodes + 1 ):
-            main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
-
-        # -- INIT SECTION, ONLY RUNS ONCE -- #
-        main.startUp = imp.load_source( wrapperFile1,
-                                        main.dependencyPath +
-                                        wrapperFile1 +
-                                        ".py" )
-
-        copyResult1 = main.ONOSbench.scp( main.Mininet1,
-                                          main.dependencyPath +
-                                          main.topology,
-                                          main.Mininet1.home,
-                                          direction="to" )
-        if main.CLIs:
-            stepResult = main.TRUE
-        else:
-            main.log.error( "Did not properly created list of ONOS CLI handle" )
-            stepResult = main.FALSE
-
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=stepResult,
-                                 onpass="Successfully construct " +
-                                        "test variables ",
-                                 onfail="Failed to construct test variables" )
-
-        if gitPull == 'True':
-            main.step( "Building ONOS in " + gitBranch + " branch" )
-            onosBuildResult = main.startUp.onosBuild( main, gitBranch )
-            stepResult = onosBuildResult
-            utilities.assert_equals( expect=main.TRUE,
-                                     actual=stepResult,
-                                     onpass="Successfully compiled " +
-                                            "latest ONOS",
-                                     onfail="Failed to compile " +
-                                            "latest ONOS" )
-        else:
-            main.log.warn( "Did not pull new code so skipping mvn " +
-                           "clean install" )
-         
-    def CASE2( self, main ):
-        """
-        - Set up cell
-            - Create cell file
-            - Set cell file
-            - Verify cell file
-        - Kill ONOS process
-        - Uninstall ONOS cluster
-        - Verify ONOS start up
-        - Install ONOS cluster
-        - Connect to cli
-        """
-
-        # main.scale[ 0 ] determines the current number of ONOS controller
-        main.numCtrls = int( main.scale[ 0 ] )
-        apps=main.apps
-        if main.diff:
-            apps = main.apps+","+main.diff.pop(0)
-        else: main.log.error( "App list is empty" )
-        main.case( "Package and start ONOS using apps:" + apps)
-
-        print "NODE COUNT = ", main.numCtrls
-
-        tempOnosIp = []
-        for i in range( main.numCtrls ):
-            tempOnosIp.append( main.ONOSip[i] )
-
-        onosUser = main.params[ 'ENV' ][ 'cellUser' ]
-        main.step("Create and Apply cell file")
-        main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
-                                       "temp",
-                                       main.Mininet1.ip_address,
-                                       apps,
-                                       tempOnosIp,
-                                       onosUser )
-
-        cellResult = main.ONOSbench.setCell( "temp" )
-        verifyResult = main.ONOSbench.verifyCell()
-        stepResult = cellResult and verifyResult
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=stepResult,
-                                 onpass="Successfully applied cell to " + \
-                                        "environment",
-                                 onfail="Failed to apply cell to environment " )
-        #kill off all onos processes
-        main.log.info( "Safety check, killing all ONOS processes" +
-                       " before initiating environment setup" )
-
-        for i in range( main.maxNodes ):
-            main.ONOSbench.onosDie( main.ONOSip[ i ] )
-
-        main.step( "Create and Install ONOS package" )
-        main.jsonFile=main.json.pop(0)
-        main.ONOSbench.handle.sendline( "cp "+main.path+"/"+main.jsonFile+".json ~/onos/tools/package/config/network-cfg.json")
-        packageResult = main.ONOSbench.onosPackage()
-        #stepResult = packageResult
-        #utilities.assert_equals( expect=main.TRUE,
-        #                         actual=stepResult,
-        #                         onpass="Successfully created ONOS package",
-        #                         onfail="Failed to create ONOS package" )
-
-        #time.sleep( main.startUpSleep )
-
-        onosInstallResult = main.TRUE
-        for i in range( main.numCtrls ):
-            onosInstallResult = onosInstallResult and \
-                    main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
-        stepResult = onosInstallResult
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=stepResult,
-                                 onpass="Successfully installed ONOS package",
-                                 onfail="Failed to install ONOS package" )
-
-        main.step( "Starting ONOS service" )
-        stopResult = main.TRUE
-        startResult = main.TRUE
-        onosIsUp = main.TRUE
-
-        for i in range( main.numCtrls ):
-            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
-        if onosIsUp == main.TRUE:
-            main.log.report( "ONOS instance is up and ready" )
-        else:
-            main.log.report( "ONOS instance may not be up, stop and " +
-                             "start ONOS again " )
-            for i in range( main.numCtrls ):
-                stopResult = stopResult and \
-                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
-            for i in range( main.numCtrls ):
-                startResult = startResult and \
-                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
-        stepResult = onosIsUp and stopResult and startResult
-
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=stepResult,
-                                 onpass="ONOS service is ready",
-                                 onfail="ONOS service did not start properly" )
-        #time.sleep( 2*main.startUpSleep )
-        #main.ONOSbench.handle.sendline( "onos-secure-ssh")
-        main.step( "Checking if ONOS CLI is ready" )
-        cellResult = main.CLIs[0].setCell( "temp" )
-
-        cliResult = main.CLIs[0].startOnosCli( main.ONOSip[ 0 ],
-                                           commandlineTimeout=60, onosStartTimeout=100 )
-        utilities.assert_equals( expect=main.TRUE,
-                             actual=cliResult,
-                             onpass="ONOS CLI is ready",
-                             onfail="ONOS CLI is not ready" )
-        for i in range( 10 ):
-            ready = True
-            output = main.CLIs[0].summary()
-            if not output:
-                ready = False
-            if ready:
-                break
-            time.sleep( 10 )
-        utilities.assert_equals( expect=True, actual=ready,
-                                 onpass="ONOS summary command succeded",
-                                 onfail="ONOS summary command failed" )
-
-        if not ready:
-            main.log.error( "ONOS startup failed!" )
-            main.cleanup()
-            main.exit()
-
-    def CASE3( self, main ):
-        """
-            Start mininet
-        """
-        main.case( "Start Leaf-Spine "+main.jsonFile+" Mininet Topology" )
-        main.log.report( "Start Mininet topology" )
-
-        main.step( "Starting Mininet Topology" )
-        args,topo=" "," "
-        #if main.topology:
-        #    topo = main.topology.pop(0)
-        #else: main.log.error( "Topo list is empty" )
-        if main.args:
-            args = "--onos 1 " + main.args.pop(0)
-        else: main.log.error( "Argument list is empty" )
-
-        topoResult = main.Mininet1.startNet( topoFile= main.dependencyPath + main.topology, args=args )
-        stepResult = topoResult
-        utilities.assert_equals( expect=main.TRUE,
-                                 actual=stepResult,
-                                 onpass="Successfully loaded topology",
-                                 onfail="Failed to load topology" )
-        # Exit if topology did not load properly
-        if not topoResult:
-            main.cleanup()
-            main.exit()
-        #main.step("Waiting for switch initialization and configuration")
-        main.step(" Check whether the flow count is bigger than 116" )
-        count =  utilities.retry( main.CLIs[0].checkFlowCount,
-                                 main.FALSE,
-                                 kwargs={'min':116},
-                                 attempts=10 )
-        utilities.assertEquals( \
-            expect=True,
-            actual=(count>0),
-            onpass="Flow count looks correct: "+str(count),
-            onfail="Flow count looks wrong: "+str(count) )
-
-        main.step( "Check whether all flow status are ADDED" )
-        flowCheck = utilities.retry( main.CLIs[0].checkFlowsState,
-                                 main.FALSE,
-                                 kwargs={'isPENDING':False},
-                                 attempts=10 )
-        utilities.assertEquals( \
-            expect=main.TRUE,
-            actual=flowCheck,
-            onpass="Flow status is correct!",
-            onfail="Flow status is wrong!" )
-        main.ONOSbench.dumpFlows( main.ONOSip[0],
-                 main.logdir, "flowsBefore" + main.jsonFile)
-        main.ONOSbench.dumpGroups( main.ONOSip[0],
-                                   main.logdir, "groupsBefore" + main.jsonFile)
-        #time.sleep( 3*main.startUpSleep)
-        main.count=1
-
-    def CASE4( self, main ):
-        main.case( "Check full connectivity" )
-        main.log.report( "Check full connectivity" )
-
-        main.step("Check full connectivity"+str(main.count))
-        pa = main.Mininet1.pingall()
-        utilities.assert_equals( expect=main.TRUE, actual=pa,
-                                 onpass="Full connectivity successfully tested",
-                                 onfail="Full connectivity failed" )
-        # cleanup mininet
-        main.ONOSbench.dumpFlows( main.ONOSip[0],
-                 main.logdir, "flowsAfter" + str(main.count) + main.jsonFile)
-        main.ONOSbench.dumpGroups( main.ONOSip[0],
-                           main.logdir, "groupsAfter" + str(main.count) + main.jsonFile)
-
-    def CASE10( self, main ):
-        '''
-            Report errors/warnings/exceptions
-        '''
-        main.case( "Logging test for " + main.jsonFile )
-        #if len(main.json) > 0 :
-        main.ONOSbench.onosStop( main.ONOSip[0] )
-        main.Mininet1.stopNet()
-        
-        main.ONOSbench.scp( main.ONOScli1 ,
-                                          "/opt/onos/log/karaf.log",
-                                          "/tmp/karaf.log",
-                                          direction="from" )
-        main.ONOSbench.cpLogsToDir("/tmp/karaf.log",main.logdir,
-                                   copyFileName="karaf.log."+main.jsonFile+str(len(main.json)))
-        #main.ONOSbench.logReport( main.ONOSip[ 0 ],
-        #                          [ "INFO" ],
-        #                          "a" )
-        #main.log.info("Error report: \n" )
-        main.ONOSbench.logReport( main.ONOSip[ 0 ],
-                                  [ "INFO",
-                                    "FOLLOWER",
-                                    "WARN",
-                                    "flow",
-                                    "ERROR",
-                                    "Except" ],
-                                  "s" )
-
-
-