diff --git a/TestON/tests/FUNC/FUNCintentRest/FUNCintentRest.topo b/TestON/tests/FUNC/FUNCintentRest/FUNCintentRest.topo
index 29c38a3..d50c434 100755
--- a/TestON/tests/FUNC/FUNCintentRest/FUNCintentRest.topo
+++ b/TestON/tests/FUNC/FUNCintentRest/FUNCintentRest.topo
@@ -72,7 +72,7 @@
         </ONOScli3>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/MISC/SCPFbatchFlowResp/SCPFbatchFlowResp.topo b/TestON/tests/MISC/SCPFbatchFlowResp/SCPFbatchFlowResp.topo
index b37a17a..05dbe31 100755
--- a/TestON/tests/MISC/SCPFbatchFlowResp/SCPFbatchFlowResp.topo
+++ b/TestON/tests/MISC/SCPFbatchFlowResp/SCPFbatchFlowResp.topo
@@ -22,7 +22,7 @@
         </ONOScli1>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/PLAT/PLATdockertest/PLATdockertest.topo b/TestON/tests/PLAT/PLATdockertest/PLATdockertest.topo
index 3501864..f6d50ae 100755
--- a/TestON/tests/PLAT/PLATdockertest/PLATdockertest.topo
+++ b/TestON/tests/PLAT/PLATdockertest/PLATdockertest.topo
@@ -32,7 +32,7 @@
         </ONOSbenchDocker>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/SAMP/SAMPstartTemplate_1node/SAMPstartTemplate_1node.topo b/TestON/tests/SAMP/SAMPstartTemplate_1node/SAMPstartTemplate_1node.topo
index a804d4a..4fee9ff 100755
--- a/TestON/tests/SAMP/SAMPstartTemplate_1node/SAMPstartTemplate_1node.topo
+++ b/TestON/tests/SAMP/SAMPstartTemplate_1node/SAMPstartTemplate_1node.topo
@@ -27,7 +27,7 @@
         </ONOScli1>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/SAMP/SAMPstartTemplate_3node/SAMPstartTemplate_3node.topo b/TestON/tests/SAMP/SAMPstartTemplate_3node/SAMPstartTemplate_3node.topo
index 9f23369..c01bb0a 100755
--- a/TestON/tests/SAMP/SAMPstartTemplate_3node/SAMPstartTemplate_3node.topo
+++ b/TestON/tests/SAMP/SAMPstartTemplate_3node/SAMPstartTemplate_3node.topo
@@ -47,7 +47,7 @@
         </ONOScli3>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/SCPF/SCPFcbench/SCPFcbench.params b/TestON/tests/SCPF/SCPFcbench/SCPFcbench.params
index d862129..fb23e90 100644
--- a/TestON/tests/SCPF/SCPFcbench/SCPFcbench.params
+++ b/TestON/tests/SCPF/SCPFcbench/SCPFcbench.params
@@ -28,7 +28,7 @@
     </CTRL>
 
     <MN>
-        <ip1>OC1</ip1>
+        <ip1>OCN</ip1>
     </MN>
 
     <BENCH>
diff --git a/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.topo b/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.topo
index 382d60f..615e957 100644
--- a/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.topo
+++ b/TestON/tests/SCPF/SCPFhostLat/SCPFhostLat.topo
@@ -87,7 +87,7 @@
         </ONOS1>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.params b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.params
index 7c3e10a..fe61b9a 100644
--- a/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.params
+++ b/TestON/tests/SCPF/SCPFintentInstallWithdrawLat/SCPFintentInstallWithdrawLat.params
@@ -68,7 +68,7 @@
     </CTRL>
 
     <MN>
-        <ip1>localhost</ip1>
+        <ip1>OCN</ip1>
     </MN>
 
     <BENCH>
diff --git a/TestON/tests/SCPF/SCPFintentInstallWithdrawLatWithFlowObj/SCPFintentInstallWithdrawLatWithFlowObj.params b/TestON/tests/SCPF/SCPFintentInstallWithdrawLatWithFlowObj/SCPFintentInstallWithdrawLatWithFlowObj.params
index e1b799a..6eeed39 100644
--- a/TestON/tests/SCPF/SCPFintentInstallWithdrawLatWithFlowObj/SCPFintentInstallWithdrawLatWithFlowObj.params
+++ b/TestON/tests/SCPF/SCPFintentInstallWithdrawLatWithFlowObj/SCPFintentInstallWithdrawLatWithFlowObj.params
@@ -68,7 +68,7 @@
     </CTRL>
 
     <MN>
-        <ip1>localhost</ip1>
+        <ip1>OCN</ip1>
     </MN>
 
     <BENCH>
diff --git a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params
index 246a177..6d684cc 100644
--- a/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params
+++ b/TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.params
@@ -69,7 +69,7 @@
     </CTRL>
 
     <MN>
-        <ip1>localhost</ip1>
+        <ip1>OCN</ip1>
     </MN>
 
     <BENCH>
diff --git a/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.params b/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.params
index 32735bc..237dc5d 100644
--- a/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.params
+++ b/TestON/tests/SCPF/SCPFintentRerouteLatWithFlowObj/SCPFintentRerouteLatWithFlowObj.params
@@ -69,7 +69,7 @@
     </CTRL>
 
     <MN>
-        <ip1>localhost</ip1>
+        <ip1>OCN</ip1>
     </MN>
 
     <BENCH>
diff --git a/TestON/tests/SCPF/SCPFmaxIntents/SCPFmaxIntents.topo b/TestON/tests/SCPF/SCPFmaxIntents/SCPFmaxIntents.topo
index 987f444..cdd16f1 100755
--- a/TestON/tests/SCPF/SCPFmaxIntents/SCPFmaxIntents.topo
+++ b/TestON/tests/SCPF/SCPFmaxIntents/SCPFmaxIntents.topo
@@ -33,7 +33,7 @@
         </ONOSrest1>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.params b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.params
index 4a758b3..0d8687e 100644
--- a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.params
+++ b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.params
@@ -26,7 +26,7 @@
     </CTRL>
 
     <MN>
-        <ip1>localhost</ip1>
+        <ip1>OCN</ip1>
     </MN>
 
     <BENCH>
diff --git a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.topo b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.topo
index 1a73a2b..c988f13 100644
--- a/TestON/tests/SCPF/SCPFportLat/SCPFportLat.topo
+++ b/TestON/tests/SCPF/SCPFportLat/SCPFportLat.topo
@@ -78,7 +78,7 @@
         </ONOS7cli>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
index 90ec8bf..79e8620 100755
--- a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
+++ b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
@@ -8,7 +8,7 @@
     # 200 - bring onos node up and balance masters
     # 1000 - report logs
 
-    <testcases>1,2,[10,300,11,100,300,11,200,300,11,1000]*3</testcases>
+    <testcases>1,[2,10,300,11,100,300,11,200,300,11,1000]*3</testcases>
 
     <DEPENDENCY>
         <path>/tests/SCPF/SCPFscaleTopo/dependencies/</path>
diff --git a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py
index 96ac405..71c7d3a 100644
--- a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py
+++ b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py
@@ -118,6 +118,13 @@
         - Install ONOS cluster
         - Connect to cli
         """
+        main.log.info( "Checking if mininet is already running" )
+        if len( main.topoScale ) < main.topoScaleSize:
+            main.log.info( "Mininet is already running. Stopping mininet." )
+            main.Mininet1.stopNet()
+            time.sleep(main.MNSleep)
+        else:
+            main.log.info( "Mininet was not running" )
 
         main.case( "Starting up " + str( main.numCtrls ) +
                    " node(s) ONOS cluster" )
@@ -211,6 +218,7 @@
 
         main.step( "Start ONOS cli" )
         cliResult = main.TRUE
+        main.activeNodes = []
         for i in range( main.numCtrls ):
             cliResult = cliResult and \
                         main.CLIs[ i ].startOnosCli( main.ONOSip[ i ] )
@@ -220,7 +228,7 @@
                                  actual=stepResult,
                                  onpass="Successfully start ONOS cli",
                                  onfail="Failed to start ONOS cli" )
-
+        time.sleep( main.startUpSleep )
 
     def CASE10( self, main ):
         """
@@ -231,27 +239,9 @@
         main.case( "Starting up Mininet and verifying topology" )
         main.caseExplanation = "Starting Mininet with a scalling topology and " +\
                 "comparing topology elements between Mininet and ONOS"
-
-        main.log.info( "Checking if mininet is already running" )
-        if len( main.topoScale ) < main.topoScaleSize:
-            main.log.info( "Mininet is already running. Stopping mininet." )
-            main.Mininet1.stopNet()
-            time.sleep(main.MNSleep)
-        else:
-            main.log.info( "Mininet was not running" )
-
         if main.topoScale:
             main.currScale = main.topoScale.pop(0)
         else: main.log.error( "topology scale is empty" )
-
-        # remove device before setup topology
-        devices = main.topo.getAllDevices( main )
-        if( devices[0] != '[]' ): # because devices is a list witch contain 3 string, not contain list!
-            temp = json.loads( devices[0] )
-            devicesIdList = []
-            for d in temp:
-                main.CLIs[0].deviceRemove( d.get('id').encode() )
-
         main.step( "Starting up TORUS %sx%s topology" % (main.currScale, main.currScale) )
 
         main.log.info( "Constructing Mininet command" )
diff --git a/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.topo b/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.topo
index a1ec149..8e58be7 100755
--- a/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.topo
+++ b/TestON/tests/SCPF/SCPFscalingMaxIntents/SCPFscalingMaxIntents.topo
@@ -88,7 +88,7 @@
         </ONOScli7>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/SCPF/SCPFscalingMaxIntentsWithFlowObj/SCPFscalingMaxIntentsWithFlowObj.topo b/TestON/tests/SCPF/SCPFscalingMaxIntentsWithFlowObj/SCPFscalingMaxIntentsWithFlowObj.topo
index a1ec149..8e58be7 100755
--- a/TestON/tests/SCPF/SCPFscalingMaxIntentsWithFlowObj/SCPFscalingMaxIntentsWithFlowObj.topo
+++ b/TestON/tests/SCPF/SCPFscalingMaxIntentsWithFlowObj/SCPFscalingMaxIntentsWithFlowObj.topo
@@ -88,7 +88,7 @@
         </ONOScli7>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
index 6035fba..da371c3 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
+++ b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
@@ -26,8 +26,7 @@
     </CTRL>
 
     <MN>
-        <ip1>localhost</ip1>
-        <ip2>localhost</ip2>
+        <ip1>OCN</ip1>
     </MN>
 
     <BENCH>
diff --git a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.topo b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.topo
index 8b05fd0..4addd4a 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.topo
+++ b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.topo
@@ -87,7 +87,7 @@
         </ONOS1>
 
         <Mininet1>
-            <host>localhost</host>
+            <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/README.md b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/README.md
new file mode 100755
index 0000000..21e7773
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/README.md
@@ -0,0 +1,22 @@
+This test verifies control plane resilience from a CLUSTER restart
+
+It consists of 
+
+1) Configure and Install ONOS
+2) Start Mininet and check flow state
+3) Pingall
+4) Induce a CLUSTER restart
+5) check flow state
+6) Pingall
+
+Requirements
+
+ - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+    1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+
+The test runs for different topologies:
+ - 2x2 Leaf-Spine and 3-node ONOS cluster
+ - 4x4 Leaf-Spine and 3-node ONOS cluster
+ - Single switch and 3-node ONOS cluster
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.params b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.params
similarity index 100%
copy from TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.params
copy to TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.params
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.py b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.py
new file mode 100755
index 0000000..ae70e00
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.py
@@ -0,0 +1,106 @@
+# CASE1: 2x2 topo + 3-node ONOS CLUSTER + IP connectivity test + CLUSTER restart
+# CASE2: 4x4 topo + 3-node ONOS CLUSTER + IP connectivity test + CLUSTER restart
+# CASE3: Single switch + 3-node ONOS CLUSTER + IP connectivity test + CLUSTER restart
+
+class SRClusterRestart:
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start 2x2 Leaf-Spine topology
+        Pingall
+        Induce CLUSTER restart
+        Pingall
+        """
+        description = "Cluster Restart test with 2x2 Leaf-spine "
+        main.case( description )
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        main.cfgName = '2x2'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py' )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main, "CASE1" )
+        run.killOnos( main, [ 0, 1, 2 ], '4', '8', '0' )
+        run.pingAll( main, 'CASE1_Failure', dumpflows=False )
+        run.recoverOnos( main, [ 0, 1, 2 ], '4', '8', '3' )
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main, 'CASE1_Failure' )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE2( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start 4x4 Leaf-Spine topology
+        Pingall
+        Induce Cluster Restart
+        Pingall
+        """
+        description = "Cluster Restart test with 4x4 Leaf-spine "
+        main.case( description )
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        main.cfgName = '4x4'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main, 'CASE2' )
+        run.killOnos( main, [ 0, 1, 2 ], '8', '32', '0' )
+        run.pingAll( main, 'CASE2_Failure', dumpflows=False )
+        run.recoverOnos( main, [ 0, 1, 2 ], '8', '32', '3' )
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main, 'CASE3_Recovery' )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE3( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start single switch topology
+        Pingall
+        Induce Cluster Restart
+        Pingall
+        """
+        description = "Cluster Restart test with single switch "
+        main.case( description )
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        main.cfgName = '0x1'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=1 --spine=0" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=15 )
+        run.pingAll( main, 'CASE3' )
+        run.killOnos( main, [ 0, 1, 2 ], '1', '0', '0' )
+        run.pingAll( main, 'CASE3_Failure', dumpflows=False )
+        run.recoverOnos( main, [ 0, 1, 2 ], '1', '0', '3' )
+        run.checkFlows( main, minFlowCount=15 )
+        run.pingAll( main, 'CASE3_Failure' )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
similarity index 61%
rename from TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo
rename to TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
index 546a023..c531f7d 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
@@ -42,13 +42,45 @@
             </COMPONENTS>
         </ONOScli3>
 
+        <ONOSrest1>
+            <host>OC1</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest1>
+
+        <ONOSrest2>
+            <host>OC2</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest2>
+
+        <ONOSrest3>
+            <host>OC3</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest3>
 
         <Mininet1>
             <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
-            <connect_order>5</connect_order>
+            <connect_order>7</connect_order>
             <COMPONENTS>
                 <home>~/mininet/custom/</home>
             </COMPONENTS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/__init__.py
similarity index 100%
copy from TestON/tests/USECASE/SegmentRouting/SRLinkDown/__init__.py
copy to TestON/tests/USECASE/SegmentRouting/SRClusterRestart/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.py b/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.py
deleted file mode 100755
index db5469d..0000000
--- a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.py
+++ /dev/null
@@ -1,107 +0,0 @@
-# This test should always succeed. it runs cases 1,2,3,4
-#CASE1: Get and Build ONOS
-#CASE2: Package and Install ONOS
-#CASE3: Start Mininet and check flows
-#CASE4: Ping all
-#CASE5: Link Failure
-#CASE6: Switch Failure
-#CASE7: ONOS Failure
-#CASE8: CLUSTER Failure
-#CASE10: Logging
-
-class SRLinkDown:
-
-    def __init__( self ):
-        self.default = ''
-
-    def CASE1( self, main ):
-        """
-        Sets up 1-node Onos-cluster
-        Start 2x2 Leaf-Spine topology
-        Pingall
-        """
-        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as run
-        if not hasattr(main,'apps'):
-            run.initTest(main)
-
-        description = "Bridging and Routing sanity test with 2x2 Leaf-spine "
-        main.case( description )
-
-        main.cfgName = '2x2'
-        main.numCtrls = 1
-        run.installOnos(main)
-        run.startMininet(main, 'cord_fabric.py')
-        #pre-configured routing and bridging test
-        run.checkFlows(main, flowCount=116)
-        run.pingAll(main)
-        run.killLink(main, 'spine101', 'leaf2', switches='4', links='6')
-        run.pingAll(main, "CASE1_AfterLinkDown")
-        run.restoreLink(main, 'spine101', 'leaf2','of:0000000000000101',
-                        'of:0000000000000002', '2', '1', '4', '8')
-        run.pingAll(main, "CASE1_AfterLinkUp")
-        #TODO Dynamic config of hosts in subnet
-        #TODO Dynamic config of host not in subnet
-        #TODO Dynamic config of vlan xconnect
-        #TODO Vrouter integration
-        #TODO Mcast integration
-        run.cleanup(main)
-
-    def CASE2( self, main ):
-        """
-        Sets up 1-node Onos-cluster
-        Start 4x4 Leaf-Spine topology
-        Pingall
-        """
-        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as run
-        if not hasattr(main,'apps'):
-            run.initTest(main)
-        description = "Bridging and Routing sanity test with 4x4 Leaf-spine "
-        main.case( description )
-        main.cfgName = '4x4'
-        main.numCtrls = 1
-        run.installOnos(main)
-        run.startMininet(main, 'cord_fabric.py', args="--leaf=4 --spine=4")
-        #pre-configured routing and bridging test
-        run.checkFlows(main, flowCount=350)
-        run.pingAll(main)
-        run.killLink(main, 'spine101', 'leaf2', switches='8', links='30')
-        run.pingAll(main, "CASE2_AfterLinkDown")
-        run.restoreLink(main, 'spine101', 'leaf2','of:0000000000000101',
-                    'of:0000000000000002', '2', '1', '8', '32')
-        run.pingAll(main, "CASE2_AfterLinkUp")
-        #TODO Dynamic config of hosts in subnet
-        #TODO Dynamic config of host not in subnet
-        #TODO preconfigured xconnect
-        #TODO Vrouter integration
-        #TODO Mcast integration
-        run.cleanup(main)
-
-    def CASE3( self, main ):
-        """
-        Sets up 3-node Onos-cluster
-        Start 2x2 Leaf-Spine topology
-        Pingall
-        """
-        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as run
-        if not hasattr(main,'apps'):
-            run.initTest(main)
-        description = "Bridging and Routing sanity test with 2x2 Leaf-spine "
-        main.case( description )
-        main.cfgName = '2x2'
-        main.numCtrls = 3
-        run.installOnos(main)
-        run.startMininet(main, 'cord_fabric.py')
-        #pre-configured routing and bridging test
-        run.checkFlows(main, flowCount=116)
-        run.pingAll(main)
-        run.killLink(main, 'spine101', 'leaf2', switches='4', links='6')
-        run.pingAll(main, "CASE3_AfterLinkDown")
-        run.restoreLink(main, 'spine101', 'leaf2','of:0000000000000101',
-                        'of:0000000000000002', '2', '1', '4', '8')
-        run.pingAll(main, "CASE3_AfterLinkUp")
-        #TODO Dynamic config of hosts in subnet
-        #TODO Dynamic config of host not in subnet
-        #TODO Dynamic config of vlan xconnect
-        #TODO Vrouter integration
-        #TODO Mcast integration
-        run.cleanup(main)
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/README.md b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/README.md
similarity index 100%
rename from TestON/tests/USECASE/SegmentRouting/SRLinkDown/README.md
rename to TestON/tests/USECASE/SegmentRouting/SRLinkFailure/README.md
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.params b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.params
new file mode 100755
index 0000000..8f3ade9
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.params
@@ -0,0 +1,46 @@
+<PARAMS>
+
+    <testcases>1,2,4,5</testcases>
+
+    <SCALE>
+        <size>3</size>
+        <max>3</max>
+    </SCALE>
+
+    <DEPENDENCY>
+        <wrapper1>startUp</wrapper1>
+        <topology>cord_fabric.py</topology>
+    </DEPENDENCY>
+
+    <ENV>
+        <cellName>productionCell</cellName>
+        <cellApps>drivers,segmentrouting</cellApps>
+        <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
+        <cellUser>sdn</cellUser>
+    </ENV>
+
+    <GIT>
+        <pull>False</pull>
+        <branch>master</branch>
+    </GIT>
+
+    <CTRL>
+        <port>6653</port>
+    </CTRL>
+
+    <timers>
+        <LinkDiscovery>12</LinkDiscovery>
+        <SwitchDiscovery>12</SwitchDiscovery>
+    </timers>
+
+    <kill>
+        <switch> spine101 </switch>
+        <dpid> 000000000101 </dpid>
+        <links> leaf1 leaf2 </links>
+    </kill>
+
+    <SLEEP>
+        <startup>10</startup>
+    </SLEEP>
+
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.py b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.py
new file mode 100755
index 0000000..b388a64
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.py
@@ -0,0 +1,147 @@
+# In this test we perform a link failure and then test for connectivity
+# CASE1: 2x2 topo + link failure + IP connectivity test
+# CASE2: 4x4 topo + link failure + IP connectivity test
+# CASE4: 2x2 topo + 3-node ONOS CLUSTER + link failure + IP connectivity test
+# CASE5: 4x4 topo + 3-node ONOS CLUSTER + link failure + IP connectivity test
+
+class SRLinkFailure:
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        """
+        Sets up 1-node Onos-cluster
+        Start 2x2 Leaf-Spine topology
+        Pingall
+        Cause link failure
+        Pingall
+        """
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+
+        description = "Bridging and Routing sanity test with 2x2 Leaf-spine "
+        main.case( description )
+
+        main.cfgName = '2x2'
+        main.numCtrls = 1
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py' )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main )
+        # link failure
+        run.killLink( main, 'spine101', 'leaf2', switches='4', links='6' )
+        run.pingAll( main, "CASE1_Failure" )
+        run.restoreLink( main, 'spine101', 'leaf2', 'of:0000000000000101',
+                         'of:0000000000000002', '2', '1', '4', '8' )
+        run.pingAll( main, "CASE1_Recovery" )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE2( self, main ):
+        """
+        Sets up 1-node Onos-cluster
+        Start 4x4 Leaf-Spine topology
+        Pingall
+        Cause link failure
+        Pingall
+        """
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        description = "Bridging and Routing sanity test with 4x4 Leaf-spine "
+        main.case( description )
+        main.cfgName = '4x4'
+        main.numCtrls = 1
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main )
+        # link failure
+        run.killLink( main, 'spine101', 'leaf2', switches='8', links='30' )
+        run.pingAll( main, "CASE2_Failure" )
+        run.restoreLink( main, 'spine101', 'leaf2', 'of:0000000000000101',
+                         'of:0000000000000002', '2', '1', '8', '32' )
+        run.pingAll( main, "CASE2_Recovery" )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO preconfigured xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE4( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start 2x2 Leaf-Spine topology
+        Pingall
+        Cause link failure
+        Pingall
+        """
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        description = "Bridging and Routing sanity test with 2x2 Leaf-spine "
+        main.case( description )
+        main.cfgName = '2x2'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py' )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main )
+        # link failure
+        run.killLink( main, 'spine101', 'leaf2', switches='4', links='6' )
+        run.pingAll( main, "CASE3_Failure" )
+        run.restoreLink( main, 'spine101', 'leaf2', 'of:0000000000000101',
+                         'of:0000000000000002', '2', '1', '4', '8' )
+        run.pingAll( main, "CASE3_Recovery" )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE5( self, main ):
+        """
+        Sets up 1-node Onos-cluster
+        Start 4x4 Leaf-Spine topology
+        Pingall
+        Cause link failure
+        Pingall
+        """
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        description = "Bridging and Routing sanity test with 4x4 Leaf-spine "
+        main.case( description )
+        main.cfgName = '4x4'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main )
+        # link failure
+        run.killLink( main, 'spine101', 'leaf2', switches='8', links='30' )
+        run.pingAll( main, "CASE2_Failure" )
+        run.restoreLink( main, 'spine101', 'leaf2', 'of:0000000000000101',
+                         'of:0000000000000002', '2', '1', '8', '32' )
+        run.pingAll( main, "CASE2_Recovery" )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO preconfigured xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo
similarity index 61%
copy from TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo
copy to TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo
index 546a023..2d69c35 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo
@@ -21,7 +21,7 @@
             <COMPONENTS>
             </COMPONENTS>
         </ONOScli1>
-        
+
         <ONOScli2>
             <host>OC2</host>
             <user>sdn</user>
@@ -42,13 +42,45 @@
             </COMPONENTS>
         </ONOScli3>
 
+        <ONOSrest1>
+            <host>OC1</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest1>
+
+        <ONOSrest2>
+            <host>OC2</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest2>
+
+        <ONOSrest3>
+            <host>OC3</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest3>
 
         <Mininet1>
             <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
-            <connect_order>5</connect_order>
+            <connect_order>8</connect_order>
             <COMPONENTS>
                 <home>~/mininet/custom/</home>
             </COMPONENTS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/__init__.py
similarity index 100%
rename from TestON/tests/USECASE/SegmentRouting/SRLinkDown/__init__.py
rename to TestON/tests/USECASE/SegmentRouting/SRLinkFailure/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/README.md b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/README.md
new file mode 100755
index 0000000..94677cb
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/README.md
@@ -0,0 +1,22 @@
+This test verifies basic control plane resilience from an ONOS Instance failure using SegmentRouting via pingall
+
+It consists of 
+
+1) Configure and Install ONOS
+2) Start Mininet and check flow state
+3) Pingall
+4) Induce a ONOS failure
+5) check flow state
+6) Pingall
+
+Requirements
+
+ - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+    1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+
+The test runs for different topologies:
+ - 2x2 Leaf-Spine and 3-node ONOS cluster
+ - 4x4 Leaf-Spine and 3-node ONOS cluster
+ - Single switch and 3-node ONOS cluster
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.params b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params
similarity index 100%
rename from TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.params
rename to TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.py b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.py
new file mode 100755
index 0000000..53daa33
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.py
@@ -0,0 +1,106 @@
+# CASE1: 2x2 topo + 3-node ONOS CLUSTER + IP connectivity test + Control plane resilience
+# CASE2: 4x4 topo + 3-node ONOS CLUSTER + IP connectivity test + Control plane resilience
+# CASE3: Single switch + 3-node ONOS CLUSTER + IP connectivity test + Control plane resilience
+
+class SROnosFailure:
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start 2x2 Leaf-Spine topology
+        Pingall
+        Induce ONOS Failure
+        Pingall
+        """
+        description = "ONOS Failure test with 2x2 Leaf-spine "
+        main.case( description )
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        main.cfgName = '2x2'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py' )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main, "CASE1" )
+        run.killOnos( main, [ 0 ], '4', '8', '2' )
+        run.pingAll( main, 'CASE1_Failure' )
+        run.recoverOnos( main, [ 0 ], '4', '8', '3' )
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main, 'CASE1_Failure' )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE2( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start 4x4 Leaf-Spine topology
+        Pingall
+        Induce ONOS Failure
+        Pingall
+        """
+        description = "ONOS Failure test with 4x4 Leaf-spine "
+        main.case( description )
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        main.cfgName = '4x4'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main, 'CASE2' )
+        run.killOnos( main, [ 0 ], '8', '32', '2' )
+        run.pingAll( main, 'CASE2_Failure' )
+        run.recoverOnos( main, [ 0 ], '8', '32', '3' )
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main, 'CASE3_Recovery' )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE3( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start single switch topology
+        Pingall
+        Induce ONOS Failure
+        Pingall
+        """
+        description = "ONOS Failure test with single switch "
+        main.case( description )
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        main.cfgName = '0x1'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=1 --spine=0" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=15 )
+        run.pingAll( main, 'CASE3' )
+        run.killOnos( main, [ 0 ], '1', '0', '2' )
+        run.pingAll( main, 'CASE3_Failure' )
+        run.recoverOnos( main, [ 0 ], '1', '0', '3' )
+        run.checkFlows( main, minFlowCount=15 )
+        run.pingAll( main, 'CASE3_Failure' )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
similarity index 61%
copy from TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo
copy to TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
index 546a023..c531f7d 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
@@ -42,13 +42,45 @@
             </COMPONENTS>
         </ONOScli3>
 
+        <ONOSrest1>
+            <host>OC1</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest1>
+
+        <ONOSrest2>
+            <host>OC2</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest2>
+
+        <ONOSrest3>
+            <host>OC3</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest3>
 
         <Mininet1>
             <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
-            <connect_order>5</connect_order>
+            <connect_order>7</connect_order>
             <COMPONENTS>
                 <home>~/mininet/custom/</home>
             </COMPONENTS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/__init__.py b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/__init__.py
similarity index 100%
copy from TestON/tests/USECASE/SegmentRouting/SRLinkDown/__init__.py
copy to TestON/tests/USECASE/SegmentRouting/SROnosFailure/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
index 59c1214..e20287e 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
@@ -1,7 +1,6 @@
-This test verifies basic connectivity using SegmentRouting via pingall,
-it should not fail.
+This test verifies basic IP connectivity using SegmentRouting via pingall
 
-It consists of 
+It consists of
 
 1) Configure and Install ONOS
 2) Start Mininet and check flow state
@@ -13,4 +12,10 @@
 
 The test is executed using the netcfg subsystem:
     1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
-The topology is a 2x2 Leaf-spine
+The test runs for different topologies:
+ - 2x2 Leaf-Spine and 1-node ONOS cluster
+ - 4x4 Leaf-Spine and 1-node ONOS cluster
+ - Single switch and 1-node ONOS cluster
+ - 2x2 Leaf-Spine and 3-node ONOS cluster
+ - 4x4 Leaf-Spine and 3-node ONOS cluster
+ - Single switch and 3-node ONOS cluster
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py
index c69645f..e62dc78 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.py
@@ -1,4 +1,3 @@
-# This test should always succeed. it runs cases 1,2,3
 # CASE1: 2x2 Leaf-Spine topo and test IP connectivity
 # CASE2: 4x4 topo + IP connectivity test
 # CASE3: Single switch topo + IP connectivity test
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/README.md b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/README.md
similarity index 100%
copy from TestON/tests/USECASE/SegmentRouting/SRLinkDown/README.md
copy to TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/README.md
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.params b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.params
new file mode 100755
index 0000000..8f3ade9
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.params
@@ -0,0 +1,46 @@
+<PARAMS>
+
+    <testcases>1,2,4,5</testcases>
+
+    <SCALE>
+        <size>3</size>
+        <max>3</max>
+    </SCALE>
+
+    <DEPENDENCY>
+        <wrapper1>startUp</wrapper1>
+        <topology>cord_fabric.py</topology>
+    </DEPENDENCY>
+
+    <ENV>
+        <cellName>productionCell</cellName>
+        <cellApps>drivers,segmentrouting</cellApps>
+        <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
+        <cellUser>sdn</cellUser>
+    </ENV>
+
+    <GIT>
+        <pull>False</pull>
+        <branch>master</branch>
+    </GIT>
+
+    <CTRL>
+        <port>6653</port>
+    </CTRL>
+
+    <timers>
+        <LinkDiscovery>12</LinkDiscovery>
+        <SwitchDiscovery>12</SwitchDiscovery>
+    </timers>
+
+    <kill>
+        <switch> spine101 </switch>
+        <dpid> 000000000101 </dpid>
+        <links> leaf1 leaf2 </links>
+    </kill>
+
+    <SLEEP>
+        <startup>10</startup>
+    </SLEEP>
+
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.py b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.py
new file mode 100755
index 0000000..b516cc0
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.py
@@ -0,0 +1,151 @@
+# In this test we perform a switch failure and then test for connectivity
+# CASE1: 2x2 topo + swtich failure + IP connectivity test
+# CASE2: 4x4 topo + switch failure + IP connectivity test
+# CASE4: 2x2 topo + 3-node ONOS CLUSTER + switch failure + IP connectivity test
+# CASE5: 4x4 topo + 3-node ONOS CLUSTER + switch failure + IP connectivity test
+
+class SRSwitchFailure:
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        """
+        Sets up 1-node Onos-cluster
+        Start 2x2 Leaf-Spine topology
+        Pingall
+        Cause switch failure
+        Pingall
+        """
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+
+        description = "Bridging and Routing sanity test with 2x2 Leaf-spine "
+        main.case( description )
+
+        main.cfgName = '2x2'
+        main.numCtrls = 1
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py' )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main )
+        # switch failure\
+        switch = main.params[ 'kill' ][ 'switch' ]
+        run.killSwitch( main, switch, switches='3', links='4' )
+        run.pingAll( main, "CASE1_Failure" )
+        run.recoverSwitch( main, switch, switches='4', links='8' )
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main, "CASE1_Recovery" )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE2( self, main ):
+        """
+        Sets up 1-node Onos-cluster
+        Start 4x4 Leaf-Spine topology
+        Pingall
+        Cause switch failure
+        Pingall
+        """
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        description = "Bridging and Routing sanity test with 4x4 Leaf-spine "
+        main.case( description )
+        main.cfgName = '4x4'
+        main.numCtrls = 1
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main )
+        # switch failure
+        switch = main.params[ 'kill' ][ 'switch' ]
+        run.killSwitch( main, switch, switches='7', links='24' )
+        run.pingAll( main, "CASE2_Failure" )
+        run.recoverSwitch( main, switch, switches='8', links='32' )
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main, "CASE2_Recovery" )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO preconfigured xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE4( self, main ):
+        """
+        Sets up 3-node Onos-cluster
+        Start 2x2 Leaf-Spine topology
+        Pingall
+        Cause link failure
+        Pingall
+        """
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        description = "Bridging and Routing sanity test with 2x2 Leaf-spine "
+        main.case( description )
+        main.cfgName = '2x2'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py' )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main )
+        # switch failure
+        switch = main.params[ 'kill' ][ 'switch' ]
+        run.killSwitch( main, switch, switches='3', links='4' )
+        run.pingAll( main, "CASE3_Failure" )
+        run.recoverSwitch( main, switch, switches='4', links='8' )
+        run.checkFlows( main, minFlowCount=116 )
+        run.pingAll( main, "CASE3_Recovery" )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO Dynamic config of vlan xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
+
+    def CASE5( self, main ):
+        """
+        Sets up 1-node Onos-cluster
+        Start 4x4 Leaf-Spine topology
+        Pingall
+        Cause link failure
+        Pingall
+        """
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+            Testcaselib as run
+        if not hasattr( main, 'apps' ):
+            run.initTest( main )
+        description = "Bridging and Routing sanity test with 4x4 Leaf-spine "
+        main.case( description )
+        main.cfgName = '4x4'
+        main.numCtrls = 3
+        run.installOnos( main )
+        run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+        # pre-configured routing and bridging test
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main )
+        # switch failure
+        switch = main.params[ 'kill' ][ 'switch' ]
+        run.killSwitch( main, switch, switches='7', links='24' )
+        run.pingAll( main, "CASE4_Failure" )
+        run.recoverSwitch( main, switch, switches='8', links='32' )
+        run.checkFlows( main, minFlowCount=350 )
+        run.pingAll( main, "CASE4_Recovery" )
+        # TODO Dynamic config of hosts in subnet
+        # TODO Dynamic config of host not in subnet
+        # TODO preconfigured xconnect
+        # TODO Vrouter integration
+        # TODO Mcast integration
+        run.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo
similarity index 61%
copy from TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo
copy to TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo
index 546a023..2d69c35 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/SRLinkDown.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo
@@ -21,7 +21,7 @@
             <COMPONENTS>
             </COMPONENTS>
         </ONOScli1>
-        
+
         <ONOScli2>
             <host>OC2</host>
             <user>sdn</user>
@@ -42,13 +42,45 @@
             </COMPONENTS>
         </ONOScli3>
 
+        <ONOSrest1>
+            <host>OC1</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>5</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest1>
+
+        <ONOSrest2>
+            <host>OC2</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>6</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest2>
+
+        <ONOSrest3>
+            <host>OC3</host>
+            <port>8181</port>
+            <user>onos</user>
+            <password>rocks</password>
+            <type>OnosRestDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS>
+            </COMPONENTS>
+        </ONOSrest3>
 
         <Mininet1>
             <host>OCN</host>
             <user>sdn</user>
             <password>rocks</password>
             <type>MininetCliDriver</type>
-            <connect_order>5</connect_order>
+            <connect_order>8</connect_order>
             <COMPONENTS>
                 <home>~/mininet/custom/</home>
             </COMPONENTS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkDown/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/__init__.py
similarity index 100%
copy from TestON/tests/USECASE/SegmentRouting/SRLinkDown/__init__.py
copy to TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 1e31cd3..8565930 100755
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -25,23 +25,23 @@
         main.diff = main.params[ 'ENV' ][ 'diffApps' ]
         gitBranch = main.params[ 'GIT' ][ 'branch' ]
         main.path = os.path.dirname( main.testFile )
-        main.dependencyPath = main.path +"/../dependencies/"
+        main.dependencyPath = main.path + "/../dependencies/"
         main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
         wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
-        main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
+        main.scale = (main.params[ 'SCALE' ][ 'size' ]).split( "," )
         main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
-        #main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
+        # main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
         main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
-        main.cellData = {} # for creating cell file
-        main.CLIs = []
-        main.ONOSip = []
-        main.RESTs= []
+        main.cellData = { }  # for creating cell file
+        main.CLIs = [ ]
+        main.ONOSip = [ ]
+        main.RESTs = [ ]
 
         # Assigning ONOS cli handles to a list
-        for i in range( 1,  main.maxNodes + 1 ):
+        for i in range( 1, main.maxNodes + 1 ):
             main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
             main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) )
-            main.ONOSip.append( main.CLIs[i-1].ip_address )
+            main.ONOSip.append( main.CLIs[ i - 1 ].ip_address )
         # -- INIT SECTION, ONLY RUNS ONCE -- #
         main.startUp = imp.load_source( wrapperFile1,
                                         main.dependencyPath +
@@ -79,18 +79,19 @@
         - Connect to cli
         """
         # main.scale[ 0 ] determines the current number of ONOS controller
-        apps=main.apps
+        apps = main.apps
         if main.diff:
             apps = main.apps + "," + main.diff
-        else: main.log.error( "App list is empty" )
-        main.case( "Package and start ONOS using apps:" + apps)
+        else:
+            main.log.error( "App list is empty" )
+        main.case( "Package and start ONOS using apps:" + apps )
         print "NODE COUNT = ", main.numCtrls
         print main.ONOSip
-        tempOnosIp = []
+        tempOnosIp = [ ]
         for i in range( main.numCtrls ):
-            tempOnosIp.append( main.ONOSip[i] )
+            tempOnosIp.append( main.ONOSip[ i ] )
         onosUser = main.params[ 'ENV' ][ 'cellUser' ]
-        main.step("Create and Apply cell file")
+        main.step( "Create and Apply cell file" )
         main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
                                        "temp",
                                        main.Mininet1.ip_address,
@@ -98,32 +99,33 @@
                                        tempOnosIp,
                                        onosUser )
         cellResult = main.ONOSbench.setCell( "temp" )
-        verifyResult = main.ONOSbench.verifyCell()
+        verifyResult = main.ONOSbench.verifyCell( )
         stepResult = cellResult and verifyResult
         utilities.assert_equals( expect=main.TRUE,
                                  actual=stepResult,
                                  onpass="Successfully applied cell to " + \
                                         "environment",
                                  onfail="Failed to apply cell to environment " )
-        #kill off all onos processes
+        # kill off all onos processes
         main.log.info( "Safety check, killing all ONOS processes" +
                        " before initiating environment setup" )
         for i in range( main.maxNodes ):
             main.ONOSbench.onosDie( main.ONOSip[ i ] )
         main.step( "Create and Install ONOS package" )
-        packageResult = main.ONOSbench.onosPackage()
+        packageResult = main.ONOSbench.onosPackage( )
 
         onosInstallResult = main.TRUE
         for i in range( main.numCtrls ):
             onosInstallResult = onosInstallResult and \
-                                main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
+                                main.ONOSbench.onosInstall(
+                                        node=main.ONOSip[ i ] )
         stepResult = onosInstallResult
         utilities.assert_equals( expect=main.TRUE,
                                  actual=stepResult,
                                  onpass="Successfully installed ONOS package",
                                  onfail="Failed to install ONOS package" )
         main.step( "Starting ONOS service" )
-        stopResult,startResult, onosIsUp= main.TRUE, main.TRUE, main.TRUE,
+        stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE,
         for i in range( main.numCtrls ):
             onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
         if onosIsUp == main.TRUE:
@@ -145,17 +147,18 @@
                                  onfail="ONOS service did not start properly" )
         main.step( "Checking if ONOS CLI is ready" )
         for i in range( main.numCtrls ):
-            main.CLIs[i].startCellCli()
-            cliResult = main.CLIs[i].startOnosCli( main.ONOSip[ i ],
-                                               commandlineTimeout=60, onosStartTimeout=100 )
+            main.CLIs[ i ].startCellCli( )
+            cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
+                                                     commandlineTimeout=60,
+                                                     onosStartTimeout=100 )
         utilities.assert_equals( expect=main.TRUE,
                                  actual=cliResult,
                                  onpass="ONOS CLI is ready",
                                  onfail="ONOS CLI is not ready" )
-        main.active=0
+        main.active = 0
         for i in range( 10 ):
             ready = True
-            output = main.CLIs[main.active].summary()
+            output = main.CLIs[ main.active ].summary( )
             if not output:
                 ready = False
             if ready:
@@ -166,25 +169,27 @@
                                  onfail="ONOS summary command failed" )
 
         with open( main.dependencyPath + "/" + main.cfgName + ".json" ) as cfg:
-            main.RESTs[main.active].setNetCfg( json.load(cfg) )
+            main.RESTs[ main.active ].setNetCfg( json.load( cfg ) )
 
         if not ready:
             main.log.error( "ONOS startup failed!" )
-            main.cleanup()
-            main.exit()
+            main.cleanup( )
+            main.exit( )
 
         for i in range( main.numCtrls ):
-            main.CLIs[i].logSet( "DEBUG", "org.onosproject.segmentrouting" )
-            main.CLIs[i].logSet( "DEBUG", "org.onosproject.driver.pipeline" )
-            main.CLIs[i].logSet( "DEBUG", "org.onosproject.store.group.impl" )
-            main.CLIs[i].logSet( "DEBUG", "org.onosproject.net.flowobjective.impl" )
+            main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.segmentrouting" )
+            main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.driver.pipeline" )
+            main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.store.group.impl" )
+            main.CLIs[ i ].logSet( "DEBUG",
+                                   "org.onosproject.net.flowobjective.impl" )
 
     @staticmethod
     def startMininet( main, topology, args="" ):
         main.step( "Starting Mininet Topology" )
         arg = "--onos %d %s" % (main.numCtrls, args)
-        main.topology=topology
-        topoResult = main.Mininet1.startNet( topoFile= main.dependencyPath + main.topology, args=arg )
+        main.topology = topology
+        topoResult = main.Mininet1.startNet(
+                topoFile=main.dependencyPath + main.topology, args=arg )
         stepResult = topoResult
         utilities.assert_equals( expect=main.TRUE,
                                  actual=stepResult,
@@ -192,51 +197,53 @@
                                  onfail="Failed to load topology" )
         # Exit if topology did not load properly
         if not topoResult:
-            main.cleanup()
-            main.exit()
+            main.cleanup( )
+            main.exit( )
 
     @staticmethod
-    def checkFlows(main, minFlowCount):
-        main.step(" Check whether the flow count is bigger than %s" % minFlowCount)
-        count =  utilities.retry(main.CLIs[main.active].checkFlowCount,
+    def checkFlows( main, minFlowCount ):
+        main.step(
+                " Check whether the flow count is bigger than %s" % minFlowCount )
+        count = utilities.retry( main.CLIs[ main.active ].checkFlowCount,
                                  main.FALSE,
-                                 kwargs={'min':minFlowCount},
+                                 kwargs={ 'min': minFlowCount },
                                  attempts=10,
-                                 sleep=10)
+                                 sleep=10 )
         utilities.assertEquals( \
                 expect=True,
-                actual=(count>0),
-                onpass="Flow count looks correct: "+str(count),
-                onfail="Flow count looks wrong: "+str(count) )
+                actual=(count > 0),
+                onpass="Flow count looks correct: " + str( count ),
+                onfail="Flow count looks wrong: " + str( count ) )
 
         main.step( "Check whether all flow status are ADDED" )
-        flowCheck = utilities.retry( main.CLIs[main.active].checkFlowsState,
+        flowCheck = utilities.retry( main.CLIs[ main.active ].checkFlowsState,
                                      main.FALSE,
-                                     kwargs={'isPENDING':False},
+                                     kwargs={ 'isPENDING': False },
                                      attempts=10,
-                                     sleep=10)
+                                     sleep=10 )
         utilities.assertEquals( \
                 expect=main.TRUE,
                 actual=flowCheck,
                 onpass="Flow status is correct!",
                 onfail="Flow status is wrong!" )
-        main.ONOSbench.dumpFlows( main.ONOSip[main.active],
-                                  main.logdir, "flowsBefore" + main.cfgName)
-        main.ONOSbench.dumpGroups( main.ONOSip[0],
-                                   main.logdir, "groupsBefore" + main.cfgName)
+        main.ONOSbench.dumpFlows( main.ONOSip[ main.active ],
+                                  main.logdir, "flowsBefore" + main.cfgName )
+        main.ONOSbench.dumpGroups( main.ONOSip[ 0 ],
+                                   main.logdir, "groupsBefore" + main.cfgName )
 
     @staticmethod
-    def pingAll( main, tag=""):
+    def pingAll( main, tag="", dumpflows=True ):
         main.log.report( "Check full connectivity" )
-        main.step("Check full connectivity %s" %tag)
-        pa = main.Mininet1.pingall()
+        main.step( "Check full connectivity %s" % tag )
+        pa = main.Mininet1.pingall( )
         utilities.assert_equals( expect=main.TRUE, actual=pa,
                                  onpass="Full connectivity successfully tested",
                                  onfail="Full connectivity failed" )
-        main.ONOSbench.dumpFlows( main.ONOSip[main.active],
-                                  main.logdir, "flowsOn" + tag)
-        main.ONOSbench.dumpGroups( main.ONOSip[main.active],
-                                   main.logdir, "groupsOn" + tag)
+        if dumpflows:
+            main.ONOSbench.dumpFlows( main.ONOSip[ main.active ],
+                                      main.logdir, "flowsOn" + tag )
+            main.ONOSbench.dumpGroups( main.ONOSip[ main.active ],
+                                       main.logdir, "groupsOn" + tag )
 
     @staticmethod
     def killLink( main, end1, end2, switches, links ):
@@ -246,22 +253,25 @@
         Kill a link and verify ONOS can see the proper link change
         """
         main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
-        main.step( "Kill link between %s and %s" %(end1, end2))
+        main.step( "Kill link between %s and %s" % (end1, end2) )
         LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
-        main.log.info( "Waiting %s seconds for link down to be discovered" % main.linkSleep )
+        main.log.info(
+                "Waiting %s seconds for link down to be discovered" % main.linkSleep )
         time.sleep( main.linkSleep )
-        topology =  utilities.retry( main.CLIs[ main.active ].checkStatus,
-                                     main.FALSE,
-                                     kwargs={'numoswitch':switches, 'numolink':links},
-                                     attempts=10,
-                                     sleep=main.linkSleep)
+        topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
+                                    main.FALSE,
+                                    kwargs={ 'numoswitch': switches,
+                                             'numolink': links },
+                                    attempts=10,
+                                    sleep=main.linkSleep )
         result = topology & LinkDown
         utilities.assert_equals( expect=main.TRUE, actual=result,
                                  onpass="Link down successful",
                                  onfail="Failed to turn off link?" )
 
     @staticmethod
-    def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches, links ):
+    def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
+                     links ):
         """
         Params:
             end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
@@ -270,21 +280,23 @@
             switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
         Kill a link and verify ONOS can see the proper link change
         """
-        main.step( "Restore link between %s and %s" %( end1, end2 ) )
+        main.step( "Restore link between %s and %s" % (end1, end2) )
         result = False
-        count=0
+        count = 0
         while True:
-            count+=0
+            count += 0
             main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
             main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
-            main.log.info( "Waiting %s seconds for link up to be discovered" % main.linkSleep )
+            main.log.info(
+                    "Waiting %s seconds for link up to be discovered" % main.linkSleep )
             time.sleep( main.linkSleep )
-            main.CLIs[main.active].portstate( dpid=dpid1, port=port1 )
-            main.CLIs[main.active].portstate( dpid=dpid2, port=port2 )
+            main.CLIs[ main.active ].portstate( dpid=dpid1, port=port1 )
+            main.CLIs[ main.active ].portstate( dpid=dpid2, port=port2 )
             time.sleep( main.linkSleep )
 
-            result = main.CLIs[main.active].checkStatus( numoswitch=switches, numolink=links )
-            if count>5 or result:
+            result = main.CLIs[ main.active ].checkStatus( numoswitch=switches,
+                                                           numolink=links )
+            if count > 5 or result:
                 break
         utilities.assert_equals( expect=main.TRUE, actual=result,
                                  onpass="Link up successful",
@@ -300,13 +312,16 @@
         main.step( "Kill " + switch )
         main.log.info( "Stopping" + switch )
         main.Mininet1.switch( SW=switch, OPTION="stop" )
-        main.log.info( "Waiting %s seconds for switch down to be discovered" % ( main.switchSleep ) )
+        # todo make this repeatable
+        main.log.info( "Waiting %s seconds for switch down to be discovered" % (
+            main.switchSleep) )
         time.sleep( main.switchSleep )
-        topology =  utilities.retry( main.CLIs[main.active].checkStatus,
-                                     main.FALSE,
-                                     kwargs={'numoswitch':switches, 'numolink':links},
-                                     attempts=10,
-                                     sleep=main.switchSleep)
+        topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
+                                    main.FALSE,
+                                    kwargs={ 'numoswitch': switches,
+                                             'numolink': links },
+                                    attempts=10,
+                                    sleep=main.switchSleep )
         utilities.assert_equals( expect=main.TRUE, actual=topology,
                                  onpass="Kill switch successful",
                                  onfail="Failed to kill switch?" )
@@ -317,16 +332,19 @@
         Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
         Recover a switch and verify ONOS can see the proper change
         """
+        # todo make this repeatable
         main.step( "Recovering " + switch )
         main.log.info( "Starting" + switch )
-        main.Mininet1.switch( SW=switch, OPTION="start")
-        main.log.info( "Waiting %s seconds for switch up to be discovered" %(main.switchSleep))
+        main.Mininet1.switch( SW=switch, OPTION="start" )
+        main.log.info( "Waiting %s seconds for switch up to be discovered" % (
+            main.switchSleep) )
         time.sleep( main.switchSleep )
-        topology =  utilities.retry( main.CLIs[main.active].checkStatus,
-                                     main.FALSE,
-                                     kwargs={'numoswitch':switches, 'numolink':links},
-                                     attempts=10,
-                                     sleep=main.switchSleep)
+        topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
+                                    main.FALSE,
+                                    kwargs={ 'numoswitch': switches,
+                                             'numolink': links },
+                                    attempts=10,
+                                    sleep=main.switchSleep )
         utilities.assert_equals( expect=main.TRUE, actual=topology,
                                  onpass="Switch recovery successful",
                                  onfail="Failed to recover switch?" )
@@ -338,10 +356,93 @@
         Stops Mininet
         Copies ONOS log
         """
-        main.Mininet1.stopNet()
+        main.Mininet1.stopNet( )
         main.ONOSbench.scp( main.ONOScli1, "/opt/onos/log/karaf.log",
-                           "/tmp/karaf.log", direction="from" )
+                            "/tmp/karaf.log", direction="from" )
         main.ONOSbench.cpLogsToDir( "/tmp/karaf.log", main.logdir,
-                                   copyFileName="karaf.log."+main.cfgName )
+                                    copyFileName="karaf.log." + main.cfgName )
         for i in range( main.numCtrls ):
-            main.ONOSbench.onosStop( main.ONOSip[i] )
+            main.ONOSbench.onosStop( main.ONOSip[ i ] )
+
+    @staticmethod
+    def killOnos( main, nodes, switches, links, expNodes ):
+        """
+        Params: nodes, integer array with position of the ONOS nodes in the CLIs array
+        switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
+        Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
+        """
+        main.step( "Killing ONOS instance" )
+        for i in nodes:
+            killResult = main.ONOSbench.onosDie( main.CLIs[ i ].ip_address )
+            utilities.assert_equals( expect=main.TRUE, actual=killResult,
+                                     onpass="ONOS instance Killed",
+                                     onfail="Error killing ONOS instance" )
+            if i == main.active:
+                main.active = (i + 1) % main.numCtrls
+        time.sleep( 12 )
+        if len( nodes ) < main.numCtrls:
+            topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
+                                        main.FALSE,
+                                        kwargs={ 'numoswitch': switches,
+                                                 'numolink': links,
+                                                 'numoctrl': expNodes },
+                                        attempts=10,
+                                        sleep=12 )
+            utilities.assert_equals( expect=main.TRUE, actual=topology,
+                                     onpass="ONOS Instance down successful",
+                                     onfail="Failed to turn off ONOS Instance" )
+        else:
+            main.active = -1
+
+    @staticmethod
+    def recoverOnos( main, nodes, switches, links, expNodes ):
+        """
+        Params: nodes, integer array with position of the ONOS nodes in the CLIs array
+        switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
+        Recover an ONOS instance and verify the ONOS cluster can see the proper change
+        """
+        main.step( "Recovering ONOS instance" )
+        [ main.ONOSbench.onosStart( main.CLIs[ i ].ip_address ) for i in nodes ]
+        for i in nodes:
+            isUp = main.ONOSbench.isup( main.ONOSip[ i ] )
+            utilities.assert_equals( expect=main.TRUE, actual=isUp,
+                                     onpass="ONOS service is ready",
+                                     onfail="ONOS service did not start properly" )
+        for i in nodes:
+            main.step( "Checking if ONOS CLI is ready" )
+            main.CLIs[ i ].startCellCli( )
+            cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
+                                                     commandlineTimeout=60,
+                                                     onosStartTimeout=100 )
+            utilities.assert_equals( expect=main.TRUE,
+                                     actual=cliResult,
+                                     onpass="ONOS CLI is ready",
+                                     onfail="ONOS CLI is not ready" )
+            main.active = i if main.active == -1 else main.active
+
+        topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
+                                    main.FALSE,
+                                    kwargs={ 'numoswitch': switches,
+                                             'numolink': links,
+                                             'numoctrl': expNodes },
+                                    attempts=10,
+                                    sleep=12 )
+        utilities.assert_equals( expect=main.TRUE, actual=topology,
+                                 onpass="ONOS Instance down successful",
+                                 onfail="Failed to turn off ONOS Instance" )
+
+        for i in range( 10 ):
+            ready = True
+            output = main.CLIs[ main.active ].summary( )
+            if not output:
+                ready = False
+            if ready:
+                break
+            time.sleep( 10 )
+        utilities.assert_equals( expect=True, actual=ready,
+                                 onpass="ONOS summary command succeded",
+                                 onfail="ONOS summary command failed" )
+        if not ready:
+            main.log.error( "ONOS startup failed!" )
+            main.cleanup( )
+            main.exit( )
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/cord_fabric.py b/TestON/tests/USECASE/SegmentRouting/dependencies/cord_fabric.py
index a2a8a8e..c890177 100755
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/cord_fabric.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/cord_fabric.py
@@ -10,84 +10,93 @@
 from mininet.log import setLogLevel
 from mininet.cli import CLI
 
-# Parse command line options and dump results
-def parseOptions():
-    """Parse command line options"""
-    parser = OptionParser()
-    parser.add_option('--spine', dest='spine', type='int', default=2,
-                      help='number of spine switches, default=2')
-    parser.add_option('--leaf', dest='leaf', type='int', default=2,
-                      help='number of leaf switches, default=2')
-    parser.add_option('--fanout', dest='fanout', type='int', default=2,
-                      help='number of hosts per leaf switch, default=2')
-    parser.add_option('--onos', dest='onos', type='int', default=0,
-                      help='number of ONOS Instances, default=0, 0 means localhost, 1 will use OC1 and so on')
 
-    (options, args) = parser.parse_args()
+# Parse command line options and dump results
+def parseOptions( ):
+    """Parse command line options"""
+    parser = OptionParser( )
+    parser.add_option( '--spine', dest='spine', type='int', default=2,
+                       help='number of spine switches, default=2' )
+    parser.add_option( '--leaf', dest='leaf', type='int', default=2,
+                       help='number of leaf switches, default=2' )
+    parser.add_option( '--fanout', dest='fanout', type='int', default=2,
+                       help='number of hosts per leaf switch, default=2' )
+    parser.add_option( '--onos', dest='onos', type='int', default=0,
+                       help='number of ONOS Instances, default=0, 0 means localhost, 1 will use OC1 and so on' )
+
+    (options, args) = parser.parse_args( )
     return options, args
 
-opts, args = parseOptions()
 
-class LeafAndSpine(Topo):
-    def __init__(self, spine=2, leaf=2, fanout=2, **opts):
+opts, args = parseOptions( )
+
+
+class LeafAndSpine( Topo ):
+    def __init__( self, spine=2, leaf=2, fanout=2, **opts ):
         "Create Leaf and Spine Topo."
 
-        Topo.__init__(self, **opts)
+        Topo.__init__( self, **opts )
 
         # Add spine switches
-        spines = {}
-        for s in range(spine):
-            spines[s] = self.addSwitch('spine10%s' % (s + 1), dpid="00000000010%s" % (s + 1))
+        spines = { }
+        for s in range( spine ):
+            spines[ s ] = self.addSwitch( 'spine10%s' % (s + 1),
+                                          dpid="00000000010%s" % (s + 1) )
         # Set link speeds to 100Mb/s
-        linkopts = dict(bw=100)
+        linkopts = dict( bw=100 )
 
         # Add Leaf switches
-        for ls in range(leaf):
-            leafSwitch = self.addSwitch('leaf%s' % (ls + 1), dpid="00000000000%s" % (1+ls))
+        for ls in range( leaf ):
+            leafSwitch = self.addSwitch( 'leaf%s' % (ls + 1),
+                                         dpid="00000000000%s" % (1 + ls) )
             # Connect leaf to all spines
-            for s in range(spine):
-                switch = spines[s]
-                self.addLink(leafSwitch, switch, **linkopts)
+            for s in range( spine ):
+                switch = spines[ s ]
+                self.addLink( leafSwitch, switch, **linkopts )
             # Add hosts under a leaf, fanout hosts per leaf switch
-            for f in range(fanout):
-                host = self.addHost('h%s' % (ls * fanout + f + 1),
-                                         cls=IpHost,
-                                         ip='10.0.%s.%s/24' % ((ls + 1), (f + 1)),
-                                         gateway='10.0.%s.254' % (ls + 1))
-                self.addLink(host, leafSwitch, **linkopts)
+            for f in range( fanout ):
+                host = self.addHost( 'h%s' % (ls * fanout + f + 1),
+                                     cls=IpHost,
+                                     ip='10.0.%s.%s/24' % ((ls + 1), (f + 1)),
+                                     gateway='10.0.%s.254' % (ls + 1) )
+                self.addLink( host, leafSwitch, **linkopts )
 
-class IpHost(Host):
-    def __init__(self, name, gateway, *args, **kwargs):
-        super(IpHost, self).__init__(name, *args, **kwargs)
+
+class IpHost( Host ):
+    def __init__( self, name, gateway, *args, **kwargs ):
+        super( IpHost, self ).__init__( name, *args, **kwargs )
         self.gateway = gateway
 
-    def config(self, **kwargs):
-        Host.config(self, **kwargs)
-        mtu = "ifconfig "+self.name+"-eth0 mtu 1490"
-        self.cmd(mtu)
-        self.cmd('ip route add default via %s' % self.gateway)
+    def config( self, **kwargs ):
+        Host.config( self, **kwargs )
+        mtu = "ifconfig " + self.name + "-eth0 mtu 1490"
+        self.cmd( mtu )
+        self.cmd( 'ip route add default via %s' % self.gateway )
 
-def config(opts):
+
+def config( opts ):
     spine = opts.spine
     leaf = opts.leaf
-    fanout = opts.fanout 
-    controllers= [ os.environ['OC%s' % i] for i in range(1,opts.onos+1) ] if (opts.onos) else ['127.0.0.1']
-    topo = LeafAndSpine(spine=spine, leaf=leaf, fanout=fanout)
-    net = Mininet(topo=topo, link=TCLink, build=False,
-                  switch=UserSwitch,
-                  controller = None,
-                  autoSetMacs = True)
+    fanout = opts.fanout
+    controllers = [ os.environ[ 'OC%s' % i ] for i in
+                    range( 1, opts.onos + 1 ) ] if (opts.onos) else [
+        '127.0.0.1' ]
+    topo = LeafAndSpine( spine=spine, leaf=leaf, fanout=fanout )
+    net = Mininet( topo=topo, link=TCLink, build=False,
+                   switch=UserSwitch,
+                   controller=None,
+                   autoSetMacs=True )
     i = 0
     for ip in controllers:
-        net.addController( "c%s" % (i), controller=RemoteController, ip=ip)
+        net.addController( "c%s" % (i), controller=RemoteController, ip=ip )
         i += 1;
-    net.build()
-    net.start()
-    CLI(net)
-    net.stop()
+    net.build( )
+    net.start( )
+    CLI( net )
+    net.stop( )
+
 
 if __name__ == '__main__':
-    setLogLevel('info')
-    config(opts)
-    os.system('sudo mn -c')
-
+    setLogLevel( 'info' )
+    config( opts )
+    os.system( 'sudo mn -c' )