[ONOS-7502] Add switch failure test cases to SRRouting

Change-Id: Id8c876c64518cd854a6d8be4e8558bf9d8b9bdb4
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index 9344021..dddc584 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -18,7 +18,7 @@
                                ipv4=1,
                                ipv6=0,
                                countFlowsGroups=False,
-                               description="Ping between all ipv4 hosts in the topology")
+                               description="Ping between all ipv4 hosts in the topology" )
 
     def CASE2( self, main ):
         """
@@ -35,7 +35,7 @@
                                ipv4=0,
                                ipv6=1,
                                countFlowsGroups=False,
-                               description="Ping between all ipv6 hosts in the topology")
+                               description="Ping between all ipv6 hosts in the topology" )
 
     def CASE3( self, main ):
         """
@@ -52,7 +52,7 @@
                                ipv4=1,
                                ipv6=1,
                                countFlowsGroups=False,
-                               description="Ping between all ipv4 and ipv6 hosts in the topology")
+                               description="Ping between all ipv4 and ipv6 hosts in the topology" )
 
     def CASE4( self, main ):
         """
@@ -70,7 +70,7 @@
                                ipv6=0,
                                description="Ping between all ipv4 hosts in the topology and check connectivity to external hosts",
                                checkExternalHost=True,
-                               countFlowsGroups=False)
+                               countFlowsGroups=False )
 
     def CASE5( self, main ):
         """
@@ -88,7 +88,7 @@
                                ipv6=1,
                                description="Ping between all ipv6 hosts in the topology and check connectivity to external hosts",
                                checkExternalHost=True,
-                               countFlowsGroups=False)
+                               countFlowsGroups=False )
 
     def CASE6( self, main ):
         """
@@ -106,7 +106,7 @@
                                ipv6=1,
                                description="Ping between all ipv4 and ipv6 hosts in the topology and check connectivity to external hosts",
                                checkExternalHost=True,
-                               countFlowsGroups=False)
+                               countFlowsGroups=False )
 
     def CASE7( self, main ):
         """
@@ -127,7 +127,7 @@
                                description="Ping between from ipv4 hosts to external host configured with route-add command.",
                                checkExternalHost=False,
                                countFlowsGroups=False,
-                               staticRouteConfigure=True)
+                               staticRouteConfigure=True )
 
     def CASE8( self, main ):
         """
@@ -148,7 +148,7 @@
                                description="Ping between from ipv6 hosts to external host configured with route-add command.",
                                checkExternalHost=False,
                                countFlowsGroups=False,
-                               staticRouteConfigure=True)
+                               staticRouteConfigure=True )
 
     def CASE9( self, main ):
         """
@@ -169,4 +169,184 @@
                                description="Ping between from ipv4 and ipv6 hosts to external hosts configured with route-add command.",
                                checkExternalHost=False,
                                countFlowsGroups=False,
-                               staticRouteConfigure=True)
+                               staticRouteConfigure=True )
+
+    def CASE201( self, main ):
+        """
+        Kill and recover spine switches
+        Ping between all ipv4 hosts in the topology.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=201,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=0,
+                               countFlowsGroups=False,
+                               description="Test switch failures with IPv4 hosts",
+                               switchFailure=True )
+
+    def CASE202( self, main ):
+        """
+        Kill and recover spine switches
+        Ping between all ipv6 hosts in the topology.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=202,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=0,
+                               ipv6=1,
+                               countFlowsGroups=False,
+                               description="Test switch failures with IPv6 hosts",
+                               switchFailure=True )
+
+    def CASE203( self, main ):
+        """
+        Kill and recover spine switches
+        Ping between all ipv4 and ipv6 hosts in the topology.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=203,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=1,
+                               countFlowsGroups=False,
+                               description="Test switch failures with IPv4 and IPv6 hosts",
+                               switchFailure=True )
+
+    def CASE204( self, main ):
+        """
+        Kill and recover spine switches
+        Ping between all ipv4 hosts in the topology and check connectivity to external ipv4 hosts
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=204,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=0,
+                               description="Test switch failures with IPv4 hosts (including external hosts)",
+                               checkExternalHost=True,
+                               switchFailure=True )
+
+    def CASE205( self, main ):
+        """
+        Kill and recover spine switches
+        Ping between all ipv6 hosts in the topology and check connectivity to external ipv6 hosts
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=205,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=0,
+                               ipv6=1,
+                               description="Test switch failures with IPv6 hosts (including external hosts)",
+                               checkExternalHost=True,
+                               switchFailure=True )
+
+    def CASE206( self, main ):
+        """
+        Kill and recover spine switches
+        Ping between all ipv4 and ipv6 hosts in the topology and check connectivity to external ipv4 and ipv6 hosts
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=206,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=1,
+                               description="Test switch failures with IPv4 and IPv6 hosts (including external hosts)",
+                               checkExternalHost=True,
+                               switchFailure=True )
+
+    def CASE207( self, main ):
+        """
+        Kill and recover spine switches
+        Ping between ipv4 hosts and an external host that is not configured in
+        external router config, but reachable through the use of route-add command.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=207,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=0,
+                               description="Test switch failures with IPv4 hosts (including external host configured with route-add command)",
+                               checkExternalHost=False,
+                               countFlowsGroups=False,
+                               staticRouteConfigure=True,
+                               switchFailure=True )
+
+    def CASE208( self, main ):
+        """
+        Kill and recover spine switches
+        Ping between ipv6 hosts and an external host that is not configured in
+        external router config, but reachable through the use of route-add command.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=208,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=0,
+                               ipv6=1,
+                               description="Test switch failures with IPv6 hosts (including external host configured with route-add command)",
+                               checkExternalHost=False,
+                               countFlowsGroups=False,
+                               staticRouteConfigure=True,
+                               switchFailure=True )
+
+    def CASE209( self, main ):
+        """
+        Kill and recover spine switches
+        Ping between ipv4 and pv6 hosts and external hosts that is not configured in
+        external router config, but reachable through the use of route-add command.
+        """
+
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+        SRRoutingTest.runTest( main,
+                               test_idx=209,
+                               onosNodes=3,
+                               dhcp=1,
+                               routers=1,
+                               ipv4=1,
+                               ipv6=1,
+                               description="Test switch failures with IPv4 and IPv6 hosts (including external host configured with route-add command)",
+                               checkExternalHost=False,
+                               countFlowsGroups=False,
+                               staticRouteConfigure=True,
+                               switchFailure=True )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index a44efd9..eb747b0 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -32,7 +32,7 @@
     @staticmethod
     def runTest( main, test_idx, onosNodes, dhcp, routers, ipv4, ipv6,
                  description, countFlowsGroups=False, checkExternalHost=False,
-                 staticRouteConfigure=False):
+                 staticRouteConfigure=False, switchFailure=False ):
 
         skipPackage = False
         init = False
@@ -58,6 +58,8 @@
 
         run.installOnos( main, skipPackage=skipPackage, cliSleep=5,
                          parallel=False )
+
+        # Load configuration files
         run.loadJson( main )
         run.loadChart( main )
 
@@ -69,10 +71,12 @@
             if (ipv6):
                 run.addStaticOnosRoute( main, "2000::8700/120", "2000::101")
 
-        if (countFlowsGroups):
+        if countFlowsGroups:
             run.loadCount( main )
+        if switchFailure:
+            run.loadSwitchFailureChart( main )
 
-        # wait some
+        # wait some time
         time.sleep( 5 )
 
         if hasattr( main, 'Mininet1' ):
@@ -94,9 +98,21 @@
         run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
 
         # check flows / groups numbers
-        if (countFlowsGroups):
+        if countFlowsGroups:
             run.checkFlowsGroupsFromFile(main)
 
+        # Test switch failures
+        if switchFailure:
+            for switch, expected in main.switchFailureChart.items():
+                run.killSwitch( main, switch, expected['switches_after_failure'], expected['links_after_failure'] )
+                run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+                if countFlowsGroups:
+                    run.checkFlowsGroupsFromFile(main)
+                run.recoverSwitch( main, switch, expected['switches_before_failure'], expected['links_before_failure'] )
+                run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+                if countFlowsGroups:
+                    run.checkFlowsGroupsFromFile(main)
+
         if hasattr( main, 'Mininet1' ):
             run.cleanup( main )
         else:
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=0_ipv6=1_dhcp=1_routers=1.switchFailureChart b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=0_ipv6=1_dhcp=1_routers=1.switchFailureChart
new file mode 100644
index 0000000..a1d69ae
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=0_ipv6=1_dhcp=1_routers=1.switchFailureChart
@@ -0,0 +1,10 @@
+{
+    "spine101": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 },
+    "spine102": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 }
+}
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=0_ipv6=1_dhcp=1_routers=1_external=1.switchFailureChart b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=0_ipv6=1_dhcp=1_routers=1_external=1.switchFailureChart
new file mode 100644
index 0000000..a1d69ae
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=0_ipv6=1_dhcp=1_routers=1_external=1.switchFailureChart
@@ -0,0 +1,10 @@
+{
+    "spine101": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 },
+    "spine102": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 }
+}
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=0_ipv6=1_dhcp=1_routers=1_static=1.switchFailureChart b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=0_ipv6=1_dhcp=1_routers=1_static=1.switchFailureChart
new file mode 100644
index 0000000..a1d69ae
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=0_ipv6=1_dhcp=1_routers=1_static=1.switchFailureChart
@@ -0,0 +1,10 @@
+{
+    "spine101": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 },
+    "spine102": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 }
+}
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=1_routers=1.switchFailureChart b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=1_routers=1.switchFailureChart
new file mode 100644
index 0000000..a1d69ae
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=1_routers=1.switchFailureChart
@@ -0,0 +1,10 @@
+{
+    "spine101": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 },
+    "spine102": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 }
+}
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=1_routers=1_external=1.switchFailureChart b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=1_routers=1_external=1.switchFailureChart
new file mode 100644
index 0000000..a1d69ae
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=1_routers=1_external=1.switchFailureChart
@@ -0,0 +1,10 @@
+{
+    "spine101": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 },
+    "spine102": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 }
+}
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=1_routers=1_static=1.switchFailureChart b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=1_routers=1_static=1.switchFailureChart
new file mode 100644
index 0000000..a1d69ae
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=1_routers=1_static=1.switchFailureChart
@@ -0,0 +1,10 @@
+{
+    "spine101": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 },
+    "spine102": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 }
+}
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1.switchFailureChart b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1.switchFailureChart
new file mode 100644
index 0000000..a1d69ae
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1.switchFailureChart
@@ -0,0 +1,10 @@
+{
+    "spine101": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 },
+    "spine102": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 }
+}
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1_external=1.switchFailureChart b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1_external=1.switchFailureChart
new file mode 100644
index 0000000..a1d69ae
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1_external=1.switchFailureChart
@@ -0,0 +1,10 @@
+{
+    "spine101": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 },
+    "spine102": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 }
+}
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1_static=1.switchFailureChart b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1_static=1.switchFailureChart
new file mode 100644
index 0000000..a1d69ae
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/switchFailure/COMCAST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1_static=1.switchFailureChart
@@ -0,0 +1,10 @@
+{
+    "spine101": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 },
+    "spine102": { "switches_before_failure": 7,
+                  "links_before_failure": 40,
+                  "switches_after_failure": 6,
+                  "links_after_failure": 22 }
+}