[SDFAB-283] Paired-Leaf Switch Failure tests (ONL Reboot)

Change-Id: I2e6648baf15a1af49afea1689aec69c25b745c03
diff --git a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
index a49aef3..0f69f41 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRStaging/SRpairedLeaves/SRpairedLeaves.py
@@ -105,7 +105,6 @@
         main.funcs.linkUp( killDevice, killPort, srcComponent, dstComponent, shortDesc, longDesc,
                            bidirectional=False )
         # Cleanup
-        # TODO: Verify Cleanup works as intended, even with multiple testcases running in a row
         main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
         main.funcs.cleanup( main )
 
@@ -148,7 +147,6 @@
         main.funcs.linkUp( killDevice, killPort, srcComponent, dstComponent, shortDesc, longDesc,
                            bidirectional=False )
         # Cleanup
-        # TODO: Verify Cleanup works as intended, even with multiple testcases running in a row
         main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
         main.funcs.cleanup( main )
 
@@ -191,7 +189,6 @@
         main.funcs.linkUp( killDevice, killPort, srcComponent, dstComponent, shortDesc, longDesc,
                 bidirectional=False )
         # Cleanup
-        # TODO: Verify Cleanup works as intended, even with multiple testcases running in a row
         main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
         main.funcs.cleanup( main )
 
@@ -234,6 +231,169 @@
         main.funcs.linkUp( killDevice, killPort, srcComponent, dstComponent, shortDesc, longDesc,
                 bidirectional=False )
         # Cleanup
-        # TODO: Verify Cleanup works as intended, even with multiple testcases running in a row
+        main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+        main.funcs.cleanup( main )
+
+    def CASE201( self, main ):
+        """
+        Connect to Pod
+        Create Flow between 2 dual homed hosts
+        Kill the leaf that traffic first flows into
+        Verify flow continues using other link
+        Collect logs and analyze results
+        """
+        try:
+            from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+            import json
+        except ImportError:
+            main.log.error( "SRStagingTest not found. Exiting the test" )
+            main.cleanAndExit()
+        try:
+            main.funcs
+        except ( NameError, AttributeError ):
+            main.funcs = SRStagingTest()
+
+        descPrefix = "CASE201-Source-Leaf"
+        pod = main.params['GRAPH'].get( 'nodeCluster', "hardware" )
+        main.funcs.setupTest( main,
+                              topology='0x2',
+                              onosNodes=3,
+                              description="%s tests on the %s pod" % ( descPrefix, pod ) )
+        srcComponent = getattr( main, 'Compute1' )
+        dstComponent = getattr( main, 'Compute2' )
+
+        targets = main.funcs.getHostConnections( main, srcComponent )
+        shortDescFailure = descPrefix + "-Failure"
+        longDescFailure = "%s Failure: Bring down switch with traffic from %s" % ( descPrefix, srcComponent.name )
+        shortDescRecovery = descPrefix + "-Recovery"
+        longDescRecovery = "%s Recovery: Bring up switch previously killed" % descPrefix
+        main.funcs.onlReboot( targets, srcComponent, dstComponent,
+                              shortDescFailure, longDescFailure,
+                              shortDescRecovery, longDescRecovery,
+                              stat='packetsReceived', bidirectional=False )
+        # Cleanup
+        main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+        main.funcs.cleanup( main )
+
+    def CASE202( self, main ):
+        """
+        Connect to Pod
+        Create Flow between 2 dual homed hosts
+        Kill the last leaf that traffic flows out of
+        Verify flow continues using other link
+        Collect logs and analyze results
+        """
+        try:
+            from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+            import json
+        except ImportError:
+            main.log.error( "SRStagingTest not found. Exiting the test" )
+            main.cleanAndExit()
+        try:
+            main.funcs
+        except ( NameError, AttributeError ):
+            main.funcs = SRStagingTest()
+
+        descPrefix = "CASE202-Destination-Leaf"
+        pod = main.params['GRAPH'].get( 'nodeCluster', "hardware" )
+        main.funcs.setupTest( main,
+                              topology='0x2',
+                              onosNodes=3,
+                              description="%s tests on the %s pod" % ( descPrefix, pod ) )
+        srcComponent = getattr( main, 'Compute1' )
+        dstComponent = getattr( main, 'Compute2' )
+
+        targets = main.funcs.getHostConnections( main, dstComponent )
+        shortDescFailure = descPrefix + "-Failure"
+        longDescFailure = "%s Failure: Bring down switch with traffic from %s" % ( descPrefix, srcComponent.name )
+        shortDescRecovery = descPrefix + "-Recovery"
+        longDescRecovery = "%s Recovery: Bring up switch previously killed" % descPrefix
+        main.funcs.onlReboot( targets, srcComponent, dstComponent,
+                              shortDescFailure, longDescFailure,
+                              shortDescRecovery, longDescRecovery,
+                              stat='packetsSent', bidirectional=False )
+        # Cleanup
+        main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+        main.funcs.cleanup( main )
+
+    def CASE203( self, main ):
+        """
+        Connect to Pod
+        Create Flow between 1 dual homed host and 1 single homed host
+        Kill the leaf that traffic first flows into
+        Verify flow continues using other link
+        Collect logs and analyze results
+        """
+        try:
+            from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+            import json
+        except ImportError:
+            main.log.error( "SRStagingTest not found. Exiting the test" )
+            main.cleanAndExit()
+        try:
+            main.funcs
+        except ( NameError, AttributeError ):
+            main.funcs = SRStagingTest()
+
+        descPrefix = "CASE203-Source-Leaf"
+        pod = main.params['GRAPH'].get( 'nodeCluster', "hardware" )
+        main.funcs.setupTest( main,
+                              topology='0x2',
+                              onosNodes=3,
+                              description="%s tests on the %s pod" % ( descPrefix, pod ) )
+        srcComponentList = [ getattr( main, name ) for name in [ 'ManagmentServer', 'Compute1', 'Compute2' ] ]
+        dstComponent = getattr( main, 'Compute3' )
+
+        targets = main.funcs.getHostConnections( main, srcComponentList, excludedDIDs=[ 'leaf2' ] )
+        shortDescFailure = descPrefix + "-Failure"
+        longDescFailure = "%s Failure: Bring down switch with traffic to %s" % ( descPrefix, dstComponent.name )
+        shortDescRecovery = descPrefix + "-Recovery"
+        longDescRecovery = "%s Recovery: Bring up switch previously killed" % descPrefix
+        main.funcs.onlReboot( targets, srcComponentList, dstComponent,
+                              shortDescFailure, longDescFailure,
+                              shortDescRecovery, longDescRecovery,
+                              stat='packetsReceived', bidirectional=False )
+        # Cleanup
+        main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
+        main.funcs.cleanup( main )
+
+    def CASE204( self, main ):
+        """
+        Connect to Pod
+        Create Flow between 1 dual homed host and 1 single homed host
+        Kill the last leaf that traffic flows out of
+        Verify flow continues using other link
+        Collect logs and analyze results
+        """
+        try:
+            from tests.USECASE.SegmentRouting.SRStaging.dependencies.SRStagingTest import SRStagingTest
+            import json
+        except ImportError:
+            main.log.error( "SRStagingTest not found. Exiting the test" )
+            main.cleanAndExit()
+        try:
+            main.funcs
+        except ( NameError, AttributeError ):
+            main.funcs = SRStagingTest()
+
+        descPrefix = "CASE204-Destination-Leaf"
+        pod = main.params['GRAPH'].get( 'nodeCluster', "hardware" )
+        main.funcs.setupTest( main,
+                              topology='0x2',
+                              onosNodes=3,
+                              description="%s tests on the %s pod" % ( descPrefix, pod ) )
+        srcComponent = getattr( main, 'Compute3' )
+        dstComponentList = [ getattr( main, name ) for name in [ 'ManagmentServer', 'Compute1', 'Compute2' ] ]
+
+        targets = main.funcs.getHostConnections( main, dstComponentList, excludedDIDs=[ 'leaf2' ] )
+        shortDescFailure = descPrefix + "-Failure"
+        longDescFailure = "%s Failure: Bring down switch with traffic from %s" % ( descPrefix, srcComponent.name )
+        shortDescRecovery = descPrefix + "-Recovery"
+        longDescRecovery = "%s Recovery: Bring up switch previously killed" % descPrefix
+        main.funcs.onlReboot( targets, srcComponent, dstComponentList,
+                              shortDescFailure, longDescFailure,
+                              shortDescRecovery, longDescRecovery,
+                              stat='packetsSent', bidirectional=False )
+        # Cleanup
         main.log.warn( json.dumps( main.downtimeResults, indent=4, sort_keys=True ) )
         main.funcs.cleanup( main )