[ONOS-7650] SRRouting: add case 602 fix case 620 and 622

Change-Id: Ie6cf28c5df71db16491519c305587ecb8aee715c
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
index 52a05fd..0658e5f 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>101,102,103,104,105,106,107,108,109,201,202,203,204,205,206,207,208,209,301,302,303,304,305,306,307,308,309</testcases>
+    <testcases>101,102,103,104,105,106,107,108,109,201,202,203,204,205,206,207,208,209,301,302,303,304,305,306,307,308,309,601,602,603,604,605,606,620,621,622,630,640,641,642,643,651</testcases>
 
     <GRAPH>
         <nodeCluster>Fabric</nodeCluster>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index dafbacd..53f52bb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -527,6 +527,59 @@
         verify( main, disconnected=False, external=False )
         lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
 
+    def CASE602( self, main ):
+        """"
+        Take down a leaf switch that is paired and has a dual homed host
+        Restore the leaf switch
+        Repeat for various dual homed hosts and paired switches
+        """
+        from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import *
+        from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+        main.case( "Drop a leaf switch that is paired and has a dual homed host." )
+        setupTest( main, test_idx=602, onosNodes=3 )
+        verify( main, disconnected=False )
+        # Kill leaf-2
+        lib.killSwitch( main, "leaf2", 9, 38 )
+        hostLocations = { "h4v6": "of:0000000000000003/6",
+                          "h5v6": "of:0000000000000003/7",
+                          "h4v4": "of:0000000000000003/10",
+                          "h5v4": "of:0000000000000003/11" }
+        lib.verifyHostLocations( main, hostLocations )
+        main.disconnectedIpv4Hosts = [ "h3v4" ]
+        main.disconnectedIpv6Hosts = [ "h3v6" ]
+        verify( main )
+        # Recover leaf-2
+        lib.recoverSwitch( main, "leaf2", 10, 48, rediscoverHosts=True)
+        hostLocations = { "h4v6": [ "of:0000000000000002/7", "of:0000000000000003/6" ],
+                          "h5v6": [ "of:0000000000000002/8", "of:0000000000000003/7" ],
+                          "h4v4": [ "of:0000000000000002/10", "of:0000000000000003/10" ],
+                          "h5v4": [ "of:0000000000000002/11", "of:0000000000000003/11" ] }
+        lib.verifyHostLocations( main, hostLocations )
+        main.disconnectedIpv4Hosts = []
+        main.disconnectedIpv6Hosts = []
+        verify( main, disconnected=False )
+        # Kill leaf-4
+        lib.killSwitch( main, "leaf4", 9, 38 )
+        hostLocations = { "h9v6": "of:0000000000000005/6",
+                          "h10v6": "of:0000000000000005/7",
+                          "h9v4": "of:0000000000000005/9",
+                          "h10v4": "of:0000000000000005/10" }
+        lib.verifyHostLocations( main, hostLocations )
+        main.disconnectedIpv4Hosts = [ "h8v4" ]
+        main.disconnectedIpv6Hosts = [ "h8v6" ]
+        verify( main )
+        # Recover leaf-4
+        lib.recoverSwitch( main, "leaf4", 10, 48, rediscoverHosts=True)
+        hostLocations = { "h9v6": [ "of:0000000000000004/7", "of:0000000000000005/6" ],
+                          "h10v6": [ "of:0000000000000004/8", "of:0000000000000005/7" ],
+                          "h9v4": [ "of:0000000000000004/10", "of:0000000000000005/9" ],
+                          "h10v4": [ "of:0000000000000004/11", "of:0000000000000005/10" ] }
+        lib.verifyHostLocations( main, hostLocations )
+        main.disconnectedIpv4Hosts = []
+        main.disconnectedIpv6Hosts = []
+        verify( main, disconnected=False )
+        lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
+
     def CASE603( self, main ):
         """"
         Drop HAGG-1 device and test connectivity.
@@ -745,15 +798,61 @@
         main.case( "Take down one of double links towards the spine" )
         setupTest( main, test_idx=620, onosNodes=3 )
         verify( main, disconnected=False )
+        groupBuckets = { "of:0000000000000002": { "10.1.0.0/24": 4, "10.1.10.0/24": 4,
+                                                  "10.2.0.0/24": 1, "10.2.30.0/24": 1, "10.2.20.0/24": 1,
+                                                  "10.2.10.0/24": 4, "10.2.40.0/24": 4,
+                                                  "10.3.0.0/24": 4, "10.3.10.0/24": 8, "10.3.30.0/24": 8,
+                                                  "10.3.20.0/24": 4, "10.5.10.0/24": 4, "10.5.20.0/24": 4 },
+                         "of:0000000000000003": { "10.1.0.0/24": 4, "10.1.10.0/24": 4,
+                                                  "10.2.0.0/24": 4, "10.2.30.0/24": 1, "10.2.20.0/24": 1,
+                                                  "10.2.10.0/24": 1, "10.2.40.0/24": 1,
+                                                  "10.3.0.0/24": 4, "10.3.10.0/24": 8, "10.3.30.0/24": 8,
+                                                  "10.3.20.0/24": 4, "10.5.10.0/24": 4, "10.5.20.0/24": 4 },
+                         "of:0000000000000004": { "10.1.0.0/24": 4, "10.1.10.0/24": 4,
+                                                  "10.2.0.0/24": 4, "10.2.30.0/24": 8, "10.2.20.0/24": 8,
+                                                  "10.2.10.0/24": 4, "10.2.40.0/24": 4,
+                                                  "10.3.0.0/24": 1, "10.3.10.0/24": 1, "10.3.30.0/24": 1,
+                                                  "10.3.20.0/24": 4, "10.5.10.0/24": 4, "10.5.20.0/24": 4 },
+                         "of:0000000000000005": { "10.1.0.0/24": 4, "10.1.10.0/24": 4,
+                                                  "10.2.0.0/24": 4, "10.2.30.0/24": 8, "10.2.20.0/24": 8,
+                                                  "10.2.10.0/24": 4, "10.2.40.0/24": 4,
+                                                  "10.3.0.0/24": 4, "10.3.10.0/24": 1, "10.3.30.0/24": 1,
+                                                  "10.3.20.0/24": 1, "10.5.10.0/24": 4, "10.5.20.0/24": 4 } }
+        for switch, subnets in groupBuckets.items():
+            lib.checkGroupsForBuckets( main, switch, subnets )
+        # Take down one of double links
         portsToDisable = [ [ "of:0000000000000002", 1 ], [ "of:0000000000000002", 3 ],
                            [ "of:0000000000000003", 1 ], [ "of:0000000000000003", 3 ],
                            [ "of:0000000000000004", 1 ], [ "of:0000000000000004", 3 ],
                            [ "of:0000000000000005", 1 ], [ "of:0000000000000005", 3 ] ]
         lib.disablePortBatch( main, portsToDisable, 10, 32 )
-        # TODO: check buckets in groups
+        groupBucketsNew = { "of:0000000000000002": { "10.1.0.0/24": 2, "10.1.10.0/24": 2,
+                                                     "10.2.0.0/24": 1, "10.2.30.0/24": 1, "10.2.20.0/24": 1,
+                                                     "10.2.10.0/24": 2, "10.2.40.0/24": 2,
+                                                     "10.3.0.0/24": 2, "10.3.10.0/24": 4, "10.3.30.0/24": 4,
+                                                     "10.3.20.0/24": 2, "10.5.10.0/24": 2, "10.5.20.0/24": 2 },
+                            "of:0000000000000003": { "10.1.0.0/24": 2, "10.1.10.0/24": 2,
+                                                     "10.2.0.0/24": 2, "10.2.30.0/24": 1, "10.2.20.0/24": 1,
+                                                     "10.2.10.0/24": 1, "10.2.40.0/24": 1,
+                                                     "10.3.0.0/24": 2, "10.3.10.0/24": 4, "10.3.30.0/24": 4,
+                                                     "10.3.20.0/24": 2, "10.5.10.0/24": 2, "10.5.20.0/24": 2 },
+                            "of:0000000000000004": { "10.1.0.0/24": 2, "10.1.10.0/24": 2,
+                                                     "10.2.0.0/24": 2, "10.2.30.0/24": 4, "10.2.20.0/24": 4,
+                                                     "10.2.10.0/24": 2, "10.2.40.0/24": 2,
+                                                     "10.3.0.0/24": 1, "10.3.10.0/24": 1, "10.3.30.0/24": 1,
+                                                     "10.3.20.0/24": 2, "10.5.10.0/24": 2, "10.5.20.0/24": 2 },
+                            "of:0000000000000005": { "10.1.0.0/24": 2, "10.1.10.0/24": 2,
+                                                     "10.2.0.0/24": 2, "10.2.30.0/24": 4, "10.2.20.0/24": 4,
+                                                     "10.2.10.0/24": 2, "10.2.40.0/24": 2,
+                                                     "10.3.0.0/24": 2, "10.3.10.0/24": 1, "10.3.30.0/24": 1,
+                                                     "10.3.20.0/24": 1, "10.5.10.0/24": 2, "10.5.20.0/24": 2 } }
+        for switch, subnets in groupBucketsNew.items():
+            lib.checkGroupsForBuckets( main, switch, subnets )
         verify( main, disconnected=False )
+        # Bring up the links
         lib.enablePortBatch( main, portsToDisable, 10, 48 )
-        # TODO: check buckets in groups
+        for switch, subnets in groupBuckets.items():
+            lib.checkGroupsForBuckets( main, switch, subnets )
         verify( main, disconnected=False )
         lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
 
@@ -818,7 +917,6 @@
         main.case( "Take down all uplinks from a paired leaf switch" )
         setupTest( main, test_idx=622, onosNodes=3 )
         verify( main, disconnected=False )
-        ctrl = main.Cluster.active( 0 )
         hostLocations = { "h4v6": [ "of:0000000000000002/7", "of:0000000000000003/6" ],
                           "h5v6": [ "of:0000000000000002/8", "of:0000000000000003/7" ],
                           "h4v4": [ "of:0000000000000002/10", "of:0000000000000003/10" ],
@@ -827,6 +925,8 @@
         linksToRemove = [ ["spine101", "leaf2"], ["spine102", "leaf2"] ]
         lib.killLinkBatch( main, linksToRemove, 40, 10 )
         # TODO: more verifications are required
+        main.disconnectedIpv4Hosts = [ "h3v4" ]
+        main.disconnectedIpv6Hosts = [ "h3v6" ]
         verify( main )
         hostLocations = { "h4v6": "of:0000000000000003/6",
                           "h5v6": "of:0000000000000003/7",
@@ -834,7 +934,9 @@
                           "h5v4": "of:0000000000000003/11" }
         lib.verifyHostLocations( main, hostLocations )
         lib.restoreLinkBatch( main, linksToRemove, 48, 10 )
-        verify( main )
+        main.disconnectedIpv4Hosts = []
+        main.disconnectedIpv6Hosts = []
+        verify( main, disconnected=False )
         hostLocations = { "h4v6": [ "of:0000000000000002/7", "of:0000000000000003/6" ],
                           "h5v6": [ "of:0000000000000002/8", "of:0000000000000003/7" ],
                           "h4v4": [ "of:0000000000000002/10", "of:0000000000000003/10" ],
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 8cb4de1..03ea850 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -310,34 +310,33 @@
                                  onfail="route-add command failed")
 
     @staticmethod
-    def checkGroupsForBuckets( main, deviceId, subnetDict, routingTable=30):
+    def checkGroupsForBuckets( main, deviceId, subnetDict, routingTable=30 ):
         """
         Check number of groups for each subnet on device deviceId and matches
         it with an expected value. subnetDict is a dictionarty containing values
         of the type "10.0.1.0/24" : 5.
         """
-        main.step("Checking if number of groups for subnets in device {0} is as expected.".format(deviceId))
-        groups = main.Cluster.active( 0 ).CLI.getGroups(deviceId, group_type="select")
-        flows = main.Cluster.active( 0 ).CLI.flows(jsonFormat=False, device=deviceId)
+        main.step( "Checking if number of groups for subnets in device {0} is as expected.".format( deviceId ) )
+        groups = main.Cluster.active( 0 ).CLI.getGroups( deviceId, groupType="select" )
+        flows = main.Cluster.active( 0 ).CLI.flows( jsonFormat=False, device=deviceId )
 
-        for subnet, number_in_select in subnetDict.iteritems():
+        result = main.TRUE
+        for subnet, numberInSelect in subnetDict.iteritems():
             for flow in flows.splitlines():
-                if "tableId={0}".format(routingTable) in flow and subnet in flow:
-
+                if "tableId={0}".format( routingTable ) in flow and subnet in flow:
                     # this will match the group id that this flow entry points to, for example :
                     # 0x70000041 in flow entry which contains "deferred=[GROUP:0x70000041], transition=TABLE:60,"
-                    group_id = re.search(r".*GROUP:(0x.*)], transition.*", flow).groups()[0]
-
+                    groupId = re.search( r".*GROUP:(0x.*)], transition.*", flow ).groups()[0]
                     count = 0
                     for group in groups.splitlines():
-                        if 'id={0}'.format(group_id) in group:
+                        if 'id={0}'.format( groupId ) in group:
                             count += 1
-
-                    utilities.assert_equals( expect=True, actual=(count-1 == number_in_select),
-                                             onpass="Number of buckets in select group is correct",
-                                             onfail="Mismatch in number of buckets of select group, found {0}, expected {1} for subnet {2} on device {3}".format(count - 1, number_in_select, subnet, deviceId))
-                else:
-                    continue
+                    if count - 1 != numberInSelect:
+                        result = main.FALSE
+                        main.log.warn( "Mismatch in number of buckets of select group, found {0}, expected {1} for subnet {2} on device {3}".format( count - 1, numberInSelect, subnet, deviceId ) )
+        utilities.assert_equals( expect=main.TRUE, actual=result,
+                                 onpass="All bucket numbers are as expected",
+                                 onfail="Some bucket numbers are not as expected" )
 
     @staticmethod
     def checkFlows( main, minFlowCount, tag="", dumpflows=True, sleep=10 ):