Merge "Adding documentations for the Jenkins file"
diff --git a/TestON/JenkinsFile/wikiGraphRScripts/trendCHO.R b/TestON/JenkinsFile/wikiGraphRScripts/trendCHO.R
index 32b11e7..67d5142 100644
--- a/TestON/JenkinsFile/wikiGraphRScripts/trendCHO.R
+++ b/TestON/JenkinsFile/wikiGraphRScripts/trendCHO.R
@@ -276,10 +276,10 @@
 events_dataFrame$Type <- as.character( events_dataFrame$Type )
 events_dataFrame$Type <- factor( events_dataFrame$Type, levels = unique( events_dataFrame$Type ) )
 
-events_dataFrame$timeStamps <- rev( gsub('^(.{11})(.*)$', '\\1\n\\2', event_fileData$Time ) )
+events_dataFrame$timeStamps <- gsub('^(.{11})(.*)$', '\\1\n\\2', event_fileData$Time )
 
 # Adding a temporary reversed iterative list to the events_dataFrame so that there are no gaps in-between build numbers.
-events_dataFrame$iterative <- rev( seq( 1, nrow( event_fileData ), by = 1 ) )
+events_dataFrame$iterative <- seq( 1, nrow( event_fileData ), by = 1 )
 
 # Omit any data that doesn't exist
 events_dataFrame <- na.omit( events_dataFrame )
@@ -315,10 +315,10 @@
 failures_dataFrame$Type <- as.character( failures_dataFrame$Type )
 failures_dataFrame$Type <- factor( failures_dataFrame$Type, levels = unique( failures_dataFrame$Type ) )
 
-failures_dataFrame$timeStamps <- rev( gsub('^(.{11})(.*)$', '\\1\n\\2', failure_fileData$Time ) )
+failures_dataFrame$timeStamps <- gsub('^(.{11})(.*)$', '\\1\n\\2', failure_fileData$Time )
 
 # Adding a temporary reversed iterative list to the failures_dataFrame so that there are no gaps in-between build numbers.
-failures_dataFrame$iterative <- rev( seq( 1, nrow( failure_fileData ), by = 1 ) )
+failures_dataFrame$iterative <- seq( 1, nrow( failure_fileData ), by = 1 )
 
 # Omit any data that doesn't exist
 failures_dataFrame <- na.omit( failures_dataFrame )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/SRMulticastTest.py b/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/SRMulticastTest.py
index be8e4c9..b0907d7 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/SRMulticastTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/SRMulticastTest.py
@@ -67,7 +67,6 @@
     """
     from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
     for routeName in main.mcastRoutes.keys():
-        main.step( "Verify {} multicast route".format( routeName ) )
         installMcastRoute( main, routeName )
         lib.verifyMulticastTraffic( main, routeName, True )
 
@@ -75,6 +74,7 @@
     """
     Install a multicast route
     """
+    main.step( "Install {} multicast route".format( routeName ) )
     routeData = main.multicastConfig[ routeName ]
     src = main.mcastRoutes[ routeName ][ "src" ]
     dst = main.mcastRoutes[ routeName ][ "dst" ]
@@ -88,7 +88,7 @@
     Verify removal of a multicast route
     """
     routeData = main.multicastConfig[ routeName ]
-    main.step( "Verify removal of {} route".format( routeName ) )
+    main.step( "Remove {} route".format( routeName ) )
     main.Cluster.active( 0 ).CLI.mcastSinkDelete( routeData[ "src" ][ 0 ][ "ip" ], routeData[ "group" ] )
     # TODO: verify the deletion
 
@@ -99,7 +99,7 @@
     from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
     routeData = main.multicastConfig[ routeName ]
     sinkId = routeData[ "dst" ][ sinkIndex ][ "id" ]
-    main.step( "Verify removal of {} sink {}".format( routeName, sinkId ) )
+    main.step( "Remove sink {} of route {}".format( sinkId, routeName ) )
     main.Cluster.active( 0 ).CLI.mcastSinkDelete( routeData[ "src" ][ 0 ][ "ip" ], routeData[ "group" ], sinkId )
     time.sleep( float( main.params[ "timers" ][ "mcastSleep" ] ) )
     lib.verifyMulticastTraffic( main, routeName, expect )
@@ -111,7 +111,7 @@
     from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
     routeData = main.multicastConfig[ routeName ]
     sourceId = [ routeData[ "src" ][ sourceIndex ][ "id" ] ]
-    main.step( "Verify removal of {} source {}".format( routeName, sourceId ) )
+    main.step( "Remove source {} of route {}".format( sourceId, routeName ) )
     main.Cluster.active( 0 ).CLI.mcastSourceDelete( routeData[ "src" ][ 0 ][ "ip" ], routeData[ "group" ], sourceId )
     time.sleep( float( main.params[ "timers" ][ "mcastSleep" ] ) )
     lib.verifyMulticastTraffic( main, routeName, expect )
@@ -145,8 +145,8 @@
     lib.restoreLinkBatch( main, link, int( main.params[ "TOPO" ][ "linkNum" ] ), int( main.params[ "TOPO" ][ "switchNum" ] ) )
     if hostsToDiscover:
         main.Network.discoverHosts( hostList=hostsToDiscover )
-    for host, loc in hostLocations.items():
-        lib.verifyHostLocation( main, host, loc, retry=5 )
+    if hostLocations:
+        lib.verifyHostLocations( main, hostLocations, retry=5 )
     for routeName in expectList.keys():
         lib.verifyMulticastTraffic( main, routeName, True )
 
@@ -165,8 +165,8 @@
     main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="enable" )
     if hostsToDiscover:
         main.Network.discoverHosts( hostList=hostsToDiscover )
-    for host, loc in hostLocations.items():
-        lib.verifyHostLocation( main, host, loc, retry=5 )
+    if hostLocations:
+        lib.verifyHostLocations( main, hostLocations, retry=5 )
     for routeName in expectList.keys():
         lib.verifyMulticastTraffic( main, routeName, True )
 
@@ -183,8 +183,8 @@
         lib.verifyMulticastTraffic( main, routeName, expectList[ routeName ] )
     # Recover the switch(es)
     lib.recoverSwitch( main, switchName, int( main.params[ "TOPO" ][ "switchNum" ] ), int( main.params[ "TOPO" ][ "linkNum" ] ), True if hostsToDiscover else False, hostsToDiscover )
-    for host, loc in hostLocations.items():
-        lib.verifyHostLocation( main, host, loc, retry=5 )
+    if hostLocations:
+        lib.verifyHostLocations( main, hostLocations, retry=5 )
     for routeName in expectList.keys():
         lib.verifyMulticastTraffic( main, routeName, True )
 
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index 2952614..9c97ebb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -656,12 +656,10 @@
                            [ "of:0000000000000003", 1 ], [ "of:0000000000000003", 3 ],
                            [ "of:0000000000000004", 1 ], [ "of:0000000000000004", 3 ],
                            [ "of:0000000000000005", 1 ], [ "of:0000000000000005", 3 ] ]
-        for dpid, port in portsToDisable:
-            main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="disable" )
+        lib.disablePortBatch( main, portsToDisable, 10, 32 )
         # TODO: check buckets in groups
         verify( main, disconnected=False )
-        for dpid, port in portsToDisable:
-            main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="enable" )
+        lib.enablePortBatch( main, portsToDisable, 10, 48 )
         # TODO: check buckets in groups
         verify( main, disconnected=False )
         lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
@@ -704,16 +702,12 @@
                            [ "of:0000000000000006", 5 ], [ "of:0000000000000006", 6 ] ]
         for i in range( 0, 3 ):
             lib.killLinkBatch( main, linksToRemove, 0, 10 )
-            for dpid, port in portsToDisable:
-                main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="disable" )
-            time.sleep( 10 )
+            lib.disablePortBatch( main, portsToDisable, 10, 0 )
             main.disconnectedIpv4Hosts = main.internalIpv4Hosts
             main.disconnectedIpv6Hosts = main.internalIpv6Hosts
             verify( main )
             lib.restoreLinkBatch( main, linksToRemove, 48, 10 )
-            for dpid, port in portsToDisable:
-                main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="enable" )
-            time.sleep( 30 )
+            lib.enablePortBatch( main, portsToDisable, 10, 48 )
             main.Network.discoverHosts( hostList=main.disconnectedIpv4Hosts + main.disconnectedIpv6Hosts )
             time.sleep( 10 )
             main.disconnectedIpv4Hosts = []
@@ -733,43 +727,27 @@
         setupTest( main, test_idx=622, onosNodes=3 )
         verify( main, disconnected=False )
         ctrl = main.Cluster.active( 0 )
-        result1 = ctrl.CLI.verifyHostLocation( "1003::3fe",
-                                               [ "of:0000000000000002/7", "of:0000000000000003/6" ] )
-        result2 = ctrl.CLI.verifyHostLocation( "1004::3fe",
-                                               [ "of:0000000000000002/8", "of:0000000000000003/7" ] )
-        result3 = ctrl.CLI.verifyHostLocation( "10.2.30.1",
-                                               [ "of:0000000000000002/10", "of:0000000000000003/10" ] )
-        result4 = ctrl.CLI.verifyHostLocation( "10.2.20.1",
-                                               [ "of:0000000000000002/11", "of:0000000000000003/11" ] )
-        utilities.assert_equals( expect=main.TRUE, actual=result1 and result2 and result3 and result4,
-                                 onpass="Host locations are correct",
-                                 onfail="Not all host locations are correct" )
+        hostLocations = { "h4v6": [ "of:0000000000000002/7", "of:0000000000000003/6" ],
+                          "h5v6": [ "of:0000000000000002/8", "of:0000000000000003/7" ],
+                          "h4v4": [ "of:0000000000000002/10", "of:0000000000000003/10" ],
+                          "h5v4": [ "of:0000000000000002/11", "of:0000000000000003/11" ] }
+        lib.verifyHostLocations( main, hostLocations )
         linksToRemove = [ ["spine101", "leaf2"], ["spine102", "leaf2"] ]
         lib.killLinkBatch( main, linksToRemove, 40, 10 )
         # TODO: more verifications are required
         verify( main )
-        main.step( "Verify some dual-homed hosts become single-homed" )
-        result1 = ctrl.CLI.verifyHostLocation( "1003::3fe", "of:0000000000000003/6" )
-        result2 = ctrl.CLI.verifyHostLocation( "1004::3fe", "of:0000000000000003/7" )
-        result3 = ctrl.CLI.verifyHostLocation( "10.2.30.1", "of:0000000000000003/10" )
-        result4 = ctrl.CLI.verifyHostLocation( "10.2.20.1", "of:0000000000000003/11" )
-        utilities.assert_equals( expect=main.TRUE, actual=result1 and result2 and result3 and result4,
-                                 onpass="Host locations are correct",
-                                 onfail="Not all host locations are correct" )
+        hostLocations = { "h4v6": "of:0000000000000003/6",
+                          "h5v6": "of:0000000000000003/7",
+                          "h4v4": "of:0000000000000003/10",
+                          "h5v4": "of:0000000000000003/11" }
+        lib.verifyHostLocations( main, hostLocations )
         lib.restoreLinkBatch( main, linksToRemove, 48, 10 )
         verify( main )
-        main.step( "Verify the hosts changed back to be dual-homed" )
-        result1 = ctrl.CLI.verifyHostLocation( "1003::3fe",
-                                               [ "of:0000000000000002/7", "of:0000000000000003/6" ] )
-        result2 = ctrl.CLI.verifyHostLocation( "1004::3fe",
-                                               [ "of:0000000000000002/8", "of:0000000000000003/7" ] )
-        result3 = ctrl.CLI.verifyHostLocation( "10.2.30.1",
-                                               [ "of:0000000000000002/10", "of:0000000000000003/10" ] )
-        result4 = ctrl.CLI.verifyHostLocation( "10.2.20.1",
-                                               [ "of:0000000000000002/11", "of:0000000000000003/11" ] )
-        utilities.assert_equals( expect=main.TRUE, actual=result1 and result2 and result3 and result4,
-                                 onpass="Host locations are correct",
-                                 onfail="Not all host locations are correct" )
+        hostLocations = { "h4v6": [ "of:0000000000000002/7", "of:0000000000000003/6" ],
+                          "h5v6": [ "of:0000000000000002/8", "of:0000000000000003/7" ],
+                          "h4v4": [ "of:0000000000000002/10", "of:0000000000000003/10" ],
+                          "h5v4": [ "of:0000000000000002/11", "of:0000000000000003/11" ] }
+        lib.verifyHostLocations( main, hostLocations )
         lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
 
     def CASE630( self, main ):
@@ -813,7 +791,6 @@
         main.Cluster.active( 0 ).CLI.balanceMasters()
         time.sleep( float( main.params[ 'timers' ][ 'balanceMasterSleep' ] ) )
         verify( main )
-
         portsToDisable = [ [ "of:0000000000000001", 1 ], [ "of:0000000000000103", 1 ],
                            [ "of:0000000000000006", 1 ], [ "of:0000000000000103", 2 ],
                            [ "of:0000000000000101", 9 ], [ "of:0000000000000103", 3 ],
@@ -825,13 +802,17 @@
                            [ "of:0000000000000003", 3 ], [ "of:0000000000000102", 3 ],
                            [ "of:0000000000000004", 3 ], [ "of:0000000000000102", 5 ],
                            [ "of:0000000000000005", 3 ], [ "of:0000000000000102", 7 ] ]
-        for dpid, port in portsToDisable:
-            main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="disable" )
+        lib.disablePortBatch( main, portsToDisable,
+                              int( main.params[ "TOPO" ][ "switchNum" ] ),
+                              int( main.params[ "TOPO" ][ "linkNum" ] ) - len( portsToDisable ),
+                              sleep=0 )
         lib.killOnos( main, [ 2, ], int( main.params[ "TOPO" ][ "switchNum" ] ),
                       int( main.params[ "TOPO" ][ "linkNum" ] ) - len( portsToDisable ), 2 )
         verify( main )
-        for dpid, port in portsToDisable:
-            main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="enable" )
+        lib.enablePortBatch( main, portsToDisable,
+                             int( main.params[ "TOPO" ][ "switchNum" ] ),
+                             int( main.params[ "TOPO" ][ "linkNum" ] ),
+                             sleep=0 )
         lib.recoverOnos( main, [ 2, ], int( main.params[ "TOPO" ][ "switchNum" ] ),
                          int( main.params[ "TOPO" ][ "linkNum" ] ), 3 )
         verify( main )
@@ -857,7 +838,6 @@
         main.Cluster.active( 0 ).CLI.balanceMasters()
         time.sleep( float( main.params[ 'timers' ][ 'balanceMasterSleep' ] ) )
         verify( main )
-
         portsToDisable = [ [ "of:0000000000000001", 1 ], [ "of:0000000000000103", 1 ],
                            [ "of:0000000000000006", 1 ], [ "of:0000000000000103", 2 ],
                            [ "of:0000000000000101", 9 ], [ "of:0000000000000103", 3 ],
@@ -869,20 +849,13 @@
                            [ "of:0000000000000003", 3 ], [ "of:0000000000000102", 3 ],
                            [ "of:0000000000000004", 3 ], [ "of:0000000000000102", 5 ],
                            [ "of:0000000000000005", 3 ], [ "of:0000000000000102", 7 ] ]
-        for dpid, port in portsToDisable[ : -1 ]:
-            main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="disable" )
-        # To trigger sleep for link down discovery and topology check
-        lib.portstate( main, portsToDisable[ -1 ][ 0 ], portsToDisable[ -1 ][ 1 ], "disable",
-                       int( main.params[ "TOPO" ][ "switchNum" ] ),
-                       int( main.params[ "TOPO" ][ "linkNum" ] ) - len( portsToDisable ) )
-
+        lib.disablePortBatch( main, portsToDisable,
+                              int( main.params[ "TOPO" ][ "switchNum" ] ),
+                              int( main.params[ "TOPO" ][ "linkNum" ] ) - len( portsToDisable ) )
         verify( main )
-        for dpid, port in portsToDisable[ : -1 ]:
-            main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="enable" )
-        # To trigger sleep for link up discovery and topology check
-        lib.portstate( main, portsToDisable[ -1 ][ 0 ], portsToDisable[ -1 ][ 1 ], "enable",
-                       int( main.params[ "TOPO" ][ "switchNum" ] ),
-                       int( main.params[ "TOPO" ][ "linkNum" ] ) )
+        lib.enablePortBatch( main, portsToDisable,
+                             int( main.params[ "TOPO" ][ "switchNum" ] ),
+                             int( main.params[ "TOPO" ][ "linkNum" ] ) )
         lib.killOnos( main, [ 2, ], int( main.params[ "TOPO" ][ "switchNum" ] ),
                       int( main.params[ "TOPO" ][ "linkNum" ] ), 2 )
         verify( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index 0d7c6f8..f4b5481 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -88,23 +88,25 @@
     """
     from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
     # Verify connected hosts
-    main.step("Verify reachability of connected internal hosts")
     if ipv4:
         lib.verifyPing( main,
                         [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
-                        [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ] )
+                        [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
+                        stepMsg="Verify reachability of connected internal IPv4 hosts" )
     if ipv6:
         lib.verifyPing( main,
                         [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
                         [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
-                        ipv6=True, acceptableFailed=7 )
+                        ipv6=True, acceptableFailed=7,
+                        stepMsg="Verify reachability of connected internal IPv6 hosts" )
     # Verify disconnected hosts
     if disconnected:
-        main.step("Verify unreachability of disconnected internal hosts")
         if main.disconnectedIpv4Hosts:
-            lib.verifyPing( main, main.internalIpv4Hosts, main.disconnectedIpv4Hosts, expect=False )
+            lib.verifyPing( main, main.internalIpv4Hosts, main.disconnectedIpv4Hosts, expect=False,
+                            stepMsg="Verify unreachability of disconnected internal IPv4 hosts" )
         if main.disconnectedIpv6Hosts:
-            lib.verifyPing( main, main.internalIpv6Hosts, main.disconnectedIpv6Hosts, ipv6=True, expect=False )
+            lib.verifyPing( main, main.internalIpv6Hosts, main.disconnectedIpv6Hosts, ipv6=True, expect=False,
+                            stepMsg="Verify unreachability of disconnected internal IPv6 hosts" )
 
 def verifyPingExternal( main, ipv4=True, ipv6=True, disconnected=True ):
     """
@@ -113,39 +115,43 @@
     """
     from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
     # Verify connected hosts
-    main.step("Verify reachability from connected internal hosts to external hosts")
     if ipv4:
         lib.verifyPing( main,
                         [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
-                        [ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ] )
+                        [ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ],
+                        stepMsg="Verify reachability from connected internal IPv4 hosts to external IPv4 hosts" )
     if ipv6:
         lib.verifyPing( main,
                         [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
                         [ h for h in main.externalIpv6Hosts if h not in main.disconnectedExternalIpv6Hosts ],
-                        ipv6=True, acceptableFailed=7 )
+                        ipv6=True, acceptableFailed=7,
+                        stepMsg="Verify reachability from connected internal IPv6 hosts to external IPv6 hosts" )
     # Verify disconnected hosts
     if disconnected:
-        main.step("Verify unreachability of disconnected internal hosts to external hosts")
         # Disconnected internal to connected external
         if main.disconnectedIpv4Hosts:
             lib.verifyPing( main, main.disconnectedIpv4Hosts,
                             [ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ],
-                            expect=False )
+                            expect=False,
+                            stepMsg="Verify unreachability of disconnected internal IPv4 hosts to connected external IPv4 hosts" )
         if main.disconnectedIpv6Hosts:
             lib.verifyPing( main, main.disconnectedIpv6Hosts,
                             [ h for h in main.externalIpv6Hosts if h not in main.disconnectedExternalIpv6Hosts ],
-                            ipv6=True, expect=False )
+                            ipv6=True, expect=False,
+                            stepMsg="Verify unreachability of disconnected internal IPv6 hosts to connected external IPv6 hosts" )
         # Connected internal to disconnected external
         if main.disconnectedExternalIpv4Hosts:
             lib.verifyPing( main,
                             [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
                             main.disconnectedExternalIpv4Hosts,
-                            expect=False )
+                            expect=False,
+                            stepMsg="Verify unreachability of connected internal IPv4 hosts to disconnected external IPv4 hosts" )
         if main.disconnectedExternalIpv6Hosts:
             lib.verifyPing( main,
                             [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
                             main.disconnectedExternalIpv6Hosts,
-                            ipv6=True, expect=False )
+                            ipv6=True, expect=False,
+                            stepMsg="Verify unreachability of connected internal IPv6 hosts to disconnected external IPv6 hosts" )
 
 def verifyPing( main, ipv4=True, ipv6=True, disconnected=False, internal=True, external=True ):
     """
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 752296a..0daf074 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -588,6 +588,52 @@
                                  onfail="Link batch up failed" )
 
     @staticmethod
+    def disablePortBatch( main, ports, switches=None, links=None, sleep=None ):
+        """
+        Disable a list of switch ports using 'portstate' and verify ONOS can see the proper link change
+        ports: a list of ports to disable ex. [ [ "of:0000000000000001", 1 ] ]
+        switches, links: number of expected switches and links after link change, ex.: '4', '6'
+        """
+        if sleep is None:
+            sleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
+        else:
+            sleep = float( sleep )
+        main.step( "Disable a batch of ports" )
+        for dpid, port in ports:
+            main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="disable" )
+        main.log.info( "Waiting {} seconds for port down to be discovered".format( sleep ) )
+        time.sleep( sleep )
+        if switches and links:
+            result = main.Cluster.active( 0 ).CLI.checkStatus( numoswitch=switches,
+                                                               numolink=links )
+            utilities.assert_equals( expect=main.TRUE, actual=result,
+                                     onpass="Port down successful",
+                                     onfail="Port down failed" )
+
+    @staticmethod
+    def enablePortBatch( main, ports, switches, links, sleep=None ):
+        """
+        Enable a list of switch ports using 'portstate' and verify ONOS can see the proper link change
+        ports: a list of ports to enable ex. [ [ "of:0000000000000001", 1 ] ]
+        switches, links: number of expected switches and links after link change, ex.: '4', '6'
+        """
+        if sleep is None:
+            sleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
+        else:
+            sleep = float( sleep )
+        main.step( "Enable a batch of ports" )
+        for dpid, port in ports:
+            main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="enable" )
+        main.log.info( "Waiting {} seconds for port up to be discovered".format( sleep ) )
+        time.sleep( sleep )
+        if switches and links:
+            result = main.Cluster.active( 0 ).CLI.checkStatus( numoswitch=switches,
+                                                               numolink=links )
+            utilities.assert_equals( expect=main.TRUE, actual=result,
+                                     onpass="Port up successful",
+                                     onfail="Port up failed" )
+
+    @staticmethod
     def restoreLink( main, end1, end2, switches, links,
                      portUp=False, dpid1='', dpid2='', port1='', port2='' ):
         """
@@ -681,27 +727,6 @@
                                  onfail="Failed to recover switch?" )
 
     @staticmethod
-    def portstate( main, dpid, port, state, switches, links ):
-        """
-        Disable/enable a switch port using 'portstate' and verify ONOS can see the proper link change
-        Params:
-            dpid: dpid of the switch, ex.: 'of:0000000000000002'
-            port: port of the switch to disable/enable, ex.:'1'
-            state: disable or enable
-            switches, links: number of expected switches and links after link change, ex.: '4', '6'
-        """
-        main.step( "Port %s on %s:%s" % ( state, dpid, port ) )
-        main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
-        main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state=state )
-        main.log.info( "Waiting %s seconds for port %s to be discovered" % ( main.linkSleep, state ) )
-        time.sleep( main.linkSleep )
-        result = main.Cluster.active( 0 ).CLI.checkStatus( numoswitch=switches,
-                                                           numolink=links )
-        utilities.assert_equals( expect=main.TRUE, actual=result,
-                                 onpass="Port %s successful" % state,
-                                 onfail="Port %s failed" % state )
-
-    @staticmethod
     def cleanup( main, copyKarafLog=True, removeHostComponent=False ):
         """
         Stop Onos-cluster.
@@ -1047,11 +1072,13 @@
             main.topo
         except ( NameError, AttributeError ):
             main.topo = Topology()
+        main.step( "Verify {} multicast traffic".format( routeName ) )
         routeData = main.multicastConfig[ routeName ]
         srcs = main.mcastRoutes[ routeName ][ "src" ]
         dsts = main.mcastRoutes[ routeName ][ "dst" ]
         main.log.info( "Sending multicast traffic from {} to {}".format( [ routeData[ "src" ][ i ][ "host" ] for i in srcs ],
                                                                          [ routeData[ "dst" ][ i ][ "host" ] for i in dsts ] ) )
+        result = main.TRUE
         for src in srcs:
             srcEntry = routeData[ "src" ][ src ]
             for dst in dsts:
@@ -1079,17 +1106,22 @@
                                                                                 routeData[ "group" ], srcEntry[ "Ether" ] )
                 trafficResult = main.topo.sendScapyPackets( sender, receiver, pktFilter, pkt, sIface, dIface,
                                                             expectedResult, maxRetry, True, t3Cmd )
-                utilities.assert_equals( expect=main.TRUE,
-                                         actual=trafficResult,
-                                         onpass="{} to {}: Pass".format( srcEntry[ "host" ], dstEntry[ "host" ] ),
-                                         onfail="{} to {}: Fail".format( srcEntry[ "host" ], dstEntry[ "host" ] ) )
-                if skipOnFail and trafficResult != main.TRUE:
-                    Testcaselib.saveOnosDiagnostics( main )
-                    Testcaselib.cleanup( main, copyKarafLog=False )
-                    main.skipCase()
+                if not trafficResult:
+                    result = main.FALSE
+                    main.log.warn( "Scapy result from {} to {} is not as expected".format( srcEntry[ "host" ],
+                                                                                           dstEntry[ "host" ] ) )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=result,
+                                 onpass="Verify {} multicast traffic: Pass".format( routeName ),
+                                 onfail="Verify {} multicast traffic: Fail".format( routeName ) )
+        if skipOnFail and trafficResult != main.TRUE:
+            Testcaselib.saveOnosDiagnostics( main )
+            Testcaselib.cleanup( main, copyKarafLog=False )
+            main.skipCase()
 
     @staticmethod
-    def verifyPing( main, srcList, dstList, ipv6=False, expect=True, wait=1, acceptableFailed=0, skipOnFail=True ):
+    def verifyPing( main, srcList, dstList, ipv6=False, expect=True, wait=1,
+                    acceptableFailed=0, skipOnFail=True, stepMsg="Verify Ping" ):
         """
         Verify reachability from each host in srcList to each host in dstList
         """
@@ -1098,32 +1130,47 @@
             main.topo
         except ( NameError, AttributeError ):
             main.topo = Topology()
+        main.step( stepMsg )
         pingResult = main.topo.ping( srcList, dstList, ipv6, expect, wait, acceptableFailed, skipOnFail )
+        utilities.assert_equals( expect=main.TRUE,
+                                 actual=pingResult,
+                                 onpass="{}: Pass".format( stepMsg ),
+                                 onfail="{}: Fail".format( stepMsg ) )
         if not pingResult and skipOnFail:
             Testcaselib.saveOnosDiagnostics( main )
             Testcaselib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
             main.skipCase()
 
     @staticmethod
-    def verifyHostLocation( main, hostName, locations, ipv6=False, retry=0 ):
+    def verifyHostLocations( main, locationDict, retry=2 ):
         """
         Verify if the specified host is discovered by ONOS on the given locations
         Required:
-            hostName: name of the host. ex. "h1"
-            locations: expected locations of the host. ex. "of:0000000000000005/8"
-                       Could be a string or list
-        Optional:
-            ipv6: Use True for IPv6 only hosts
+            locationDict: a dictionary that maps host names to expected locations.
+                          locations could be a string or a list.
+                          ex. { "h1v4": ["of:0000000000000005/8"] }
         Returns:
             main.TRUE if host is discovered on all locations provided, otherwise main.FALSE
         """
-        main.log.info( "Verify host {} is discovered at {}".format( hostName, locations ) )
-        hostIp = main.Network.getIPAddress( hostName, proto='IPV6' if ipv6 else 'IPV4' )
-        result = utilities.retry( main.Cluster.active( 0 ).CLI.verifyHostLocation,
-                                  main.FALSE,
-                                  args=( hostIp, locations ),
-                                  attempts=retry + 1,
-                                  sleep=10 )
+        main.step( "Verify locations of hosts {}".format( locationDict.keys() ) )
+        result = main.TRUE
+        for hostName, locations in locationDict.items():
+            main.log.info( "Verify host {} is discovered at {}".format( hostName, locations ) )
+            hostIp = main.Network.getIPAddress( hostName, proto='IPV4' )
+            if not hostIp:
+                hostIp = main.Network.getIPAddress( hostName, proto='IPV6' )
+            if not hostIp:
+                main.log.warn( "Failed to find IP address for host {}, skipping location verification".format( hostName ) )
+                result = main.FALSE
+                continue
+            locationResult = utilities.retry( main.Cluster.active( 0 ).CLI.verifyHostLocation,
+                                              main.FALSE,
+                                              args=( hostIp, locations ),
+                                              attempts=retry + 1,
+                                              sleep=10 )
+            if not locationResult:
+                result = main.FALSE
+                main.log.warn( "location verification for host {} failed".format( hostName ) )
         utilities.assert_equals( expect=main.TRUE, actual=result,
-                                 onpass="Location verification for Host {} passed".format( hostName ),
-                                 onfail="Location verification for Host {} failed".format( hostName ) )
+                                 onpass="Location verification passed",
+                                 onfail="Location verification failed" )
diff --git a/TestON/tests/dependencies/topology.py b/TestON/tests/dependencies/topology.py
index e4f195b..4d3bafb 100644
--- a/TestON/tests/dependencies/topology.py
+++ b/TestON/tests/dependencies/topology.py
@@ -256,7 +256,6 @@
                 main.log.info( "Starting CLI on host {}".format( src ) )
                 hostHandle.startHostCli()
             srcIpList[ src ] = main.Network.getIPAddress( src, proto='IPV6' if ipv6 else 'IPV4' )
-
         unexpectedPings = []
         for dst in dstList:
             dstIp = main.Network.getIPAddress( dst, proto='IPV6' if ipv6 else 'IPV4' )
@@ -284,11 +283,7 @@
                 srcIp, dstIp = thread.name.split( "-" )
                 if expect and not thread.result or not expect and thread.result:
                     unexpectedPings.append( [ srcIp, dstIp, "fail" if expect else "pass" ] )
-
-        utilities.assert_equals( expect=[],
-                                 actual=unexpectedPings,
-                                 onpass="Pings from {} to {} all as expected".format( srcList, dstList ),
-                                 onfail="Unexpected pings: {}".format( unexpectedPings ) )
+        main.log.info( "Unexpected pings: {}".format( unexpectedPings ) )
         if collectT3:
             for unexpectedPing in unexpectedPings:
                 if unexpectedPing[ 2 ] == "no IP":