Several fixes to SRMulticast
- Add host discovery and location verifications
- Flush arp entries before host discovery
- Update mcast join and delete commands
Change-Id: If5c6d85e7441010c63ebf45f333552e24a07352c
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 59e1441..c5ee51d 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -605,6 +605,8 @@
main.log.warn( "No IP addresses configured on host {}, skipping discovery".format( host ) )
discoveryResult = main.FALSE
if cmd:
+ self.handle.sendline( "{} ip neigh flush all".format( host ) )
+ self.handle.expect( "mininet>", timeout=wait + 1 )
self.handle.sendline( cmd )
self.handle.expect( "mininet>", timeout=wait + 1 )
return discoveryResult
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 7af7e4f..d806da6 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1179,9 +1179,10 @@
for loc in locations:
discovered = False
for locDiscovered in locationsDiscovered:
- if loc in locDiscovered:
- discovered = True
+ locToMatch = locDiscovered if "/" in loc else locDiscovered.split( "/" )[0]
+ if loc == locToMatch:
main.log.debug( "Host {} discovered with location {}".format( hostIp, loc ) )
+ discovered = True
break
if discovered:
locationsDiscovered.remove( locDiscovered )
@@ -6253,7 +6254,7 @@
Create a multicast route by calling 'mcast-host-join' command
sAddr: we can provide * for ASM or a specific address for SSM
gAddr: specifies multicast group address
- srcs: a list of the source connect points e.g. ["of:0000000000000003/12"]
+ srcs: a list of HostId of the sources e.g. ["00:AA:00:00:00:01/None"]
sinks: a list of HostId of the sinks e.g. ["00:AA:00:00:01:05/40"]
Returns main.TRUE if mcast route is added; Otherwise main.FALSE
"""
@@ -6330,12 +6331,52 @@
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+ def mcastSinkDelete( self, sAddr, gAddr, sink=None ):
+ """
+ Delete multicast sink(s) by calling 'mcast-sink-delete' command
+ sAddr: we can provide * for ASM or a specific address for SSM
+ gAddr: specifies multicast group address
+ host: HostId of the sink e.g. "00:AA:00:00:01:05/40",
+ will delete the route if not specified
+ Returns main.TRUE if the mcast sink is deleted; Otherwise main.FALSE
+ """
+ try:
+ cmdStr = "mcast-sink-delete"
+ cmdStr += " -sAddr " + str( sAddr )
+ cmdStr += " -gAddr " + str( gAddr )
+ if sink:
+ cmdStr += " -s " + str( sink )
+ handle = self.sendline( cmdStr )
+ assert handle is not None, "Error in sendline"
+ assert "Command not found:" not in handle, handle
+ assert "Unsupported command:" not in handle, handle
+ assert "Error executing command" not in handle, handle
+ if "Updated the mcast route" in handle:
+ return main.TRUE
+ elif "Deleted the mcast route" in handle:
+ return main.TRUE
+ else:
+ return main.FALSE
+ except AssertionError:
+ main.log.exception( "" )
+ return None
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
def mcastSourceDelete( self, sAddr, gAddr, srcs=None ):
"""
Delete multicast src(s) by calling 'mcast-source-delete' command
sAddr: we can provide * for ASM or a specific address for SSM
gAddr: specifies multicast group address
- srcs: a list of connect points of the sources e.g. ["00:AA:00:00:01:05/40"],
+ srcs: a list of host IDs of the sources e.g. ["00:AA:00:00:01:05/40"],
will delete the route if not specified
Returns main.TRUE if mcast sink is deleted; Otherwise main.FALSE
"""
diff --git a/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.py b/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.py
index 5849a6f..144b4cf 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.py
@@ -307,7 +307,7 @@
main.mcastRoutes = { "ipv4": { "src": [ 0 ], "dst": [ 0, 1, 2 ] }, "ipv6": { "src": [ 0 ], "dst": [ 0 ] } }
setupTest( main, test_idx=202, onosNodes=3 )
verifyMcastRoutes( main )
- verifySwitchDown( main, "leaf2", 10, { "ipv4": False, "ipv6": False } )
+ verifySwitchDown( main, "leaf2", 10, { "ipv4": False, "ipv6": False }, [ "h4v4" ] )
verifyMcastRemoval( main, removeDHT1=False )
lib.cleanup( main, copyKarafLog=False )
@@ -327,7 +327,7 @@
main.mcastRoutes = { "ipv4": { "src": [ 0 ], "dst": [ 0, 1, 2 ] }, "ipv6": { "src": [ 0 ], "dst": [ 0 ] } }
setupTest( main, test_idx=203, onosNodes=3 )
verifyMcastRoutes( main )
- verifySwitchDown( main, "leaf5", 10 )
+ verifySwitchDown( main, "leaf5", 10, hostsToDiscover=[ "h10v4" ] )
verifyMcastRemoval( main, removeDHT1=False )
lib.cleanup( main, copyKarafLog=False )
@@ -347,7 +347,7 @@
main.mcastRoutes = { "ipv4": { "src": [ 0 ], "dst": [ 0, 1, 2 ] }, "ipv6": { "src": [ 0 ], "dst": [ 0 ] } }
setupTest( main, test_idx=204, onosNodes=3 )
verifyMcastRoutes( main )
- verifySwitchDown( main, "leaf4", 10, { "ipv4": [ True, False, True ], "ipv6": True } )
+ verifySwitchDown( main, "leaf4", 10, { "ipv4": [ True, False, True ], "ipv6": True }, [ "h8v4", "h10v4" ] )
verifyMcastRemoval( main, removeDHT1=False )
lib.cleanup( main, copyKarafLog=False )
@@ -367,7 +367,7 @@
main.mcastRoutes = { "ipv4": { "src": [ 0 ], "dst": [ 0, 1, 2 ] }, "ipv6": { "src": [ 0 ], "dst": [ 0 ] } }
setupTest( main, test_idx=205, onosNodes=3 )
verifyMcastRoutes( main )
- verifySwitchDown( main, [ "leaf1", "leaf3", "leaf4", "leaf5" ], 32, { "ipv4": [ True, False, False ], "ipv6": False } )
+ verifySwitchDown( main, [ "leaf1", "leaf3", "leaf4", "leaf5" ], 32, { "ipv4": [ True, False, False ], "ipv6": False }, [ "h4v4", "h8v4", "h10v4", "h1v6"] )
verifyMcastRemoval( main, removeDHT1=False )
lib.cleanup( main, copyKarafLog=False )
@@ -411,14 +411,13 @@
main.mcastRoutes = { "ipv4": { "src": [ 0 ], "dst": [ 0, 1, 2 ] }, "ipv6": { "src": [ 0 ], "dst": [ 0 ] } }
setupTest( main, test_idx=401, onosNodes=3 )
verifyMcastRoutes( main )
- #TODO: Verify host has both locations
# Verify killing one link of dual-homed host h4
- verifyLinkDown( main, [ "leaf2", "h4v4" ], 0 )
- verifyLinkDown( main, [ "leaf3", "h4v4" ], 0 )
+ verifyPortDown( main, "of:0000000000000002", 10, hostsToDiscover=[ "h4v4" ], hostLocations={ "h4v4": ["of:0000000000000002/10", "of:0000000000000003/10"] } )
+ verifyPortDown( main, "of:0000000000000003", 10, hostsToDiscover=[ "h4v4" ], hostLocations={ "h4v4": ["of:0000000000000002/10", "of:0000000000000003/10"] } )
# Verify killing one link of dual-homed host h10
- verifyLinkDown( main, [ "leaf4", "h10v4" ], 0 )
- verifyLinkDown( main, [ "leaf5", "h10v4" ], 0 )
- verifySwitchDown( main, "leaf3", 10 )
+ verifyPortDown( main, "of:0000000000000004", 11, hostsToDiscover=[ "h10v4" ], hostLocations={ "h10v4": ["of:0000000000000004/11", "of:0000000000000005/10"] } )
+ verifyPortDown( main, "of:0000000000000005", 10, hostsToDiscover=[ "h10v4" ], hostLocations={ "h10v4": ["of:0000000000000004/11", "of:0000000000000005/10"] } )
+ verifySwitchDown( main, "leaf3", 10, hostsToDiscover=[ "h4v4" ] )
verifyMcastRemoval( main, removeDHT1=False )
lib.cleanup( main, copyKarafLog=False )
@@ -464,11 +463,11 @@
setupTest( main, test_idx=403, onosNodes=3 )
verifyMcastRoutes( main )
# Verify killing one link of dual-homed host h4
- verifyLinkDown( main, [ "leaf2", "h4v4" ], 0 )
- verifyLinkDown( main, [ "leaf3", "h4v4" ], 0 )
+ verifyPortDown( main, "of:0000000000000002", 10, hostsToDiscover=[ "h4v4" ], hostLocations={ "h4v4": ["of:0000000000000002/10", "of:0000000000000003/10"] } )
+ verifyPortDown( main, "of:0000000000000003", 10, hostsToDiscover=[ "h4v4" ], hostLocations={ "h4v4": ["of:0000000000000002/10", "of:0000000000000003/10"] } )
# Verify killing one link of dual-homed host h10
- verifyLinkDown( main, [ "leaf4", "h10v4" ], 0 )
- verifyLinkDown( main, [ "leaf5", "h10v4" ], 0 )
+ verifyPortDown( main, "of:0000000000000004", 11, hostsToDiscover=[ "h10v4" ], hostLocations={ "h10v4": ["of:0000000000000004/11", "of:0000000000000005/10"] } )
+ verifyPortDown( main, "of:0000000000000005", 10, hostsToDiscover=[ "h10v4" ], hostLocations={ "h10v4": ["of:0000000000000004/11", "of:0000000000000005/10"] } )
verifyLinkDown( main, [ [ "leaf3", "spine101" ], [ "leaf3", "spine102" ] ], 8 )
verifyMcastRemoval( main, removeDHT1=False )
lib.cleanup( main, copyKarafLog=False )
@@ -494,11 +493,11 @@
setupTest( main, test_idx=404, onosNodes=3 )
verifyMcastRoutes( main )
# Verify killing one link of dual-homed host h4
- verifyLinkDown( main, [ "leaf2", "h4v4" ], 0 )
- verifyLinkDown( main, [ "leaf3", "h4v4" ], 0 )
+ verifyPortDown( main, "of:0000000000000002", 10, hostsToDiscover=[ "h4v4" ], hostLocations={ "h4v4": ["of:0000000000000002/10", "of:0000000000000003/10"] } )
+ verifyPortDown( main, "of:0000000000000003", 10, hostsToDiscover=[ "h4v4" ], hostLocations={ "h4v4": ["of:0000000000000002/10", "of:0000000000000003/10"] } )
# Verify killing one link of dual-homed host h10
- verifyLinkDown( main, [ "leaf4", "h10v4" ], 0 )
- verifyLinkDown( main, [ "leaf5", "h10v4" ], 0 )
+ verifyPortDown( main, "of:0000000000000004", 11, hostsToDiscover=[ "h10v4" ], hostLocations={ "h10v4": ["of:0000000000000004/11", "of:0000000000000005/10"] } )
+ verifyPortDown( main, "of:0000000000000005", 10, hostsToDiscover=[ "h10v4" ], hostLocations={ "h10v4": ["of:0000000000000004/11", "of:0000000000000005/10"] } )
verifyLinkDown( main, [ [ "leaf3", "spine101" ], [ "leaf2", "spine102" ] ], 8 )
verifyMcastRemoval( main, removeDHT1=False )
lib.cleanup( main, copyKarafLog=False )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/SRMulticastTest.py b/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/SRMulticastTest.py
index 30b7e92..f598f96 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/SRMulticastTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/SRMulticastTest.py
@@ -75,7 +75,7 @@
src = main.mcastRoutes[ routeName ][ "src" ]
dst = main.mcastRoutes[ routeName ][ "dst" ]
main.Cluster.active( 0 ).CLI.mcastHostJoin( routeData[ "src" ][ src[ 0 ] ][ "ip" ], routeData[ "group" ],
- [ routeData[ "src" ][ i ][ "port" ] for i in src ],
+ [ routeData[ "src" ][ i ][ "id" ] for i in src ],
[ routeData[ "dst" ][ i ][ "id" ] for i in dst ] )
time.sleep( float( main.params[ "timers" ][ "mcastSleep" ] ) )
@@ -85,7 +85,7 @@
"""
routeData = main.multicastConfig[ routeName ]
main.step( "Verify removal of {} route".format( routeName ) )
- main.Cluster.active( 0 ).CLI.mcastHostDelete( routeData[ "src" ][ 0 ][ "ip" ], routeData[ "group" ] )
+ main.Cluster.active( 0 ).CLI.mcastSinkDelete( routeData[ "src" ][ 0 ][ "ip" ], routeData[ "group" ] )
# TODO: verify the deletion
def verifyMcastSinkRemoval( main, routeName, sinkIndex, expect ):
@@ -96,7 +96,7 @@
routeData = main.multicastConfig[ routeName ]
sinkId = routeData[ "dst" ][ sinkIndex ][ "id" ]
main.step( "Verify removal of {} sink {}".format( routeName, sinkId ) )
- main.Cluster.active( 0 ).CLI.mcastHostDelete( routeData[ "src" ][ 0 ][ "ip" ], routeData[ "group" ], sinkId )
+ main.Cluster.active( 0 ).CLI.mcastSinkDelete( routeData[ "src" ][ 0 ][ "ip" ], routeData[ "group" ], sinkId )
time.sleep( float( main.params[ "timers" ][ "mcastSleep" ] ) )
lib.verifyMulticastTraffic( main, routeName, expect )
@@ -106,9 +106,9 @@
"""
from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
routeData = main.multicastConfig[ routeName ]
- sourcePort = [ routeData[ "src" ][ sourceIndex ][ "port" ] ]
- main.step( "Verify removal of {} source {}".format( routeName, sourcePort ) )
- main.Cluster.active( 0 ).CLI.mcastSourceDelete( routeData[ "src" ][ 0 ][ "ip" ], routeData[ "group" ], sourcePort )
+ sourceId = [ routeData[ "src" ][ sourceIndex ][ "id" ] ]
+ main.step( "Verify removal of {} source {}".format( routeName, sourceId ) )
+ main.Cluster.active( 0 ).CLI.mcastSourceDelete( routeData[ "src" ][ 0 ][ "ip" ], routeData[ "group" ], sourceId )
time.sleep( float( main.params[ "timers" ][ "mcastSleep" ] ) )
lib.verifyMulticastTraffic( main, routeName, expect )
@@ -126,7 +126,7 @@
verifyMcastSinkRemoval( main, "ipv4", 1, [ True, False, False ] )
verifyMcastSourceRemoval( main, "ipv4", 0, False )
-def verifyLinkDown( main, link, affectedLinkNum, expectList={ "ipv4": True, "ipv6": True } ):
+def verifyLinkDown( main, link, affectedLinkNum, expectList={ "ipv4": True, "ipv6": True }, hostsToDiscover=[], hostLocations={} ):
"""
Kill a batch of links and verify traffic
Restore the links and verify traffic
@@ -139,10 +139,34 @@
lib.verifyMulticastTraffic( main, routeName, expectList[ routeName ] )
# Restore the link(s)
lib.restoreLinkBatch( main, link, int( main.params[ "TOPO" ][ "linkNum" ] ), int( main.params[ "TOPO" ][ "switchNum" ] ) )
+ if hostsToDiscover:
+ main.Network.discoverHosts( hostList=hostsToDiscover )
+ for host, loc in hostLocations.items():
+ lib.verifyHostLocation( main, host, loc, retry=5 )
for routeName in expectList.keys():
lib.verifyMulticastTraffic( main, routeName, True )
-def verifySwitchDown( main, switchName, affectedLinkNum, expectList={ "ipv4": True, "ipv6": True } ):
+def verifyPortDown( main, dpid, port, expectList={ "ipv4": True, "ipv6": True }, hostsToDiscover=[], hostLocations={} ):
+ """
+ Disable a port and verify traffic
+ Reenable the port and verify traffic
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ main.step( "Disable port {}/{}".format( dpid, port ) )
+ main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="disable" )
+ time.sleep( 10 )
+ for routeName in expectList.keys():
+ lib.verifyMulticastTraffic( main, routeName, expectList[ routeName ] )
+ # Restore the link(s)
+ main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="enable" )
+ if hostsToDiscover:
+ main.Network.discoverHosts( hostList=hostsToDiscover )
+ for host, loc in hostLocations.items():
+ lib.verifyHostLocation( main, host, loc, retry=5 )
+ for routeName in expectList.keys():
+ lib.verifyMulticastTraffic( main, routeName, True )
+
+def verifySwitchDown( main, switchName, affectedLinkNum, expectList={ "ipv4": True, "ipv6": True }, hostsToDiscover=[], hostLocations={} ):
"""
Kill a batch of switches and verify traffic
Recover the swithces and verify traffic
@@ -154,7 +178,9 @@
for routeName in expectList.keys():
lib.verifyMulticastTraffic( main, routeName, expectList[ routeName ] )
# Recover the switch(es)
- lib.recoverSwitch( main, switchName, int( main.params[ "TOPO" ][ "switchNum" ] ), int( main.params[ "TOPO" ][ "linkNum" ] ) )
+ lib.recoverSwitch( main, switchName, int( main.params[ "TOPO" ][ "switchNum" ] ), int( main.params[ "TOPO" ][ "linkNum" ] ), True if hostsToDiscover else False, hostsToDiscover )
+ for host, loc in hostLocations.items():
+ lib.verifyHostLocation( host, loc, retry=5 )
for routeName in expectList.keys():
lib.verifyMulticastTraffic( main, routeName, True )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/multicast/common.multicastConfig b/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/multicast/common.multicastConfig
index 842d540..4d06609 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/multicast/common.multicastConfig
+++ b/TestON/tests/USECASE/SegmentRouting/SRMulticast/dependencies/multicast/common.multicastConfig
@@ -6,6 +6,7 @@
{
"host": "h3v4",
"ip": "10.2.0.1",
+ "id": "00:AA:00:00:00:02/None",
"port": "of:0000000000000002/9",
"interface": "h3v4-eth0",
"Ether": "01:00:5e:02:00:01",
@@ -40,6 +41,7 @@
{
"host": "h3v6",
"ip": "1002::3fe",
+ "id": "00:BB:00:00:00:02/None",
"port": "of:0000000000000002/6",
"interface": "h3v6-eth0",
"Ether": "33:33:00:00:03:fe",
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index b8ad87f..95e3132 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -573,7 +573,7 @@
main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
main.log.info(
- "Waiting %s seconds for links down to be discovered" % main.linkSleep )
+ "Waiting %s seconds for links up to be discovered" % main.linkSleep )
time.sleep( main.linkSleep )
topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
@@ -1036,7 +1036,7 @@
main.log.debug( host.hostMac )
@staticmethod
- def verifyMulticastTraffic( main, routeName, expect, skipOnFail=True, maxRetry=0 ):
+ def verifyMulticastTraffic( main, routeName, expect, skipOnFail=True, maxRetry=1 ):
"""
Verify multicast traffic using scapy
"""
@@ -1103,7 +1103,7 @@
main.skipCase()
@staticmethod
- def verifyHostLocation( main, hostName, locations, ipv6=False ):
+ def verifyHostLocation( main, hostName, locations, ipv6=False, retry=0 ):
"""
Verify if the specified host is discovered by ONOS on the given locations
Required:
@@ -1115,7 +1115,13 @@
Returns:
main.TRUE if host is discovered on all locations provided, otherwise main.FALSE
"""
- main.step( "Verify host {} is discovered at {}".format( hostName, locations ) )
+ main.log.info( "Verify host {} is discovered at {}".format( hostName, locations ) )
hostIp = main.Network.getIPAddress( hostName, proto='IPV6' if ipv6 else 'IPV4' )
- result = main.Cluster.active( 0 ).CLI.verifyHostLocation( hostIp, locations )
- return result
+ result = utilities.retry( main.Cluster.active( 0 ).CLI.verifyHostLocation,
+ main.FALSE,
+ args=( hostIp, locations ),
+ attempts=retry + 1,
+ sleep=10 )
+ utilities.assert_equals( expect=main.TRUE, actual=result,
+ onpass="Location verification for Host {} passed".format( hostName ),
+ onfail="Location verification for Host {} failed".format( hostName ) )