[ONOS-7635][ONOS-7636] Support for complex scenarios in SRRouting test
- New ping function to execute pings in parallel
- Collect t3 output on ping failures
- Allow expected ping failures
Change-Id: I4492a89bc4c0a581ff2e35bcc1896ddd5ea64a18
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 08d8505..bd00623 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -593,10 +593,9 @@
pingResponse += str( str( host ) + " -> " )
for temp in dstList:
failedPings = 0
- if ipv6:
- pingCmd = str( host ) + cmd + str( self.getIPAddress( temp, proto='IPv6' ) )
- else:
- pingCmd = str( host ) + cmd + str( temp )
+ dstIP = self.getIPAddress( temp, proto='IPV6' if ipv6 else 'IPV4' )
+ assert dstIP, "Not able to get IP address of host {}".format( temp )
+ pingCmd = str( host ) + cmd + str( dstIP )
while failedPings <= acceptableFailed:
main.log.debug( "Pinging from " + str( host ) + " to " + str( temp ) )
self.handle.sendline( pingCmd )
@@ -616,7 +615,9 @@
pingResponse += "\n"
main.log.info( pingResponse + "Failed pings: " + str( failedPingsTotal ) )
return isReachable
-
+ except AssertionError:
+ main.log.exception( "" )
+ return main.FALSE
except pexpect.TIMEOUT:
main.log.exception( self.name + ": TIMEOUT exception" )
response = self.handle.before
@@ -750,6 +751,10 @@
'\*\*\* Unknown command: ' + pingCmd,
pexpect.TIMEOUT ],
timeout=wait + 1 )
+ # For some reason we need to send something
+ # Otherwise ping results won't be read by handle
+ self.handle.sendline( "" )
+ self.handle.expect( self.hostPrompt )
if i == 0:
response = self.handle.before
if not re.search( ',\s0\%\spacket\sloss', response ):
@@ -1268,7 +1273,7 @@
else:
main.log.error( "Connection failed to the host" )
- def getIPAddress( self, host , proto='IPV4' ):
+ def getIPAddress( self, host, proto='IPV4' ):
"""
Verifies the host's ip configured or not."""
if self.handle:
@@ -1288,7 +1293,7 @@
pattern = ''
if proto == 'IPV4':
- pattern = "inet\saddr:(\d+\.\d+\.\d+\.\d+)"
+ pattern = "inet\saddr:(\d+\.\d+\.\d+\.\d+)\s\sBcast"
else:
pattern = "inet6\saddr:\s([\w,:]*)/\d+\sScope:Global"
ipAddressSearch = re.search( pattern, response )
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 6342efc..76cc415 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -6317,3 +6317,107 @@
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()
+
+ def netcfg( self, jsonFormat=True, args="" ):
+ """
+ Run netcfg cli command with given args
+ """
+ try:
+ cmdStr = "netcfg"
+ if jsonFormat:
+ cmdStr = cmdStr + " -j"
+ if args:
+ cmdStr = cmdStr + " " + str( args )
+ handle = self.sendline( cmdStr )
+ assert handle is not None, "Error in sendline"
+ assert "Command not found:" not in handle, handle
+ assert "Unsupported command:" not in handle, handle
+ assert "Error executing command" not in handle, handle
+ return handle
+ except AssertionError:
+ main.log.exception( "" )
+ return None
+ except TypeError:
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
+ def composeT3Command( self, sAddr, dAddr, ipv6=False, verbose=True ):
+ """
+ Compose and return t3-troubleshoot cli command for given source and destination addresses
+ Options:
+ sAddr: IP address of the source host
+ dAddr: IP address of the destination host
+ """
+ try:
+ # Collect information of both hosts from onos
+ hosts = self.hosts()
+ hosts = json.loads( hosts )
+ sHost = None
+ dHost = None
+ for host in hosts:
+ if sAddr in host[ "ipAddresses" ]:
+ sHost = host
+ elif dAddr in host[ "ipAddresses" ]:
+ dHost = host
+ if sHost and dHost:
+ break
+ assert sHost, "Not able to find host with IP {}".format( sAddr )
+ assert dHost, "Not able to find host with IP {}".format( dAddr )
+ cmdStr = "t3-troubleshoot"
+ if verbose:
+ cmdStr += " -vv"
+ cmdStr += " -s " + str( sAddr )
+ # TODO: collect t3 for all locations of source host?
+ cmdStr += " -sp " + str( sHost[ "locations" ][ 0 ][ "elementId" ] ) + "/" + str( sHost[ "locations" ][ 0 ][ "port" ] )
+ cmdStr += " -sm " + str( sHost[ "mac" ] )
+ cmdStr += " -d " + str( dAddr )
+ netcfg = self.netcfg( args="devices {}".format( sHost[ "locations" ][ 0 ][ "elementId" ] ) )
+ netcfg = json.loads( netcfg )
+ assert netcfg, "Failed to get netcfg"
+ cmdStr += " -dm " + str( netcfg[ "segmentrouting" ][ "routerMac" ] )
+ if ipv6:
+ cmdStr += " -et ipv6"
+ return cmdStr
+ except AssertionError:
+ main.log.exception( "" )
+ return None
+ except ( KeyError, TypeError ):
+ main.log.exception( self.name + ": Object not as expected" )
+ return None
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
+
+ def t3( self, sAddr, dAddr, ipv6=False ):
+ """
+ Run t3-troubleshoot cli command for given source and destination addresses
+ Options:
+ sAddr: IP address of the source host
+ dAddr: IP address of the destination host
+ """
+ try:
+ cmdStr = self.composeT3Command( sAddr, dAddr, ipv6 )
+ handle = self.sendline( cmdStr )
+ assert handle is not None, "Error in sendline"
+ assert "Command not found:" not in handle, handle
+ assert "Unsupported command:" not in handle, handle
+ assert "Error executing command" not in handle, handle
+ assert "Tracing packet" in handle
+ return handle
+ except AssertionError:
+ main.log.exception( "" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanAndExit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanAndExit()
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo
index a672a8e..7b188e2 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
index 5c4cc02..ad2c2b0 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/SRDhcprelay.topo b/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/SRDhcprelay.topo
index 01316b6..dbb415d 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/SRDhcprelay.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/SRDhcprelay.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.topo b/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.topo
index 5c4cc02..ad2c2b0 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/SRDynamicConf.topo b/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/SRDynamicConf.topo
index a672a8e..7b188e2 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/SRDynamicConf.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/SRDynamicConf.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
index 5c4cc02..ad2c2b0 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo
index 5c4cc02..ad2c2b0 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.topo b/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.topo
index 1c3f8a6..b5da976 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
index 28ecb9a..9ee92d6 100755
--- a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
index e8199bd..e19bd34 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
@@ -21,7 +21,7 @@
<ENV>
<cellName>productionCell</cellName>
- <cellApps>drivers,openflow,segmentrouting,fpm,dhcprelay,netcfghostprovider,routeradvertisement</cellApps>
+ <cellApps>drivers,openflow,segmentrouting,fpm,dhcprelay,netcfghostprovider,routeradvertisement,t3</cellApps>
</ENV>
<GIT>
@@ -39,8 +39,17 @@
<OnosDiscovery>30</OnosDiscovery>
<loadNetcfgSleep>5</loadNetcfgSleep>
<startMininetSleep>25</startMininetSleep>
- <dhcpSleep>60</dhcpSleep>
+ <dhcpSleep>30</dhcpSleep>
<balanceMasterSleep>10</balanceMasterSleep>
</timers>
+ <TOPO>
+ <internalIpv4Hosts>h1v4,h2v4,h3v4,h4v4,h5v4,h6v4,h7v4,h8v4,h9v4,h10v4,h11v4,h12v4,h13v4</internalIpv4Hosts>
+ <internalIpv6Hosts>h1v6,h2v6,h3v6,h4v6,h5v6,h6v6,h7v6,h8v6,h9v6,h10v6,h11v6,h12v6,h13v6</internalIpv6Hosts>
+ <externalIpv4Hosts>rh1v4,rh2v4</externalIpv4Hosts>
+ <externalIpv6Hosts>rh1v6,rh11v6,rh2v6,rh22v6</externalIpv6Hosts>
+ <switchNum>10</switchNum>
+ <linkNum>48</linkNum>
+ </TOPO>
+
</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index 07f1260..5f4157f 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -738,3 +738,34 @@
staticRouteConfigure=True,
nodeFailure=True )
+ def CASE606( self, main ):
+ """
+ Drop SPINE-1 and test connectivity
+ Drop paired leaf and test connectivity (expect some failures)
+ Bring up SPINE-1 and test connectivity (still expect some failures)
+ Bring up the paired leaf and test connectivity
+ Repeat above with SPINE-2 and a different paired leaf
+ """
+ import time
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import *
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ main.case( "Drop spine and paired leaf" )
+ setupTest( main, test_idx=606, onosNodes=3 )
+ main.disconnectedIpv4Hosts = []
+ main.disconnectedIpv6Hosts = []
+ verifyPing( main )
+ lib.killSwitch( main, "spine101", int( main.params[ "TOPO" ][ "switchNum" ] ) - 1, int( main.params[ "TOPO" ][ "linkNum" ] ) - 18 )
+ verifyPing( main )
+ lib.killSwitch( main, "leaf2", int( main.params[ "TOPO" ][ "switchNum" ] ) - 2, int( main.params[ "TOPO" ][ "linkNum" ] ) - 24 )
+ lib.killSwitch( main, "leaf3", int( main.params[ "TOPO" ][ "switchNum" ] ) - 3, int( main.params[ "TOPO" ][ "linkNum" ] ) - 28 )
+ main.disconnectedIpv4Hosts = [ "h3v4", "h4v4", "h5v4", "h6v4", "h7v4" ]
+ main.disconnectedIpv6Hosts = [ "h3v6", "h4v6", "h5v6", "h6v6", "h7v6" ]
+ verifyPing( main )
+ lib.recoverSwitch( main, "spine101", int( main.params[ "TOPO" ][ "switchNum" ] ) - 2, int( main.params[ "TOPO" ][ "linkNum" ] ) - 18 )
+ verifyPing( main )
+ lib.recoverSwitch( main, "leaf3", int( main.params[ "TOPO" ][ "switchNum" ] ) - 1, int( main.params[ "TOPO" ][ "linkNum" ] ) - 10 )
+ lib.recoverSwitch( main, "leaf2", int( main.params[ "TOPO" ][ "switchNum" ] ), int( main.params[ "TOPO" ][ "linkNum" ] ) )
+ main.disconnectedIpv4Hosts = []
+ main.disconnectedIpv6Hosts = []
+ verifyPing( main )
+ lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo
index 01316b6..dbb415d 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index 68533bb..a9a24bb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -154,3 +154,97 @@
run.checkFlowsGroupsFromFile( main )
# ping hosts
run.pingAll( main, 'CASE%03d' % test_idx, False, acceptableFailed=5, basedOnIp=True, skipOnFail=True )
+
+
+def setupTest( main, test_idx, onosNodes ):
+ """
+ SRRouting test setup
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ skipPackage = False
+ init = False
+ if not hasattr( main, 'apps' ):
+ init = True
+ run.initTest( main )
+ # Skip onos packaging if the cluster size stays the same
+ if not init and onosNodes == main.Cluster.numCtrls:
+ skipPackage = True
+
+ main.internalIpv4Hosts = main.params[ 'TOPO' ][ 'internalIpv4Hosts' ].split( ',' )
+ main.internalIpv6Hosts = main.params[ 'TOPO' ][ 'internalIpv6Hosts' ].split( ',' )
+ main.externalIpv4Hosts = main.params[ 'TOPO' ][ 'externalIpv4Hosts' ].split( ',' )
+ main.externalIpv6Hosts = main.params[ 'TOPO' ][ 'externalIpv6Hosts' ].split( ',' )
+ main.resultFileName = "CASE%03d" % test_idx
+ main.Cluster.setRunningNode( onosNodes )
+ lib.installOnos( main, skipPackage=skipPackage, cliSleep=5 )
+ # Load configuration files
+ main.step( "Load configurations" )
+ main.cfgName = "TEST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1_external=1"
+ lib.loadJson( main )
+ time.sleep( float( main.params[ "timers" ][ "loadNetcfgSleep" ] ) )
+
+ if hasattr( main, "Mininet1" ):
+ # Run the test with Mininet
+ mininet_args = " --dhcp=1 --routers=1 --ipv6=1 --ipv4=1"
+ lib.startMininet( main, main.params[ "DEPENDENCY" ][ "topology" ], args=mininet_args )
+ time.sleep( float( main.params[ "timers" ][ "startMininetSleep" ] ) )
+ else:
+ # Run the test with physical devices
+ lib.connectToPhysicalNetwork( main, self.switchNames )
+ # Check if the devices are up
+ lib.checkDevices( main, switches=len( self.switchNames ) )
+
+ # wait some time for onos to install the rules!
+ time.sleep( float( main.params[ "timers" ][ "startMininetSleep" ] ) )
+ time.sleep( float( main.params[ "timers" ][ "dhcpSleep" ] ) )
+
+def verifyPingInternal( main, verifyDisconnected=True ):
+ """
+ Verify all connected internal hosts are able to reach each other,
+ and disconnected internal hosts cannot reach any other internal host
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ # Verify connected hosts
+ main.step("Verify reachability of connected internal hosts")
+ lib.verifyPing( main,
+ [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
+ [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ] )
+ lib.verifyPing( main,
+ [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
+ [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
+ ipv6=True, acceptableFailed=7 )
+ # Verify disconnected hosts
+ if verifyDisconnected and ( main.disconnectedIpv4Hosts or main.disconnectedIpv6Hosts ):
+ main.step("Verify unreachability of disconnected internal hosts")
+ lib.verifyPing( main, main.internalIpv4Hosts, main.disconnectedIpv4Hosts, expect=False )
+ lib.verifyPing( main, main.internalIpv6Hosts, main.disconnectedIpv6Hosts, ipv6=True, expect=False )
+
+def verifyPingExternal( main, verifyDisconnected=True ):
+ """
+ Verify all connected internal hosts are able to reach external hosts,
+ and disconnected internal hosts cannot reach any external host
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ # Verify connected hosts
+ main.step("Verify reachability of from connected internal hosts to external hosts")
+ lib.verifyPing( main,
+ [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
+ main.externalIpv4Hosts )
+ lib.verifyPing( main,
+ [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
+ main.externalIpv6Hosts,
+ ipv6=True, acceptableFailed=7 )
+ # Verify disconnected hosts
+ '''
+ if verifyDisconnected and ( main.disconnectedIpv4Hosts or main.disconnectedIpv6Hosts ):
+ main.step("Verify unreachability of disconnected internal hosts to external hosts")
+ lib.verifyPing( main, main.disconnectedIpv4Hosts, main.externalIpv4Hosts, expect=False )
+ lib.verifyPing( main, main.disconnectedIpv6Hosts, main.externalIpv6Hosts, ipv6=True, expect=False )
+ '''
+
+def verifyPing( main ):
+ """
+ Verify reachability and unreachability of connected/disconnected hosts
+ """
+ verifyPingInternal( main )
+ verifyPingExternal( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
index 28ecb9a..9ee92d6 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo
index 67a973e..a342504 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 8c67f37..96ad077 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -173,13 +173,13 @@
def startMininet( main, topology, args="" ):
copyResult = main.ONOSbench.scp( main.Mininet1,
main.topoPath + main.topology,
- main.Mininet1.home,
+ main.Mininet1.home + "custom",
direction="to" )
if main.topologyLib:
for lib in main.topologyLib.split(","):
copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
main.topoPath + lib,
- main.Mininet1.home,
+ main.Mininet1.home + "custom",
direction="to" )
if main.topologyConf:
import re
@@ -208,7 +208,7 @@
arg = "--onos-ip=%s %s" % (",".join([ctrl.ipAddress for ctrl in main.Cluster.runningNodes]), args)
main.topology = topology
topoResult = main.Mininet1.startNet(
- topoFile=main.Mininet1.home + main.topology, args=arg )
+ topoFile=main.Mininet1.home + "custom/" + main.topology, args=arg )
stepResult = topoResult
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
@@ -692,7 +692,7 @@
onfail="Port %s failed" % state )
@staticmethod
- def cleanup( main, copyKarafLog=True ):
+ def cleanup( main, copyKarafLog=True, removeHostComponent=False ):
"""
Stop Onos-cluster.
Stops Mininet
@@ -718,6 +718,11 @@
main.log.info( "Removed Scapy Host Component: {0}".format( host.name ) )
main.scapyHosts = []
+ if removeHostComponent:
+ for host in main.internalIpv4Hosts + main.internalIpv6Hosts + main.externalIpv4Hosts + main.externalIpv6Hosts:
+ if hasattr( main, host ):
+ main.Network.removeHostComponent( host )
+
if hasattr( main, 'Mininet1' ):
main.utils.mininetCleanup( main.Mininet1 )
@@ -1065,3 +1070,19 @@
main.log.debug( sender.handle.before )
packetCaptured = True if pkt in packet else False
return main.TRUE if packetCaptured == expect else main.FALSE
+
+ @staticmethod
+ def verifyPing( main, srcList, dstList, ipv6=False, expect=True, wait=1, acceptableFailed=0, skipOnFail=True ):
+ """
+ Verify reachability from each host in srcList to each host in dstList
+ """
+ from tests.dependencies.topology import Topology
+ try:
+ main.topo
+ except ( NameError, AttributeError ):
+ main.topo = Topology()
+ pingResult = main.topo.ping( srcList, dstList, ipv6, expect, wait, acceptableFailed, skipOnFail )
+ if not pingResult and skipOnFail:
+ Testcaselib.saveOnosDiagnostics( main )
+ Testcaselib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
+ main.skipCase()
diff --git a/TestON/tests/dependencies/topology.py b/TestON/tests/dependencies/topology.py
index 04e6246..6527a83 100644
--- a/TestON/tests/dependencies/topology.py
+++ b/TestON/tests/dependencies/topology.py
@@ -22,6 +22,7 @@
import re
import imp
import json
+from core import utilities
class Topology:
@@ -226,3 +227,78 @@
onpass="ONOS correctly discovered the topology",
onfail="ONOS incorrectly discovered the topology" )
return topoResults
+
+ def ping( self, srcList, dstList, ipv6=False, expect=True, wait=1, acceptableFailed=0, collectT3=True ):
+ """
+ Description:
+ Ping from every host in srcList to every host in dstList and
+ verify if ping results are as expected.
+ Pings are executed in parallel from host components
+ Options:
+ src: a list of source host names, e.g. [ "h1", "h2" ]
+ dst: a list of destination host names, e.g. [ "h3", "h4" ]
+ expect: expect ping result to pass if True, otherwise fail
+ acceptableFailed: maximum number of failed pings acceptable for
+ each src-dst host pair
+ collectT3: save t3-troubleshoot output for src and dst host that failed to ping
+ Returns:
+ main.TRUE if all ping results are expected, otherwise main.FALSE
+ """
+ main.log.info( "Pinging from {} to {}, expected result is {}".format( srcList, dstList,
+ "pass" if expect else "fail" ) )
+ # Verify host component has been created
+ srcIpList = {}
+ for src in srcList:
+ if not hasattr( main, src ):
+ main.log.info( "Creating component for host {}".format( src ) )
+ main.Network.createHostComponent( src )
+ hostHandle = getattr( main, src )
+ main.log.info( "Starting CLI on host {}".format( src ) )
+ hostHandle.startHostCli()
+ srcIpList[ src ] = main.Network.getIPAddress( src, proto='IPV6' if ipv6 else 'IPV4' )
+
+ unexpectedPings = []
+ for dst in dstList:
+ dstIp = main.Network.getIPAddress( dst, proto='IPV6' if ipv6 else 'IPV4' )
+ # Start pings from src hosts in parallel
+ pool = []
+ for src in srcList:
+ srcIp = srcIpList[ src ]
+ if srcIp == dstIp:
+ continue
+ if expect and ( not srcIp or not dstIp ):
+ unexpectedPings.append( [ src, dst, "no IP" ] )
+ continue
+ hostHandle = getattr( main, src )
+ thread = main.Thread( target=utilities.retry,
+ name="{}-{}".format( srcIp, dstIp ),
+ args=[ hostHandle.pingHostSetAlternative, [ main.FALSE ] ],
+ kwargs={ "args":( [ dstIp ], wait, ipv6 ),
+ "attempts": acceptableFailed + 1,
+ "sleep": 1 } )
+ pool.append( thread )
+ thread.start()
+ # Wait for threads to finish and check ping result
+ for thread in pool:
+ thread.join( 10 )
+ srcIp, dstIp = thread.name.split( "-" )
+ if expect and not thread.result or not expect and thread.result:
+ unexpectedPings.append( [ srcIp, dstIp, "fail" if expect else "pass" ] )
+
+ utilities.assert_equals( expect=[],
+ actual=unexpectedPings,
+ onpass="Pings from {} to {} all as expected".format( srcList, dstList ),
+ onfail="Unexpected pings: {}".format( unexpectedPings ) )
+ if collectT3:
+ for unexpectedPing in unexpectedPings:
+ if unexpectedPing[ 2 ] == "no IP":
+ continue
+ srcIp = unexpectedPing[ 0 ]
+ dstIp = unexpectedPing[ 1 ]
+ main.log.debug( "Collecting t3 with source {} and destination {}".format( srcIp, dstIp ) )
+ cmd = main.Cluster.active( 0 ).CLI.composeT3Command( srcIp, dstIp, ipv6 )
+ main.log.debug( "t3 command: {}".format( cmd ) )
+ if cmd:
+ main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, cmd, main.logdir,
+ "t3-CASE{}-{}-{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp ) )
+ return main.FALSE if unexpectedPings else main.TRUE