[ONOS-7635][ONOS-7636] Support for complex scenarios in SRRouting test
- New ping function to execute pings in parallel
- Collect t3 output on ping failures
- Allow expected ping failures
Change-Id: I4492a89bc4c0a581ff2e35bcc1896ddd5ea64a18
diff --git a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo
index a672a8e..7b188e2 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRBridging/SRBridging.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
index 5c4cc02..ad2c2b0 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/SRDhcprelay.topo b/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/SRDhcprelay.topo
index 01316b6..dbb415d 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/SRDhcprelay.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRDhcprelay/SRDhcprelay.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.topo b/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.topo
index 5c4cc02..ad2c2b0 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRDynamic/SRDynamic.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/SRDynamicConf.topo b/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/SRDynamicConf.topo
index a672a8e..7b188e2 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/SRDynamicConf.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRDynamicConf/SRDynamicConf.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
index 5c4cc02..ad2c2b0 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo
index 5c4cc02..ad2c2b0 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.topo b/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.topo
index 1c3f8a6..b5da976 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRMulticast/SRMulticast.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
index 28ecb9a..9ee92d6 100755
--- a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
index e8199bd..e19bd34 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
@@ -21,7 +21,7 @@
<ENV>
<cellName>productionCell</cellName>
- <cellApps>drivers,openflow,segmentrouting,fpm,dhcprelay,netcfghostprovider,routeradvertisement</cellApps>
+ <cellApps>drivers,openflow,segmentrouting,fpm,dhcprelay,netcfghostprovider,routeradvertisement,t3</cellApps>
</ENV>
<GIT>
@@ -39,8 +39,17 @@
<OnosDiscovery>30</OnosDiscovery>
<loadNetcfgSleep>5</loadNetcfgSleep>
<startMininetSleep>25</startMininetSleep>
- <dhcpSleep>60</dhcpSleep>
+ <dhcpSleep>30</dhcpSleep>
<balanceMasterSleep>10</balanceMasterSleep>
</timers>
+ <TOPO>
+ <internalIpv4Hosts>h1v4,h2v4,h3v4,h4v4,h5v4,h6v4,h7v4,h8v4,h9v4,h10v4,h11v4,h12v4,h13v4</internalIpv4Hosts>
+ <internalIpv6Hosts>h1v6,h2v6,h3v6,h4v6,h5v6,h6v6,h7v6,h8v6,h9v6,h10v6,h11v6,h12v6,h13v6</internalIpv6Hosts>
+ <externalIpv4Hosts>rh1v4,rh2v4</externalIpv4Hosts>
+ <externalIpv6Hosts>rh1v6,rh11v6,rh2v6,rh22v6</externalIpv6Hosts>
+ <switchNum>10</switchNum>
+ <linkNum>48</linkNum>
+ </TOPO>
+
</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index 07f1260..5f4157f 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -738,3 +738,34 @@
staticRouteConfigure=True,
nodeFailure=True )
+ def CASE606( self, main ):
+ """
+ Drop SPINE-1 and test connectivity
+ Drop paired leaf and test connectivity (expect some failures)
+ Bring up SPINE-1 and test connectivity (still expect some failures)
+ Bring up the paired leaf and test connectivity
+ Repeat above with SPINE-2 and a different paired leaf
+ """
+ import time
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import *
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ main.case( "Drop spine and paired leaf" )
+ setupTest( main, test_idx=606, onosNodes=3 )
+ main.disconnectedIpv4Hosts = []
+ main.disconnectedIpv6Hosts = []
+ verifyPing( main )
+ lib.killSwitch( main, "spine101", int( main.params[ "TOPO" ][ "switchNum" ] ) - 1, int( main.params[ "TOPO" ][ "linkNum" ] ) - 18 )
+ verifyPing( main )
+ lib.killSwitch( main, "leaf2", int( main.params[ "TOPO" ][ "switchNum" ] ) - 2, int( main.params[ "TOPO" ][ "linkNum" ] ) - 24 )
+ lib.killSwitch( main, "leaf3", int( main.params[ "TOPO" ][ "switchNum" ] ) - 3, int( main.params[ "TOPO" ][ "linkNum" ] ) - 28 )
+ main.disconnectedIpv4Hosts = [ "h3v4", "h4v4", "h5v4", "h6v4", "h7v4" ]
+ main.disconnectedIpv6Hosts = [ "h3v6", "h4v6", "h5v6", "h6v6", "h7v6" ]
+ verifyPing( main )
+ lib.recoverSwitch( main, "spine101", int( main.params[ "TOPO" ][ "switchNum" ] ) - 2, int( main.params[ "TOPO" ][ "linkNum" ] ) - 18 )
+ verifyPing( main )
+ lib.recoverSwitch( main, "leaf3", int( main.params[ "TOPO" ][ "switchNum" ] ) - 1, int( main.params[ "TOPO" ][ "linkNum" ] ) - 10 )
+ lib.recoverSwitch( main, "leaf2", int( main.params[ "TOPO" ][ "switchNum" ] ), int( main.params[ "TOPO" ][ "linkNum" ] ) )
+ main.disconnectedIpv4Hosts = []
+ main.disconnectedIpv6Hosts = []
+ verifyPing( main )
+ lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo
index 01316b6..dbb415d 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.topo
@@ -27,7 +27,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index 68533bb..a9a24bb 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -154,3 +154,97 @@
run.checkFlowsGroupsFromFile( main )
# ping hosts
run.pingAll( main, 'CASE%03d' % test_idx, False, acceptableFailed=5, basedOnIp=True, skipOnFail=True )
+
+
+def setupTest( main, test_idx, onosNodes ):
+ """
+ SRRouting test setup
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ skipPackage = False
+ init = False
+ if not hasattr( main, 'apps' ):
+ init = True
+ run.initTest( main )
+ # Skip onos packaging if the cluster size stays the same
+ if not init and onosNodes == main.Cluster.numCtrls:
+ skipPackage = True
+
+ main.internalIpv4Hosts = main.params[ 'TOPO' ][ 'internalIpv4Hosts' ].split( ',' )
+ main.internalIpv6Hosts = main.params[ 'TOPO' ][ 'internalIpv6Hosts' ].split( ',' )
+ main.externalIpv4Hosts = main.params[ 'TOPO' ][ 'externalIpv4Hosts' ].split( ',' )
+ main.externalIpv6Hosts = main.params[ 'TOPO' ][ 'externalIpv6Hosts' ].split( ',' )
+ main.resultFileName = "CASE%03d" % test_idx
+ main.Cluster.setRunningNode( onosNodes )
+ lib.installOnos( main, skipPackage=skipPackage, cliSleep=5 )
+ # Load configuration files
+ main.step( "Load configurations" )
+ main.cfgName = "TEST_CONFIG_ipv4=1_ipv6=1_dhcp=1_routers=1_external=1"
+ lib.loadJson( main )
+ time.sleep( float( main.params[ "timers" ][ "loadNetcfgSleep" ] ) )
+
+ if hasattr( main, "Mininet1" ):
+ # Run the test with Mininet
+ mininet_args = " --dhcp=1 --routers=1 --ipv6=1 --ipv4=1"
+ lib.startMininet( main, main.params[ "DEPENDENCY" ][ "topology" ], args=mininet_args )
+ time.sleep( float( main.params[ "timers" ][ "startMininetSleep" ] ) )
+ else:
+ # Run the test with physical devices
+ lib.connectToPhysicalNetwork( main, self.switchNames )
+ # Check if the devices are up
+ lib.checkDevices( main, switches=len( self.switchNames ) )
+
+ # wait some time for onos to install the rules!
+ time.sleep( float( main.params[ "timers" ][ "startMininetSleep" ] ) )
+ time.sleep( float( main.params[ "timers" ][ "dhcpSleep" ] ) )
+
+def verifyPingInternal( main, verifyDisconnected=True ):
+ """
+ Verify all connected internal hosts are able to reach each other,
+ and disconnected internal hosts cannot reach any other internal host
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ # Verify connected hosts
+ main.step("Verify reachability of connected internal hosts")
+ lib.verifyPing( main,
+ [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
+ [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ] )
+ lib.verifyPing( main,
+ [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
+ [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
+ ipv6=True, acceptableFailed=7 )
+ # Verify disconnected hosts
+ if verifyDisconnected and ( main.disconnectedIpv4Hosts or main.disconnectedIpv6Hosts ):
+ main.step("Verify unreachability of disconnected internal hosts")
+ lib.verifyPing( main, main.internalIpv4Hosts, main.disconnectedIpv4Hosts, expect=False )
+ lib.verifyPing( main, main.internalIpv6Hosts, main.disconnectedIpv6Hosts, ipv6=True, expect=False )
+
+def verifyPingExternal( main, verifyDisconnected=True ):
+ """
+ Verify all connected internal hosts are able to reach external hosts,
+ and disconnected internal hosts cannot reach any external host
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ # Verify connected hosts
+ main.step("Verify reachability of from connected internal hosts to external hosts")
+ lib.verifyPing( main,
+ [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
+ main.externalIpv4Hosts )
+ lib.verifyPing( main,
+ [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
+ main.externalIpv6Hosts,
+ ipv6=True, acceptableFailed=7 )
+ # Verify disconnected hosts
+ '''
+ if verifyDisconnected and ( main.disconnectedIpv4Hosts or main.disconnectedIpv6Hosts ):
+ main.step("Verify unreachability of disconnected internal hosts to external hosts")
+ lib.verifyPing( main, main.disconnectedIpv4Hosts, main.externalIpv4Hosts, expect=False )
+ lib.verifyPing( main, main.disconnectedIpv6Hosts, main.externalIpv6Hosts, ipv6=True, expect=False )
+ '''
+
+def verifyPing( main ):
+ """
+ Verify reachability and unreachability of connected/disconnected hosts
+ """
+ verifyPingInternal( main )
+ verifyPingExternal( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
index 28ecb9a..9ee92d6 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/SRSanity.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo
index 67a973e..a342504 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo
+++ b/TestON/tests/USECASE/SegmentRouting/SRSwitchFailure/SRSwitchFailure.topo
@@ -28,7 +28,7 @@
<type>MininetCliDriver</type>
<connect_order>2</connect_order>
<COMPONENTS>
- <home>~/mininet/custom/</home>
+ <home>~/mininet/</home>
<prompt></prompt>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 8c67f37..96ad077 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -173,13 +173,13 @@
def startMininet( main, topology, args="" ):
copyResult = main.ONOSbench.scp( main.Mininet1,
main.topoPath + main.topology,
- main.Mininet1.home,
+ main.Mininet1.home + "custom",
direction="to" )
if main.topologyLib:
for lib in main.topologyLib.split(","):
copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
main.topoPath + lib,
- main.Mininet1.home,
+ main.Mininet1.home + "custom",
direction="to" )
if main.topologyConf:
import re
@@ -208,7 +208,7 @@
arg = "--onos-ip=%s %s" % (",".join([ctrl.ipAddress for ctrl in main.Cluster.runningNodes]), args)
main.topology = topology
topoResult = main.Mininet1.startNet(
- topoFile=main.Mininet1.home + main.topology, args=arg )
+ topoFile=main.Mininet1.home + "custom/" + main.topology, args=arg )
stepResult = topoResult
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
@@ -692,7 +692,7 @@
onfail="Port %s failed" % state )
@staticmethod
- def cleanup( main, copyKarafLog=True ):
+ def cleanup( main, copyKarafLog=True, removeHostComponent=False ):
"""
Stop Onos-cluster.
Stops Mininet
@@ -718,6 +718,11 @@
main.log.info( "Removed Scapy Host Component: {0}".format( host.name ) )
main.scapyHosts = []
+ if removeHostComponent:
+ for host in main.internalIpv4Hosts + main.internalIpv6Hosts + main.externalIpv4Hosts + main.externalIpv6Hosts:
+ if hasattr( main, host ):
+ main.Network.removeHostComponent( host )
+
if hasattr( main, 'Mininet1' ):
main.utils.mininetCleanup( main.Mininet1 )
@@ -1065,3 +1070,19 @@
main.log.debug( sender.handle.before )
packetCaptured = True if pkt in packet else False
return main.TRUE if packetCaptured == expect else main.FALSE
+
+ @staticmethod
+ def verifyPing( main, srcList, dstList, ipv6=False, expect=True, wait=1, acceptableFailed=0, skipOnFail=True ):
+ """
+ Verify reachability from each host in srcList to each host in dstList
+ """
+ from tests.dependencies.topology import Topology
+ try:
+ main.topo
+ except ( NameError, AttributeError ):
+ main.topo = Topology()
+ pingResult = main.topo.ping( srcList, dstList, ipv6, expect, wait, acceptableFailed, skipOnFail )
+ if not pingResult and skipOnFail:
+ Testcaselib.saveOnosDiagnostics( main )
+ Testcaselib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
+ main.skipCase()