Kill and Recover tests in SRRouting
Change-Id: I86f6c7850b6fffaa11fd97d8805f5583afa5c5e1
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
index 7f6ede2..3358636 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
@@ -72,7 +72,9 @@
<switchNum>10</switchNum>
<linkNum>48</linkNum>
</TOPO>
-
+ <LinksToRemove>
+ [ ["spine103", "spine101"], ["leaf2", "spine101"], ["leaf3", "spine101"], ["leaf4", "spine101"], ["leaf5", "spine101"] ]
+ </LinksToRemove>
<ALARM>
<minPassPercent>80</minPassPercent>
</ALARM>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params.tucson b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params.tucson
index 0141baa..fa7076a 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params.tucson
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params.tucson
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,3,4,6,7,9</testcases>
+ <testcases>1,3,4,6,7,9,101,104</testcases>
<GRAPH>
<nodeCluster>pairedleaves</nodeCluster>
@@ -76,7 +76,7 @@
<OnosDiscovery>45</OnosDiscovery>
<loadNetcfgSleep>5</loadNetcfgSleep>
<startMininetSleep>25</startMininetSleep>
- <dhcpSleep>150</dhcpSleep>
+ <dhcpSleep>15</dhcpSleep>
<balanceMasterSleep>10</balanceMasterSleep>
</timers>
@@ -85,11 +85,36 @@
</SLEEP>
<TOPO>
- <internalIpv4Hosts>h1,h2,h3,mgmt</internalIpv4Hosts>
+ <internalIpv4Hosts>h1,h2,h3</internalIpv4Hosts>
<switchNum>2</switchNum>
<linkNum>2</linkNum>
</TOPO>
-
+ <CASE101Links>
+ <Stage1>
+ <leaf1>260</leaf1>
+ </Stage1>
+ <Stage2>
+ <leaf1>40,56,260,264</leaf1>
+ </Stage2>
+ <Stage3>
+ <expect>False</expect>
+ <leaf2>40,56,136,144</leaf2>
+ <leaf1>40,56,260,264</leaf1>
+ </Stage3>
+ </CASE101Links>
+ <CASE104Links>
+ <Stage1>
+ <leaf1>260</leaf1>
+ </Stage1>
+ <Stage2>
+ <leaf1>40,56,260,264</leaf1>
+ </Stage2>
+ <Stage3>
+ <expect>False</expect>
+ <leaf2>40,56,136,144</leaf2>
+ <leaf1>40,56,260,264</leaf1>
+ </Stage3>
+ </CASE104Links>
<ALARM>
<minPassPercent>100</minPassPercent>
</ALARM>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index d4ed62c..467231c 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -125,7 +125,24 @@
main.case( "Test link failures with IPv4 hosts" )
setupTest( main, test_idx=101, onosNodes=3, ipv6=False, external=False )
verify( main, ipv6=False, external=False, disconnected=False )
- verifyLinkFailure( main, ipv6=False, external=False )
+ MaxLinks = int( main.params["TOPO"]["linkNum"] )
+ MaxSwitches = int( main.params["TOPO"]["switchNum"] )
+ currentLinks = MaxLinks
+ expect = True
+ for stage in main.params.get( "CASE101Links", [] ):
+ main.log.debug("linksFailures" + stage )
+ linksToRemove = []
+ for switch in main.params[ "CASE101Links" ][ stage ]:
+ main.log.debug("processingSwitch" + switch )
+ if switch == "expect":
+ expect = main.params[ "CASE101Links" ][ stage ][ switch ].split() == "True"
+ continue
+ endpoints = main.params[ "CASE101Links" ][ stage ][ switch ].split(",")
+ for endpoint in endpoints:
+ linksToRemove.append( [ switch, endpoint ] )
+ currentLinks = (currentLinks - 2*len(linksToRemove))
+ verifyLinkFailure( main, linksToRemove, currentLinks, MaxSwitches, ipv6=False, external=False, skipOnFail=False, expectedConnectivity=expect )
+ verifyLinksRestored( main, linksToRemove, MaxLinks, MaxSwitches, ipv6=False, external=False, skipOnFail=False )
lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
def CASE102( self, main ):
@@ -164,7 +181,24 @@
main.case( "Test link failures with IPv4 hosts including external hosts" )
setupTest( main, test_idx=104, onosNodes=3, ipv6=False )
verify( main, ipv6=False, disconnected=False )
- verifyLinkFailure( main, ipv6=False )
+ MaxLinks = int( main.params["TOPO"]["linkNum"] )
+ MaxSwitches = int( main.params["TOPO"]["switchNum"] )
+ currentLinks = MaxLinks
+ expect = True
+ for stage in main.params.get( "CASE104Links", [] ):
+ main.log.debug("linksFailures" + stage )
+ linksToRemove = []
+ for switch in main.params[ "CASE104Links" ][ stage ]:
+ main.log.debug("processingSwitch" + switch )
+ if switch == "expect":
+ expect = main.params[ "CASE104Links" ][ stage ][ switch ].split() == "True"
+ continue
+ endpoints = main.params[ "CASE104Links" ][ stage ][ switch ].split(",")
+ for endpoint in endpoints:
+ linksToRemove.append( [ switch, endpoint ] )
+ currentLinks = (currentLinks - 2*len(linksToRemove))
+ verifyLinkFailure( main, linksToRemove, currentLinks, MaxSwitches, ipv6=False, external=False, skipOnFail=False, expectedConnectivity=expect )
+ verifyLinksRestored( main, linksToRemove, MaxLinks, MaxSwitches, ipv6=False, external=False, skipOnFail=False )
lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
def CASE105( self, main ):
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index bcf9728..5d2402f 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -114,7 +114,7 @@
main.log.exception( "Error in setupTest" )
main.skipCase( result="FAIL", msg=e )
-def verifyPingInternal( main, ipv4=True, ipv6=True, disconnected=True ):
+def verifyPingInternal( main, ipv4=True, ipv6=True, disconnected=True, skipOnFail=True, expect=True ):
"""
Verify all connected internal hosts are able to reach each other,
and disconnected internal hosts cannot reach any other internal host
@@ -125,13 +125,13 @@
lib.verifyPing( main,
[ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
[ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
- stepMsg="Verify reachability of connected internal IPv4 hosts" )
+ stepMsg="Verify reachability of connected internal IPv4 hosts", skipOnFail=skipOnFail, expect=expect )
if ipv6:
lib.verifyPing( main,
[ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
[ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
ipv6=True,
- stepMsg="Verify reachability of connected internal IPv6 hosts" )
+ stepMsg="Verify reachability of connected internal IPv6 hosts", skipOnFail=skipOnFail, expect=expect )
# Verify disconnected hosts
if disconnected:
if main.disconnectedIpv4Hosts:
@@ -141,7 +141,7 @@
lib.verifyPing( main, main.internalIpv6Hosts, main.disconnectedIpv6Hosts, ipv6=True, expect=False,
stepMsg="Verify unreachability of disconnected internal IPv6 hosts" )
-def verifyPingExternal( main, ipv4=True, ipv6=True, disconnected=True ):
+def verifyPingExternal( main, ipv4=True, ipv6=True, disconnected=True, skipOnFail=True, expect=True ):
"""
Verify all connected internal hosts are able to reach external hosts,
and disconnected internal hosts cannot reach any external host
@@ -153,14 +153,14 @@
[ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ],
[ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ],
stepMsg="Verify reachability from connected internal IPv4 hosts to external IPv4 hosts",
- t3Simple=False )
+ t3Simple=False, skipOnFail=skipOnFail, expect=expect )
if ipv6:
lib.verifyPing( main,
[ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ],
[ h for h in main.externalIpv6Hosts if h not in main.disconnectedExternalIpv6Hosts ],
ipv6=True,
stepMsg="Verify reachability from connected internal IPv6 hosts to external IPv6 hosts",
- t3Simple=False )
+ t3Simple=False, skipOnFail=skipOnFail, expect=expect )
# Verify disconnected hosts
if disconnected:
# Disconnected internal to connected external
@@ -192,39 +192,46 @@
stepMsg="Verify unreachability of connected internal IPv6 hosts to disconnected external IPv6 hosts",
t3Simple=False )
-def verifyPing( main, ipv4=True, ipv6=True, disconnected=False, internal=True, external=True ):
+def verifyPing( main, ipv4=True, ipv6=True, disconnected=False, internal=True, external=True, skipOnFail=True, expect=True ):
"""
Verify reachability and unreachability of connected/disconnected hosts
"""
if internal:
- verifyPingInternal( main, ipv4, ipv6, disconnected )
+ verifyPingInternal( main, ipv4, ipv6, disconnected, skipOnFail, expect=expect )
if external:
- verifyPingExternal( main, ipv4, ipv6, disconnected )
+ verifyPingExternal( main, ipv4, ipv6, disconnected, skipOnFail, expect=expect )
-def verifyLinkFailure( main, ipv4=True, ipv6=True, disconnected=False,
- internal=True, external=True, countFlowsGroups=False ):
+def verifyLinkFailure( main, linksToRemove, expectedLinks, expectedSwitches, ipv4=True, ipv6=True, disconnected=False,
+ internal=True, external=True, countFlowsGroups=False, skipOnFail=True, expectedConnectivity=True ):
"""
- Kill and recover all links to spine101 and 102 sequencially and run verifications
+ Kill all links sequencially and run verifications
"""
from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
- linksToRemove = [ ["spine103", "spine101"],
- ["leaf2", "spine101"],
- ["leaf3", "spine101"],
- ["leaf4", "spine101"],
- ["leaf5", "spine101"] ]
- lib.killLinkBatch( main, linksToRemove, 30, 10 )
- verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups )
- lib.restoreLinkBatch( main, linksToRemove, 48, 10 )
- verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups )
- linksToRemove = [ ["spine104", "spine102"],
- ["leaf2", "spine102"],
- ["leaf3", "spine102"],
- ["leaf4", "spine102"],
- ["leaf5", "spine102"] ]
- lib.killLinkBatch( main, linksToRemove, 30, 10 )
- verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups )
- lib.restoreLinkBatch( main, linksToRemove, 48, 10 )
- verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups )
+ #linksToRemove = [ ["leaf1", 260] ]
+ if hasattr( main, "Mininet1" ):
+ skipOnFail=True
+ lib.killLinkBatch( main, linksToRemove, expectedLinks, expectedSwitches )
+ else:
+ skipOnFail=False
+ for link in linksToRemove:
+ main.Cluster.next().portstate( dpid="device:"+link[0], port=link[1], state="disable" )
+ verifyPing( main, ipv4, ipv6, disconnected, internal, external, skipOnFail=skipOnFail, expect=expectedConnectivity )
+
+def verifyLinksRestored( main, linksToRemove, expectedLinks, expectedSwitches, ipv4=True, ipv6=True, disconnected=False,
+ internal=True, external=True, countFlowsGroups=False, skipOnFail=True, topoSleep=30 ):
+ """
+ Recover all links sequencially and run verifications
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ import time
+ if hasattr( main, "Mininet1"):
+ lib.restoreLinkBatch( main, linksToRemove, expectedLinks, expectedSwitches )
+ else:
+ for link in linksToRemove:
+ main.Cluster.next().portstate( dpid="device:"+link[0], port=link[1], state="enable" )
+ time.sleep(topoSleep)
+ lib.discoverHosts(main)
+ verifyPing( main, ipv4, ipv6, disconnected, internal, external )
def verifySwitchFailure( main, ipv4=True, ipv6=True, disconnected=False,
internal=True, external=True, countFlowsGroups=False ):
@@ -233,9 +240,9 @@
"""
from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
for switchToKill in [ "spine101", "spine102" ]:
- lib.killSwitch( main, switchToKill, 9, 30 )
+ lib.killSwitch( main, switchToKill, expectedLinks, expectedSwitches )
verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups )
- lib.recoverSwitch( main, switchToKill, 10, 48 )
+ lib.recoverSwitch( main, switchToKill, expectedLinks, expectedSwitches )
verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups )
def verifyOnosFailure( main, ipv4=True, ipv6=True, disconnected=False,