[ONOS-7747] Add router failure test cases to SRRouting
Change-Id: I06706c36caa181fff50c0f13c5976298e017940f
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
index 027aeb1..72d8925 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>101,102,103,104,105,106,107,108,109,201,202,203,204,205,206,207,208,209,301,302,303,304,305,306,307,308,309,601,602,603,604,605,606,620,621,622,630,640,641,642,643,651,652,653</testcases>
+ <testcases>101,102,103,104,105,106,107,108,109,201,202,203,204,205,206,207,208,209,301,302,303,304,305,306,307,308,309,601,602,603,604,605,606,620,621,622,630,640,641,642,643,651,652,653,660,661,662,663,664,665</testcases>
<GRAPH>
<nodeCluster>Fabric</nodeCluster>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index 281e7a5..f379896 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -1446,3 +1446,173 @@
lib.verifyTraffic( main, main.internalIpv4Hosts, "10.2.21.1", "h5v4Scapy", "h5v4-bond1" )
lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
+
+ def CASE660( self, main ):
+ """
+ External router failure
+ - Bring down quagga external router-1. Hosts that are behind router-2 should still be reachable. Hosts that are behind router-1 should not be reachable.
+ - Bring router up again, all external hosts are reachable again.
+ - Repeat this with external router-2.
+ """
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import *
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ main.case( "External router failure with cross-link" )
+ setupTest( main, test_idx=660, onosNodes=3, static=True )
+ main.externalIpv4Hosts += main.staticIpv4Hosts
+ main.externalIpv6Hosts += main.staticIpv6Hosts
+ verify( main, disconnected=False )
+ # Bring down/up external router-1
+ verifyRouterFailure( main, "r1", [ "rh5v4" ], [ "rh11v6", "rh5v6" ] )
+ # Bring down/up external router-2
+ verifyRouterFailure( main, "r2", [], [ "rh22v6" ] )
+ lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
+
+ def CASE661( self, main ):
+ """
+ External router link failure
+ - Drop a non-cross-link for external router-1. All external hosts should be reachable (via cross-link).
+ - Bring up the link. All external hosts should be reachable.
+ - Repeat the steps above with the cross-link of external router-1
+ - Repeat all steps above with external router-2
+ """
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import *
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ main.case( "External router link failure with cross-link" )
+ setupTest( main, test_idx=661, onosNodes=3, static=True )
+ main.externalIpv4Hosts += main.staticIpv4Hosts
+ main.externalIpv6Hosts += main.staticIpv6Hosts
+ verify( main, disconnected=False )
+ # Bring down/up a non-cross-link for external router-1
+ portsToDisable = [ [ "of:0000000000000005", 13 ] ]
+ lib.disablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ lib.enablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ # Bring down/up a cross-link for external router-1
+ portsToDisable = [ [ "of:0000000000000005", 14 ] ]
+ lib.disablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ lib.enablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ # Bring down/up a non-cross-link for external router-2
+ portsToDisable = [ [ "of:0000000000000004", 14 ] ]
+ lib.disablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ lib.enablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ # Bring down/up a cross-link for external router-2
+ portsToDisable = [ [ "of:0000000000000004", 13 ] ]
+ lib.disablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ lib.enablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
+
+ def CASE662( self, main ):
+ """
+ Internal router failure
+ - Bring down quagga internal router-1. All external hosts should be reachable (via cross-link).
+ - Bring the router up. All external hosts should be reachable.
+ - Repeat this with internal router-2.
+ """
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import *
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ main.case( "Internal router failure with cross-link" )
+ setupTest( main, test_idx=662, onosNodes=3, static=True )
+ main.externalIpv4Hosts += main.staticIpv4Hosts
+ main.externalIpv6Hosts += main.staticIpv6Hosts
+ verify( main, disconnected=False )
+ # Bring down/up internal router-1
+ verifyRouterFailure( main, "bgp1" )
+ # Bring down/up internal router-2
+ verifyRouterFailure( main, "bgp2" )
+ lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
+
+ def CASE663( self, main ):
+ """
+ External router failure without cross-link
+ - Drop the cross-link for both external routers. All external hosts should be reachable.
+ - Bring down quagga external router-1. Hosts that are behind router-2 should still be reachable. Hosts that are behind router-1 should not be reachable.
+ - Bring router up again, all external hosts are reachable again.
+ - Repeat this with external router-2.
+ """
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import *
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ main.case( "External router failure without cross-link" )
+ setupTest( main, test_idx=663, onosNodes=3, static=True )
+ main.externalIpv4Hosts += main.staticIpv4Hosts
+ main.externalIpv6Hosts += main.staticIpv6Hosts
+ verify( main, disconnected=False )
+ # Drop the cross-link
+ portsToDisable = [ [ "of:0000000000000004", 13 ], [ "of:0000000000000005", 14 ] ]
+ lib.disablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ # Bring down/up external router-1
+ verifyRouterFailure( main, "r1", [ "rh5v4" ], [ "rh11v6", "rh5v6" ] )
+ # Bring down/up external router-2
+ verifyRouterFailure( main, "r2", [], [ "rh22v6" ] )
+ lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
+
+ def CASE664( self, main ):
+ """
+ External router link failure without cross-link
+ - Drop the cross-link for both external routers. All external hosts should be reachable.
+ - Drop an extra link for external router-1. Only hosts connected to router-2 should be reachable.
+ - Bring up single link for external router-1. All external hosts should be reachable.
+ - Repeat the two steps above with external router-2
+ """
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import *
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ main.case( "External router link failure without cross-link" )
+ setupTest( main, test_idx=664, onosNodes=3, static=True )
+ main.externalIpv4Hosts += main.staticIpv4Hosts
+ main.externalIpv6Hosts += main.staticIpv6Hosts
+ verify( main, disconnected=False )
+ # Drop the cross-link
+ portsToDisable = [ [ "of:0000000000000004", 13 ], [ "of:0000000000000005", 14 ] ]
+ lib.disablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ # Bring down/up a non-cross-link for external router-1
+ portsToDisable = [ [ "of:0000000000000005", 13 ] ]
+ lib.disablePortBatch( main, portsToDisable, 10, 48 )
+ main.disconnectedExternalIpv4Hosts = [ 'rh5v4' ]
+ main.disconnectedExternalIpv6Hosts = [ "rh11v6", "rh5v6" ]
+ verify( main, internal=False )
+ lib.enablePortBatch( main, portsToDisable, 10, 48 )
+ main.disconnectedExternalIpv4Hosts = []
+ main.disconnectedExternalIpv6Hosts = []
+ verify( main, disconnected=False, internal=False )
+ # Bring down/up a non-cross-link for external router-2
+ portsToDisable = [ [ "of:0000000000000004", 14 ] ]
+ lib.disablePortBatch( main, portsToDisable, 10, 48 )
+ main.disconnectedExternalIpv6Hosts = [ "rh22v6" ]
+ verify( main, internal=False )
+ lib.enablePortBatch( main, portsToDisable, 10, 48 )
+ main.disconnectedExternalIpv6Hosts = []
+ verify( main, disconnected=False, internal=False )
+ lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
+
+ def CASE665( self, main ):
+ """
+ Internal router failure without cross-link
+ - Drop the cross-link for both external routers. All external hosts should be reachable.
+ - Bring down quagga internal router-1. Hosts that are behind router-2 should still be reachable. Hosts that are behind router-1 should not be reachable.
+ - Bring router up again, all external hosts are reachable again.
+ - Repeat this with internal router-2.
+ """
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import *
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ main.case( "Internal router failure without cross-link" )
+ setupTest( main, test_idx=665, onosNodes=3, static=True )
+ main.externalIpv4Hosts += main.staticIpv4Hosts
+ main.externalIpv6Hosts += main.staticIpv6Hosts
+ verify( main, disconnected=False )
+ # Drop the cross-link
+ portsToDisable = [ [ "of:0000000000000004", 13 ], [ "of:0000000000000005", 14 ] ]
+ lib.disablePortBatch( main, portsToDisable, 10, 48 )
+ verify( main, disconnected=False, internal=False )
+ # Bring down/up internal router-1
+ verifyRouterFailure( main, "bgp1", [], [ "rh22v6" ] )
+ # Bring down/up internal router-2
+ verifyRouterFailure( main, "bgp2", [ "rh5v4" ], [ "rh11v6", "rh5v6" ] )
+ lib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index 0611abc..738b6b4 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -67,8 +67,10 @@
if static:
if ipv4:
lib.addStaticOnosRoute( main, "10.0.88.0/24", "10.0.1.1")
+ lib.addStaticOnosRoute( main, "10.0.88.0/24", "10.0.5.1")
if ipv6:
lib.addStaticOnosRoute( main, "2000::8700/120", "2000::101")
+ lib.addStaticOnosRoute( main, "2000::8700/120", "2000::501")
if countFlowsGroups:
lib.loadCount( main )
@@ -248,3 +250,18 @@
lib.checkFlowsGroupsFromFile( main )
# ping hosts
verifyPing( main, ipv4, ipv6, disconnected, internal, external )
+
+def verifyRouterFailure( main, routerToKill, affectedIpv4Hosts=[], affectedIpv6Hosts=[],
+ ipv4=True, ipv6=True, countFlowsGroups=False ):
+ """
+ Kill and recover a quagga router and verify connectivities to external hosts
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib
+ lib.killRouter( main, routerToKill, 5 )
+ main.disconnectedExternalIpv4Hosts = affectedIpv4Hosts
+ main.disconnectedExternalIpv6Hosts = affectedIpv6Hosts
+ verify( main, ipv4, ipv6, True if (affectedIpv4Hosts or affectedIpv6Hosts) else False, False, True, countFlowsGroups )
+ lib.recoverRouter( main, routerToKill, 5 )
+ main.disconnectedExternalIpv4Hosts = []
+ main.disconnectedExternalIpv6Hosts = []
+ verify( main, ipv4, ipv6, False, False, True, countFlowsGroups )
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 5bd3bdd..7ec46f2 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -781,6 +781,40 @@
onfail="Failed to recover switch?" )
@staticmethod
+ def killRouter( main, router, sleep=None ):
+ """
+ Kill bgpd process on a quagga router
+ router: name of the router to be killed. E.g. "bgp1"
+ """
+ sleep = float( sleep )
+ main.step( "Kill " + str( router ) )
+ if hasattr( main, 'Mininet1' ):
+ main.Mininet1.handle.sendline( "px {}.stopProtocols()".format( router ) )
+ main.Mininet1.handle.expect( "mininet>" )
+ else:
+ # TODO: support killing router in physical network
+ pass
+ main.log.info( "Waiting %s seconds for router down to be discovered" % ( sleep ) )
+ time.sleep( sleep )
+
+ @staticmethod
+ def recoverRouter( main, router, sleep=None ):
+ """
+ Restart bgpd process on a quagga router
+ router: name of the router to be recovered. E.g. "bgp1"
+ """
+ sleep = float( sleep )
+ main.step( "Recovering " + str( router ) )
+ if hasattr( main, 'Mininet1' ):
+ main.Mininet1.handle.sendline( "px {}.startProtocols()".format( router ) )
+ main.Mininet1.handle.expect( "mininet>" )
+ else:
+ # TODO: support recovering router in physical network
+ pass
+ main.log.info( "Waiting %s seconds for router up to be discovered" % ( sleep ) )
+ time.sleep( sleep )
+
+ @staticmethod
def cleanup( main, copyKarafLog=True, removeHostComponent=False ):
"""
Stop Onos-cluster.
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/routinglib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/routinglib.py
index 621b372..13a75e2 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/routinglib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/routinglib.py
@@ -156,6 +156,14 @@
if self.defaultRoute:
self.cmd('ip route add default via %s' % self.defaultRoute)
+ def stopProtocols(self, **kwargs):
+ for p in self.protocols:
+ p.stop(**kwargs)
+
+ def startProtocols(self, **kwargs):
+ for p in self.protocols:
+ p.start(**kwargs)
+
def terminate(self, **kwargs):
self.cmd("ps ax | grep '%s' | awk '{print $1}' | xargs kill"
% (self.socket))
@@ -175,6 +183,12 @@
def config(self, **kwargs):
pass
+ def stop(self, **kwargs):
+ pass
+
+ def start(self, **kwargs):
+ pass
+
def terminate(self, **kwargs):
pass
@@ -199,6 +213,14 @@
self.qr.cmd('%s/bgpd -d -f %s -z %s -i %s'
% (QuaggaRouter.binDir, self.configFile, self.qr.socket, bgpdPidFile))
+ def stop(self, **kwargs):
+ self.qr.cmd('pkill -f %s' % self.configFile)
+
+ def start(self, **kwargs):
+ bgpdPidFile = '%s/bgpd%s.pid' % (self.qr.runDir, self.qr.name)
+ self.qr.cmd('%s/bgpd -d -f %s -z %s -i %s'
+ % (QuaggaRouter.binDir, self.configFile, self.qr.socket, bgpdPidFile))
+
def generateConfig(self):
conf = ConfigurationWriter(self.configFile)