Add ONOS node failures to SRRouting
Change-Id: I29da63d59154d880839dca2297ed648a2f98e5f0
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
index f808d4e..07f1260 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/SRRouting.py
@@ -558,4 +558,183 @@
staticRouteConfigure=True,
switchFailure=True )
+ def CASE301( self, main ):
+ """
+ Kill and recover ONOS nodes
+ Ping between all ipv4 hosts in the topology.
+ """
+
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+ SRRoutingTest.runTest( main,
+ test_idx=301,
+ onosNodes=3,
+ dhcp=1,
+ routers=1,
+ ipv4=1,
+ ipv6=0,
+ countFlowsGroups=False,
+ description="Test node failures with IPv4 hosts",
+ nodeFailure=True )
+
+ def CASE302( self, main ):
+ """
+ Kill and recover ONOS nodes
+ Ping between all ipv6 hosts in the topology.
+ """
+
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+ SRRoutingTest.runTest( main,
+ test_idx=302,
+ onosNodes=3,
+ dhcp=1,
+ routers=1,
+ ipv4=0,
+ ipv6=1,
+ countFlowsGroups=False,
+ description="Test node failures with IPv6 hosts",
+ nodeFailure=True )
+
+ def CASE303( self, main ):
+ """
+ Kill and recover ONOS nodes
+ Ping between all ipv4 and ipv6 hosts in the topology.
+ """
+
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+ SRRoutingTest.runTest( main,
+ test_idx=303,
+ onosNodes=3,
+ dhcp=1,
+ routers=1,
+ ipv4=1,
+ ipv6=1,
+ countFlowsGroups=False,
+ description="Test node failures with IPv4 and IPv6 hosts",
+ nodeFailure=True )
+
+ def CASE304( self, main ):
+ """
+ Kill and recover ONOS nodes
+ Ping between all ipv4 hosts in the topology and check connectivity to external ipv4 hosts
+ """
+
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+ SRRoutingTest.runTest( main,
+ test_idx=304,
+ onosNodes=3,
+ dhcp=1,
+ routers=1,
+ ipv4=1,
+ ipv6=0,
+ description="Test node failures with IPv4 hosts (including external hosts)",
+ checkExternalHost=True,
+ nodeFailure=True )
+
+ def CASE305( self, main ):
+ """
+ Kill and recover ONOS nodes
+ Ping between all ipv6 hosts in the topology and check connectivity to external ipv6 hosts
+ """
+
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+ SRRoutingTest.runTest( main,
+ test_idx=305,
+ onosNodes=3,
+ dhcp=1,
+ routers=1,
+ ipv4=0,
+ ipv6=1,
+ description="Test node failures with IPv6 hosts (including external hosts)",
+ checkExternalHost=True,
+ nodeFailure=True )
+
+ def CASE306( self, main ):
+ """
+ Kill and recover ONOS nodes
+ Ping between all ipv4 and ipv6 hosts in the topology and check connectivity to external ipv4 and ipv6 hosts
+ """
+
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+ SRRoutingTest.runTest( main,
+ test_idx=306,
+ onosNodes=3,
+ dhcp=1,
+ routers=1,
+ ipv4=1,
+ ipv6=1,
+ description="Test node failures with IPv4 and IPv6 hosts (including external hosts)",
+ checkExternalHost=True,
+ nodeFailure=True )
+
+ def CASE307( self, main ):
+ """
+ Kill and recover ONOS nodes
+ Ping between ipv4 hosts and an external host that is not configured in
+ external router config, but reachable through the use of route-add command.
+ """
+
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+ SRRoutingTest.runTest( main,
+ test_idx=307,
+ onosNodes=3,
+ dhcp=1,
+ routers=1,
+ ipv4=1,
+ ipv6=0,
+ description="Test node failures with IPv4 hosts (including external host configured with route-add command)",
+ checkExternalHost=False,
+ countFlowsGroups=False,
+ staticRouteConfigure=True,
+ nodeFailure=True )
+
+ def CASE308( self, main ):
+ """
+ Kill and recover ONOS nodes
+ Ping between ipv6 hosts and an external host that is not configured in
+ external router config, but reachable through the use of route-add command.
+ """
+
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+ SRRoutingTest.runTest( main,
+ test_idx=308,
+ onosNodes=3,
+ dhcp=1,
+ routers=1,
+ ipv4=0,
+ ipv6=1,
+ description="Test node failures with IPv6 hosts (including external host configured with route-add command)",
+ checkExternalHost=False,
+ countFlowsGroups=False,
+ staticRouteConfigure=True,
+ nodeFailure=True )
+
+ def CASE309( self, main ):
+ """
+ Kill and recover ONOS nodes
+ Ping between ipv4 and pv6 hosts and external hosts that is not configured in
+ external router config, but reachable through the use of route-add command.
+ """
+
+ from tests.USECASE.SegmentRouting.SRRouting.dependencies.SRRoutingTest import SRRoutingTest
+
+ SRRoutingTest.runTest( main,
+ test_idx=309,
+ onosNodes=3,
+ dhcp=1,
+ routers=1,
+ ipv4=1,
+ ipv6=1,
+ description="Test node failures with IPv4 and IPv6 hosts (including external host configured with route-add command)",
+ checkExternalHost=False,
+ countFlowsGroups=False,
+ staticRouteConfigure=True,
+ nodeFailure=True )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
index 4b58d10..5707a7c 100644
--- a/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRRouting/dependencies/SRRoutingTest.py
@@ -21,6 +21,7 @@
from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as run
import time
+import json
class SRRoutingTest ():
@@ -32,7 +33,8 @@
@staticmethod
def runTest( main, test_idx, onosNodes, dhcp, routers, ipv4, ipv6,
description, countFlowsGroups=False, checkExternalHost=False,
- staticRouteConfigure=False, switchFailure=False, linkFailure=False ):
+ staticRouteConfigure=False, switchFailure=False, linkFailure=False,
+ nodeFailure=False ):
skipPackage = False
init = False
@@ -53,7 +55,7 @@
if staticRouteConfigure:
main.cfgName += '_static=1'
- main.resultFileName = 'CASE%02d' % test_idx
+ main.resultFileName = 'CASE%03d' % test_idx
main.Cluster.setRunningNode( onosNodes )
run.installOnos( main, skipPackage=skipPackage, cliSleep=5,
@@ -97,7 +99,7 @@
time.sleep( 60 )
# ping hosts
- run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+ run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
# check flows / groups numbers
if countFlowsGroups:
@@ -107,11 +109,11 @@
if switchFailure:
for switch, expected in main.switchFailureChart.items():
run.killSwitch( main, switch, expected['switches_after_failure'], expected['links_after_failure'] )
- run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+ run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
if countFlowsGroups:
run.checkFlowsGroupsFromFile(main)
run.recoverSwitch( main, switch, expected['switches_before_failure'], expected['links_before_failure'] )
- run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+ run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
if countFlowsGroups:
run.checkFlowsGroupsFromFile(main)
@@ -124,13 +126,32 @@
linksAfter = info['links_after']
run.killLinkBatch( main, linksToRemove, linksAfter )
- run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+ run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
run.restoreLinkBatch( main, linksToRemove, linksBefore )
- run.pingAll( main, 'CASE%02d' % test_idx, acceptableFailed=5, basedOnIp=True )
+ run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
if countFlowsGroups:
run.checkFlowsGroupsFromFile(main)
+ # Test node failures
+ if nodeFailure:
+ numCtrls = len( main.Cluster.runningNodes )
+ links = len( json.loads( main.Cluster.next().links() ) )
+ switches = len( json.loads( main.Cluster.next().devices() ) )
+ for ctrl in xrange( numCtrls ):
+ run.killOnos( main, [ ctrl ], switches, links, ( numCtrls - 1 ) )
+ main.Cluster.active(0).CLI.balanceMasters()
+ run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
+ if countFlowsGroups:
+ run.checkFlowsGroupsFromFile( main )
+
+ run.recoverOnos( main, [ ctrl ], switches, links, numCtrls )
+ main.Cluster.active(0).CLI.balanceMasters()
+ run.pingAll( main, 'CASE%03d' % test_idx, acceptableFailed=5, basedOnIp=True )
+ if countFlowsGroups:
+ run.checkFlowsGroupsFromFile( main )
+
+ # Cleanup
if hasattr( main, 'Mininet1' ):
run.cleanup( main )
else:
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 0125da6..bd63a80 100644
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -97,8 +97,8 @@
# main.scale[ 0 ] determines the current number of ONOS controller
if not main.apps:
main.log.error( "App list is empty" )
- main.log.info( "NODE COUNT = " + str( main.Cluster.numCtrls ) )
- main.log.info( ''.join( main.Cluster.getIps() ) )
+ main.log.info( "Cluster size: " + str( main.Cluster.numCtrls ) )
+ main.log.info( "Cluster ips: " + ', '.join( main.Cluster.getIps() ) )
main.dynamicHosts = [ 'in1', 'out1' ]
main.testSetUp.ONOSSetUp( main.Cluster, newCell=True, cellName=main.cellName,
skipPack=skipPackage,
@@ -280,7 +280,7 @@
@staticmethod
def checkFlows( main, minFlowCount, tag="", dumpflows=True, sleep=10 ):
main.step(
- " Check whether the flow count is bigger than %s" % minFlowCount )
+ "Check whether the flow count is bigger than %s" % minFlowCount )
if tag == "":
tag = 'CASE%d' % main.CurrentTestCaseNumber
count = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowCount,
@@ -638,7 +638,7 @@
switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
"""
- main.step( "Killing ONOS instance" )
+ main.step( "Killing ONOS instances with index(es): {}".format( nodes ) )
for i in nodes:
killResult = main.ONOSbench.onosDie( main.Cluster.runningNodes[ i ].ipAddress )
@@ -685,7 +685,7 @@
switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
Recover an ONOS instance and verify the ONOS cluster can see the proper change
"""
- main.step( "Recovering ONOS instance" )
+ main.step( "Recovering ONOS instances with index(es): {}".format( nodes ) )
[ main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) for i in nodes ]
for i in nodes:
isUp = main.ONOSbench.isup( main.Cluster.runningNodes[ i ].ipAddress )