New HA tests for Segment Routing
Changes:
- Introduces new HA tests for Segment Routing
- Improve TestCaseLib adding the parsing of the params file
Change-Id: I29412200fedc7c429db0ecae24b55c1e770b5ce0
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/README.md b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/README.md
new file mode 100644
index 0000000..f057e2c
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/README.md
@@ -0,0 +1,17 @@
+These tests are meant to test the high availability of ONOS and
+SR application.
+
+It consists of:
+1) Configure and install ONOS;
+2) Pingall between hosts;
+3) Kill one ONOS instance;
+4) Kill one spine;
+5) Repeat this test a number of time;
+
+Requirements:
+1) An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+
+The topologies are 2x2 Leaf-Spine and 4x4 Leaf-Spine.
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params
new file mode 100644
index 0000000..5f990fa
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params
@@ -0,0 +1,53 @@
+<PARAMS>
+
+ <testcases>1,2,3,4,5,6,7,8</testcases>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <wrapper1>startUp</wrapper1>
+ <topology>cord_fabric.py</topology>
+ </DEPENDENCY>
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,segmentrouting</cellApps>
+ <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
+ <cellUser>sdn</cellUser>
+ </ENV>
+
+ <GIT>
+ <pull>False</pull>
+ <branch>master</branch>
+ </GIT>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ </timers>
+
+ <switches>
+ <spine1> spine101 </spine1>
+ <spine2> spine102 </spine2>
+ <spine3> spine103 </spine3>
+ <spine4> spine104 </spine4>
+ <spinedpid1> 000000000101 </spinedpid1>
+ <spinedpid2> 000000000102 </spinedpid2>
+ <spinedpid3> 000000000103 </spinedpid3>
+ <spinedpid4> 000000000104 </spinedpid4>
+ </switches>
+
+ <failures>3</failures>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.py b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.py
new file mode 100644
index 0000000..77f7d71
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.py
@@ -0,0 +1,318 @@
+# In this test we perform several failures and then test for connectivity
+# CASE1: 2x2 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures
+# CASE2: 2x2 topo + 3 ONOS + | ONOS (random instance) failure + IP connectivity test | x failures
+# CASE3: 4x4 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures
+# CASE4: 4x4 topo + 3 ONOS + | ONOS (random instance) failure + IP connectivity test | x failures
+# CASE5: 2x2 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures
+# CASE6: 2x2 topo + 3 ONOS + | ONOS (random instance) failure + Spine (random switch) failure + IP connectivity test | x failures
+# CASE7: 4x4 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures
+# CASE8: 4x4 topo + 3 ONOS + | ONOS (random instance) failure + Spine (random switch) failure + IP connectivity test | x failures
+
+
+
+class SRHighAvailability:
+
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 2x2 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause sequential ONOS failure
+ 5) Pingall
+ 6) Repeat 3), 4), 5) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS failures with 2x2 Leaf-spine "
+ main.case( description )
+ run.config(main, '2x2', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ toKill = i % main.numCtrls
+ run.killOnos( main, [ toKill ], '4', '8', '2' )
+ run.pingAll( main, 'CASE1_Failure%d' % (i+1) )
+ run.recoverOnos( main, [ toKill ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE1_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE2( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 2x2 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause random ONOS failure
+ 5) Pingall
+ 6) Repeat 3), 4), 5) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import random
+ from random import randint
+ from datetime import datetime
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS random failures with 2x2 Leaf-spine "
+ main.case( description )
+ run.config(main, '2x2', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main )
+ random.seed(datetime.now())
+ for i in range(0, main.failures):
+ toKill = randint(0, (main.numCtrls-1))
+ run.killOnos( main, [ toKill ], '4', '8', '2' )
+ run.pingAll( main, 'CASE2_Failure%d' % (i+1) )
+ run.recoverOnos( main, [ toKill ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE2_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE3( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 4x4 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause sequential ONOS failure
+ 5) Pingall
+ 6) Repeat 3), 4), 5) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS failures with 4x4 Leaf-spine "
+ main.case( description )
+ run.config(main, '4x4', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ toKill = i % main.numCtrls
+ run.killOnos( main, [ toKill ], '8', '32', '2' )
+ run.pingAll( main, 'CASE3_Failure%d' % (i+1) )
+ run.recoverOnos( main, [ toKill ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE3_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE4( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 4x4 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause random ONOS failure
+ 5) Pingall
+ 6) Repeat 3), 4), 5) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import random
+ from random import randint
+ from datetime import datetime
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS random failures with 4x4 Leaf-spine "
+ main.case( description )
+ run.config(main, '4x4', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main )
+ random.seed(datetime.now())
+ for i in range(0, main.failures):
+ toKill = randint(0, (main.numCtrls-1))
+ run.killOnos( main, [ toKill ], '8', '32', '2' )
+ run.pingAll( main, 'CASE4_Failure%d' % (i+1) )
+ run.recoverOnos( main, [ toKill ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE4_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE5( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 2x2 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause sequential ONOS failure
+ 5) Pingall
+ 6) Cause sequential Spine failure
+ 7) Pingall
+ 8) Repeat 3), 4), 5), 6), 7), 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import time
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS failures and Switch failures with 2x2 Leaf-spine "
+ main.case( description )
+ run.config(main, '2x2', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ onosToKill = i % main.numCtrls
+ switchToKill = i % len(main.spines)
+ run.killOnos( main, [ onosToKill ], '4', '8', '2' )
+ run.pingAll( main, 'CASE5_ONOS_Failure%d' % (i+1) )
+ run.killSwitch( main, main.spines[switchToKill]['name'], switches='3', links='4' )
+ time.sleep( main.switchSleep )
+ run.pingAll( main, "CASE5_SWITCH_Failure%d" % (i+1) )
+ run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='4', links='8' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, "CASE5_SWITCH_Recovery%d" % (i+1) )
+ run.recoverOnos( main, [ onosToKill ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE5_ONOS_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE6( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 2x2 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause random ONOS failure
+ 5) Pingall
+ 6) Cause random Spine failure
+ 7) Pingall
+ 8) Repeat 3), 4), 5), 6), 7) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import time
+ import random
+ from random import randint
+ from datetime import datetime
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS random failures and Switch random failures with 2x2 Leaf-spine "
+ main.case( description )
+ run.config(main, '2x2', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ onosToKill = randint(0, (main.numCtrls-1))
+ switchToKill = randint(0, 1)
+ run.killOnos( main, [ onosToKill ], '4', '8', '2' )
+ run.pingAll( main, 'CASE6_ONOS_Failure%d' % (i+1) )
+ run.killSwitch( main, main.spines[switchToKill]['name'], switches='3', links='4' )
+ time.sleep( main.switchSleep )
+ run.pingAll( main, "CASE6_SWITCH_Failure%d" % (i+1) )
+ run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='4', links='8' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, "CASE6_SWITCH_Recovery%d" % (i+1) )
+ run.recoverOnos( main, [ onosToKill ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE6_ONOS_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE7( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 4x4 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause sequential ONOS failure
+ 5) Pingall
+ 6) Cause sequential Spine failure
+ 7) Pingall
+ 8) Repeat 3), 4), 5), 6), 7), 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import time
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS failures and Switch failures with 4x4 Leaf-spine "
+ main.case( description )
+ run.config(main, '4x4', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ onosToKill = i % main.numCtrls
+ switchToKill = i % len(main.spines)
+ run.killOnos( main, [ onosToKill ], '8', '32', '2' )
+ run.pingAll( main, 'CASE7_ONOS_Failure%d' % (i+1) )
+ run.killSwitch( main, main.spines[switchToKill]['name'], switches='7', links='24' )
+ time.sleep( main.switchSleep )
+ run.pingAll( main, "CASE7_SWITCH_Failure%d" % (i+1) )
+ run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='8', links='32' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, "CASE7_SWITCH_Recovery%d" % (i+1) )
+ run.recoverOnos( main, [ onosToKill ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE7_ONOS_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE8( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 4x4 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause random ONOS failure
+ 5) Pingall
+ 6) Cause random Spine failure
+ 7) Pingall
+ 8) Repeat 3), 4), 5), 6), 7), 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import time
+ import random
+ from random import randint
+ from datetime import datetime
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS random failures and Switch random failures with 4x4 Leaf-spine "
+ main.case( description )
+ run.config(main, '4x4', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ onosToKill = randint(0, (main.numCtrls-1))
+ switchToKill = randint(0, 3)
+ run.killOnos( main, [ onosToKill ], '8', '32', '2' )
+ run.pingAll( main, 'CASE8_ONOS_Failure%d' % (i+1) )
+ run.killSwitch( main, main.spines[switchToKill]['name'], switches='7', links='24' )
+ time.sleep( main.switchSleep )
+ run.pingAll( main, "CASE8_SWITCH_Failure%d" % (i+1) )
+ run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='8', links='32' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, "CASE8_SWITCH_Recovery%d" % (i+1) )
+ run.recoverOnos( main, [ onosToKill ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE8_ONOS_Recovery%d' % (i+1) )
+ run.cleanup( main )
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
new file mode 100644
index 0000000..bf9b6be
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
@@ -0,0 +1,90 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>localhost</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ <nodes>1</nodes>
+ </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>OC1</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>OC2</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>OC3</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli3>
+
+ <ONOSrest1>
+ <host>OC1</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest1>
+
+ <ONOSrest2>
+ <host>OC2</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest2>
+
+ <ONOSrest3>
+ <host>OC3</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest3>
+
+ <Mininet1>
+ <host>OCN</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>MininetCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ <home>~/mininet/custom/</home>
+ </COMPONENTS>
+ </Mininet1>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 030ec9a..7ef28e6 100755
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -218,6 +218,51 @@
main.exit( )
@staticmethod
+ def config(main, cfgName, numCtrls):
+ main.spines = []
+
+ main.failures = int(main.params[ 'failures' ])
+ main.cfgName = cfgName
+ main.numCtrls = numCtrls
+
+ if main.cfgName == '2x2' :
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine1' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid1' ]
+ main.spines.append(spine)
+
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine2' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid2' ]
+ main.spines.append(spine)
+
+ elif main.cfgName == '4x4' :
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine1' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid1' ]
+ main.spines.append(spine)
+
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine2' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid2' ]
+ main.spines.append(spine)
+
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine3' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid3' ]
+ main.spines.append(spine)
+
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine4' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid4' ]
+ main.spines.append(spine)
+
+ else :
+ main.log.error( "Configuration failed!" )
+ main.cleanup( )
+ main.exit( )
+
+ @staticmethod
def checkFlows( main, minFlowCount, dumpflows=True ):
main.step(
" Check whether the flow count is bigger than %s" % minFlowCount )