Added ONOS Failure and Cluster Restart testcase
Change-Id: I3ab68b68e90e82ff37ddff09108059a6d9706f14
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/README.md b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/README.md
new file mode 100755
index 0000000..21e7773
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/README.md
@@ -0,0 +1,22 @@
+This test verifies control plane resilience from a CLUSTER restart
+
+It consists of
+
+1) Configure and Install ONOS
+2) Start Mininet and check flow state
+3) Pingall
+4) Induce a CLUSTER restart
+5) check flow state
+6) Pingall
+
+Requirements
+
+ - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+ 1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+
+The test runs for different topologies:
+ - 2x2 Leaf-Spine and 3-node ONOS cluster
+ - 4x4 Leaf-Spine and 3-node ONOS cluster
+ - Single switch and 3-node ONOS cluster
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.params b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.params
new file mode 100755
index 0000000..1a58556
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.params
@@ -0,0 +1,46 @@
+<PARAMS>
+
+ <testcases>1,2,3</testcases>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <wrapper1>startUp</wrapper1>
+ <topology>cord_fabric.py</topology>
+ </DEPENDENCY>
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,segmentrouting</cellApps>
+ <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
+ <cellUser>sdn</cellUser>
+ </ENV>
+
+ <GIT>
+ <pull>False</pull>
+ <branch>master</branch>
+ </GIT>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ </timers>
+
+ <kill>
+ <switch> spine101 </switch>
+ <dpid> 000000000101 </dpid>
+ <links> leaf1 leaf2 </links>
+ </kill>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.py b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.py
new file mode 100755
index 0000000..ae70e00
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.py
@@ -0,0 +1,106 @@
+# CASE1: 2x2 topo + 3-node ONOS CLUSTER + IP connectivity test + CLUSTER restart
+# CASE2: 4x4 topo + 3-node ONOS CLUSTER + IP connectivity test + CLUSTER restart
+# CASE3: Single switch + 3-node ONOS CLUSTER + IP connectivity test + CLUSTER restart
+
+class SRClusterRestart:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ """
+ Sets up 3-node Onos-cluster
+ Start 2x2 Leaf-Spine topology
+ Pingall
+ Induce CLUSTER restart
+ Pingall
+ """
+ description = "Cluster Restart test with 2x2 Leaf-spine "
+ main.case( description )
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+ main.cfgName = '2x2'
+ main.numCtrls = 3
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, "CASE1" )
+ run.killOnos( main, [ 0, 1, 2 ], '4', '8', '0' )
+ run.pingAll( main, 'CASE1_Failure', dumpflows=False )
+ run.recoverOnos( main, [ 0, 1, 2 ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE1_Failure' )
+ # TODO Dynamic config of hosts in subnet
+ # TODO Dynamic config of host not in subnet
+ # TODO Dynamic config of vlan xconnect
+ # TODO Vrouter integration
+ # TODO Mcast integration
+ run.cleanup( main )
+
+ def CASE2( self, main ):
+ """
+ Sets up 3-node Onos-cluster
+ Start 4x4 Leaf-Spine topology
+ Pingall
+ Induce Cluster Restart
+ Pingall
+ """
+ description = "Cluster Restart test with 4x4 Leaf-spine "
+ main.case( description )
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+ main.cfgName = '4x4'
+ main.numCtrls = 3
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE2' )
+ run.killOnos( main, [ 0, 1, 2 ], '8', '32', '0' )
+ run.pingAll( main, 'CASE2_Failure', dumpflows=False )
+ run.recoverOnos( main, [ 0, 1, 2 ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE3_Recovery' )
+ # TODO Dynamic config of hosts in subnet
+ # TODO Dynamic config of host not in subnet
+ # TODO Dynamic config of vlan xconnect
+ # TODO Vrouter integration
+ # TODO Mcast integration
+ run.cleanup( main )
+
+ def CASE3( self, main ):
+ """
+ Sets up 3-node Onos-cluster
+ Start single switch topology
+ Pingall
+ Induce Cluster Restart
+ Pingall
+ """
+ description = "Cluster Restart test with single switch "
+ main.case( description )
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+ main.cfgName = '0x1'
+ main.numCtrls = 3
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=1 --spine=0" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=15 )
+ run.pingAll( main, 'CASE3' )
+ run.killOnos( main, [ 0, 1, 2 ], '1', '0', '0' )
+ run.pingAll( main, 'CASE3_Failure', dumpflows=False )
+ run.recoverOnos( main, [ 0, 1, 2 ], '1', '0', '3' )
+ run.checkFlows( main, minFlowCount=15 )
+ run.pingAll( main, 'CASE3_Failure' )
+ # TODO Dynamic config of hosts in subnet
+ # TODO Dynamic config of host not in subnet
+ # TODO Dynamic config of vlan xconnect
+ # TODO Vrouter integration
+ # TODO Mcast integration
+ run.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
new file mode 100755
index 0000000..c531f7d
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/SRClusterRestart.topo
@@ -0,0 +1,90 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>localhost</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ <nodes>1</nodes>
+ </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>OC1</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>OC2</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>OC3</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli3>
+
+ <ONOSrest1>
+ <host>OC1</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest1>
+
+ <ONOSrest2>
+ <host>OC2</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest2>
+
+ <ONOSrest3>
+ <host>OC3</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest3>
+
+ <Mininet1>
+ <host>OCN</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>MininetCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ <home>~/mininet/custom/</home>
+ </COMPONENTS>
+ </Mininet1>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/__init__.py
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRClusterRestart/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/README.md b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/README.md
new file mode 100755
index 0000000..94677cb
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/README.md
@@ -0,0 +1,22 @@
+This test verifies basic control plane resilience from an ONOS Instance failure using SegmentRouting via pingall
+
+It consists of
+
+1) Configure and Install ONOS
+2) Start Mininet and check flow state
+3) Pingall
+4) Induce a ONOS failure
+5) check flow state
+6) Pingall
+
+Requirements
+
+ - An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+ 1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+
+The test runs for different topologies:
+ - 2x2 Leaf-Spine and 3-node ONOS cluster
+ - 4x4 Leaf-Spine and 3-node ONOS cluster
+ - Single switch and 3-node ONOS cluster
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params
new file mode 100755
index 0000000..1a58556
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.params
@@ -0,0 +1,46 @@
+<PARAMS>
+
+ <testcases>1,2,3</testcases>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <wrapper1>startUp</wrapper1>
+ <topology>cord_fabric.py</topology>
+ </DEPENDENCY>
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,segmentrouting</cellApps>
+ <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
+ <cellUser>sdn</cellUser>
+ </ENV>
+
+ <GIT>
+ <pull>False</pull>
+ <branch>master</branch>
+ </GIT>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ </timers>
+
+ <kill>
+ <switch> spine101 </switch>
+ <dpid> 000000000101 </dpid>
+ <links> leaf1 leaf2 </links>
+ </kill>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.py b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.py
new file mode 100755
index 0000000..53daa33
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.py
@@ -0,0 +1,106 @@
+# CASE1: 2x2 topo + 3-node ONOS CLUSTER + IP connectivity test + Control plane resilience
+# CASE2: 4x4 topo + 3-node ONOS CLUSTER + IP connectivity test + Control plane resilience
+# CASE3: Single switch + 3-node ONOS CLUSTER + IP connectivity test + Control plane resilience
+
+class SROnosFailure:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ """
+ Sets up 3-node Onos-cluster
+ Start 2x2 Leaf-Spine topology
+ Pingall
+ Induce ONOS Failure
+ Pingall
+ """
+ description = "ONOS Failure test with 2x2 Leaf-spine "
+ main.case( description )
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+ main.cfgName = '2x2'
+ main.numCtrls = 3
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, "CASE1" )
+ run.killOnos( main, [ 0 ], '4', '8', '2' )
+ run.pingAll( main, 'CASE1_Failure' )
+ run.recoverOnos( main, [ 0 ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE1_Failure' )
+ # TODO Dynamic config of hosts in subnet
+ # TODO Dynamic config of host not in subnet
+ # TODO Dynamic config of vlan xconnect
+ # TODO Vrouter integration
+ # TODO Mcast integration
+ run.cleanup( main )
+
+ def CASE2( self, main ):
+ """
+ Sets up 3-node Onos-cluster
+ Start 4x4 Leaf-Spine topology
+ Pingall
+ Induce ONOS Failure
+ Pingall
+ """
+ description = "ONOS Failure test with 4x4 Leaf-spine "
+ main.case( description )
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+ main.cfgName = '4x4'
+ main.numCtrls = 3
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE2' )
+ run.killOnos( main, [ 0 ], '8', '32', '2' )
+ run.pingAll( main, 'CASE2_Failure' )
+ run.recoverOnos( main, [ 0 ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE3_Recovery' )
+ # TODO Dynamic config of hosts in subnet
+ # TODO Dynamic config of host not in subnet
+ # TODO Dynamic config of vlan xconnect
+ # TODO Vrouter integration
+ # TODO Mcast integration
+ run.cleanup( main )
+
+ def CASE3( self, main ):
+ """
+ Sets up 3-node Onos-cluster
+ Start single switch topology
+ Pingall
+ Induce ONOS Failure
+ Pingall
+ """
+ description = "ONOS Failure test with single switch "
+ main.case( description )
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+ main.cfgName = '0x1'
+ main.numCtrls = 3
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=1 --spine=0" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=15 )
+ run.pingAll( main, 'CASE3' )
+ run.killOnos( main, [ 0 ], '1', '0', '2' )
+ run.pingAll( main, 'CASE3_Failure' )
+ run.recoverOnos( main, [ 0 ], '1', '0', '3' )
+ run.checkFlows( main, minFlowCount=15 )
+ run.pingAll( main, 'CASE3_Failure' )
+ # TODO Dynamic config of hosts in subnet
+ # TODO Dynamic config of host not in subnet
+ # TODO Dynamic config of vlan xconnect
+ # TODO Vrouter integration
+ # TODO Mcast integration
+ run.cleanup( main )
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
new file mode 100755
index 0000000..c531f7d
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/SROnosFailure.topo
@@ -0,0 +1,90 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>localhost</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ <nodes>1</nodes>
+ </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>OC1</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>OC2</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>OC3</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli3>
+
+ <ONOSrest1>
+ <host>OC1</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest1>
+
+ <ONOSrest2>
+ <host>OC2</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest2>
+
+ <ONOSrest3>
+ <host>OC3</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest3>
+
+ <Mininet1>
+ <host>OCN</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>MininetCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ <home>~/mininet/custom/</home>
+ </COMPONENTS>
+ </Mininet1>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/SROnosFailure/__init__.py b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/__init__.py
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SROnosFailure/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
index 59c1214..e20287e 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
+++ b/TestON/tests/USECASE/SegmentRouting/SRSanity/README.md
@@ -1,7 +1,6 @@
-This test verifies basic connectivity using SegmentRouting via pingall,
-it should not fail.
+This test verifies basic IP connectivity using SegmentRouting via pingall
-It consists of
+It consists of
1) Configure and Install ONOS
2) Start Mininet and check flow state
@@ -13,4 +12,10 @@
The test is executed using the netcfg subsystem:
1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
-The topology is a 2x2 Leaf-spine
+The test runs for different topologies:
+ - 2x2 Leaf-Spine and 1-node ONOS cluster
+ - 4x4 Leaf-Spine and 1-node ONOS cluster
+ - Single switch and 1-node ONOS cluster
+ - 2x2 Leaf-Spine and 3-node ONOS cluster
+ - 4x4 Leaf-Spine and 3-node ONOS cluster
+ - Single switch and 3-node ONOS cluster
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 395d747..8565930 100755
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -118,7 +118,7 @@
for i in range( main.numCtrls ):
onosInstallResult = onosInstallResult and \
main.ONOSbench.onosInstall(
- node=main.ONOSip[ i ] )
+ node=main.ONOSip[ i ] )
stepResult = onosInstallResult
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
@@ -189,7 +189,7 @@
arg = "--onos %d %s" % (main.numCtrls, args)
main.topology = topology
topoResult = main.Mininet1.startNet(
- topoFile=main.dependencyPath + main.topology, args=arg )
+ topoFile=main.dependencyPath + main.topology, args=arg )
stepResult = topoResult
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
@@ -203,7 +203,7 @@
@staticmethod
def checkFlows( main, minFlowCount ):
main.step(
- " Check whether the flow count is bigger than %s" % minFlowCount )
+ " Check whether the flow count is bigger than %s" % minFlowCount )
count = utilities.retry( main.CLIs[ main.active ].checkFlowCount,
main.FALSE,
kwargs={ 'min': minFlowCount },
@@ -232,17 +232,18 @@
main.logdir, "groupsBefore" + main.cfgName )
@staticmethod
- def pingAll( main, tag="" ):
+ def pingAll( main, tag="", dumpflows=True ):
main.log.report( "Check full connectivity" )
main.step( "Check full connectivity %s" % tag )
pa = main.Mininet1.pingall( )
utilities.assert_equals( expect=main.TRUE, actual=pa,
onpass="Full connectivity successfully tested",
onfail="Full connectivity failed" )
- main.ONOSbench.dumpFlows( main.ONOSip[ main.active ],
- main.logdir, "flowsOn" + tag )
- main.ONOSbench.dumpGroups( main.ONOSip[ main.active ],
- main.logdir, "groupsOn" + tag )
+ if dumpflows:
+ main.ONOSbench.dumpFlows( main.ONOSip[ main.active ],
+ main.logdir, "flowsOn" + tag )
+ main.ONOSbench.dumpGroups( main.ONOSip[ main.active ],
+ main.logdir, "groupsOn" + tag )
@staticmethod
def killLink( main, end1, end2, switches, links ):
@@ -255,7 +256,7 @@
main.step( "Kill link between %s and %s" % (end1, end2) )
LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
main.log.info(
- "Waiting %s seconds for link down to be discovered" % main.linkSleep )
+ "Waiting %s seconds for link down to be discovered" % main.linkSleep )
time.sleep( main.linkSleep )
topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
main.FALSE,
@@ -287,7 +288,7 @@
main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
main.log.info(
- "Waiting %s seconds for link up to be discovered" % main.linkSleep )
+ "Waiting %s seconds for link up to be discovered" % main.linkSleep )
time.sleep( main.linkSleep )
main.CLIs[ main.active ].portstate( dpid=dpid1, port=port1 )
main.CLIs[ main.active ].portstate( dpid=dpid2, port=port2 )
@@ -313,7 +314,7 @@
main.Mininet1.switch( SW=switch, OPTION="stop" )
# todo make this repeatable
main.log.info( "Waiting %s seconds for switch down to be discovered" % (
- main.switchSleep) )
+ main.switchSleep) )
time.sleep( main.switchSleep )
topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
main.FALSE,
@@ -336,7 +337,7 @@
main.log.info( "Starting" + switch )
main.Mininet1.switch( SW=switch, OPTION="start" )
main.log.info( "Waiting %s seconds for switch up to be discovered" % (
- main.switchSleep) )
+ main.switchSleep) )
time.sleep( main.switchSleep )
topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
main.FALSE,
@@ -362,3 +363,86 @@
copyFileName="karaf.log." + main.cfgName )
for i in range( main.numCtrls ):
main.ONOSbench.onosStop( main.ONOSip[ i ] )
+
+ @staticmethod
+ def killOnos( main, nodes, switches, links, expNodes ):
+ """
+ Params: nodes, integer array with position of the ONOS nodes in the CLIs array
+ switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
+ Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
+ """
+ main.step( "Killing ONOS instance" )
+ for i in nodes:
+ killResult = main.ONOSbench.onosDie( main.CLIs[ i ].ip_address )
+ utilities.assert_equals( expect=main.TRUE, actual=killResult,
+ onpass="ONOS instance Killed",
+ onfail="Error killing ONOS instance" )
+ if i == main.active:
+ main.active = (i + 1) % main.numCtrls
+ time.sleep( 12 )
+ if len( nodes ) < main.numCtrls:
+ topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
+ main.FALSE,
+ kwargs={ 'numoswitch': switches,
+ 'numolink': links,
+ 'numoctrl': expNodes },
+ attempts=10,
+ sleep=12 )
+ utilities.assert_equals( expect=main.TRUE, actual=topology,
+ onpass="ONOS Instance down successful",
+ onfail="Failed to turn off ONOS Instance" )
+ else:
+ main.active = -1
+
+ @staticmethod
+ def recoverOnos( main, nodes, switches, links, expNodes ):
+ """
+ Params: nodes, integer array with position of the ONOS nodes in the CLIs array
+ switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
+ Recover an ONOS instance and verify the ONOS cluster can see the proper change
+ """
+ main.step( "Recovering ONOS instance" )
+ [ main.ONOSbench.onosStart( main.CLIs[ i ].ip_address ) for i in nodes ]
+ for i in nodes:
+ isUp = main.ONOSbench.isup( main.ONOSip[ i ] )
+ utilities.assert_equals( expect=main.TRUE, actual=isUp,
+ onpass="ONOS service is ready",
+ onfail="ONOS service did not start properly" )
+ for i in nodes:
+ main.step( "Checking if ONOS CLI is ready" )
+ main.CLIs[ i ].startCellCli( )
+ cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
+ commandlineTimeout=60,
+ onosStartTimeout=100 )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cliResult,
+ onpass="ONOS CLI is ready",
+ onfail="ONOS CLI is not ready" )
+ main.active = i if main.active == -1 else main.active
+
+ topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
+ main.FALSE,
+ kwargs={ 'numoswitch': switches,
+ 'numolink': links,
+ 'numoctrl': expNodes },
+ attempts=10,
+ sleep=12 )
+ utilities.assert_equals( expect=main.TRUE, actual=topology,
+ onpass="ONOS Instance down successful",
+ onfail="Failed to turn off ONOS Instance" )
+
+ for i in range( 10 ):
+ ready = True
+ output = main.CLIs[ main.active ].summary( )
+ if not output:
+ ready = False
+ if ready:
+ break
+ time.sleep( 10 )
+ utilities.assert_equals( expect=True, actual=ready,
+ onpass="ONOS summary command succeded",
+ onfail="ONOS summary command failed" )
+ if not ready:
+ main.log.error( "ONOS startup failed!" )
+ main.cleanup( )
+ main.exit( )