[ONOS-7408] Refactor HA suite to be able to run with trellis
- Better support for dual homed hosts
- Support parsing more types of flows from OVS of1.3 tables
- More specific error handling in Mininet driver
- Only check attachment points if that mapping is provided
- Minor refactoring of link up/down argument names for consistency
- Use list of hosts/switches in mn instead of hard coded ranges
- Add .params.fabric for testing with fabric
- Add .params.intents for testing with intents, classic/default version
of the tests
- Add support for setting karaf log levels after startup
- Fix malformed command in cell file if no OCN is supplied
- Add back CFG for the ECFlowRuleStore now that it is the default impl
- Check Network config after connecting mininet
TODO:
- Set log levels in ONOS service files so we can set logging during startup
- Make sure we process all treatments in flows. eg drop and
clear_treatment
- Does the topology come up the same each time?
- same port numbers, etc...
- Jenkinsfiles
- use .params.fabric for HA fabric tests
Notes:
- Uses Topology and config from the SegmentRouting tests
Change-Id: I08f08ba1d3d18f710f63a45b28ac3a2868a1a5cf
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.params b/TestON/tests/HA/HAupgrade/HAupgrade.params
index 3379327..a67e7cb 100644
--- a/TestON/tests/HA/HAupgrade/HAupgrade.params
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.params
@@ -1,25 +1,30 @@
<PARAMS>
#List of test cases:
- #CASE1: Compile ONOS and push it to the test machines
- #CASE2: Assign devices to controllers
- #CASE21: Assign mastership to controllers
- #CASE3: Assign intents
- #CASE4: Ping across added host intents
- #CASE5: Reading state of ONOS
- #CASE61: The Failure inducing case.
- #CASE62: The Failure recovery case.
- #CASE7: Check state after control plane failure
- #CASE8: Compare topo
- #CASE9: Link s3-s28 down
- #CASE10: Link s3-s28 up
- #CASE11: Switch down
- #CASE12: Switch up
- #CASE13: Clean up
- #CASE14: start election app on all onos nodes
- #CASE15: Check that Leadership Election is still functional
- #CASE16: Install Distributed Primitives app
- #CASE17: Check for basic functionality with distributed primitives
- <testcases>1,2,8,21,[3,4,5,14,16,17]*1,[60,8,61,8,62,8,63,8,64,8,7,4,15,17],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ #CASE1: Compile ONOS and push it to the test machines
+ #CASE2: Assign mastership to controllers
+ #CASE21: Assign mastership to controllers
+ #CASE102: Start Spine-Leaf Topology in Mininet
+ #CASE3: Assign intents
+ #CASE4: Ping across added host intents
+ #CASE104: Ping between all hosts
+ #CASE5: Reading state of ONOS
+ #CASE60: Initialize the upgrade.
+ #CASE61: Upgrade a minority of nodes PHASE 1
+ #CASE62: Transfer to new version. PHASE 2
+ #CASE63: Upgrade the rest of the nodes
+ #CASE64: Commit to the upgrade.
+ #CASE7: Check state after control plane failure
+ #CASE8: Compare topo
+ #CASE9: Link down
+ #CASE10: Link up
+ #CASE11: Switch down
+ #CASE12: Switch up
+ #CASE13: Clean up
+ #CASE14: Start election app on all onos nodes
+ #CASE15: Check that Leadership Election is still functional
+ #CASE16: Install Distributed Primitives app
+ #CASE17: Check for basic functionality with distributed primitives
+ <testcases>1,2,8,21,[3,4,5,14,15,16,17]*1,[60,8,61,8,62,8,63,8,64,8,7,4,15,17],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<GRAPH>
<nodeCluster>VM</nodeCluster>
@@ -31,10 +36,16 @@
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
<useFlowObjectives>false</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.ECFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.ECFlowRuleStore>
</ONOS_Configuration>
+ <ONOS_Logging>
+ <org.onosproject.events>TRACE</org.onosproject.events>
+ </ONOS_Logging>
<ENV>
<cellName>HA</cellName>
- <appString>drivers,openflow,proxyarp,mobility</appString>
+ <appString>events,drivers,openflow,proxyarp,mobility</appString>
</ENV>
<GIT>
<pull>False</pull>
@@ -85,6 +96,8 @@
<gossip>5</gossip>
</timers>
<kill>
+ <linkSrc> s28 </linkSrc>
+ <linkDst> s3 </linkDst>
<switch> s5 </switch>
<dpid> 0000000000005000 </dpid>
<links> h5 s2 s1 s6 </links>
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.params.fabric b/TestON/tests/HA/HAupgrade/HAupgrade.params.fabric
new file mode 100644
index 0000000..bedea08
--- /dev/null
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.params.fabric
@@ -0,0 +1,102 @@
+<PARAMS>
+ #List of test cases:
+ #CASE1: Compile ONOS and push it to the test machines
+ #CASE2: Assign mastership to controllers
+ #CASE21: Assign mastership to controllers
+ #CASE102: Start Spine-Leaf Topology in Mininet
+ #CASE3: Assign intents
+ #CASE4: Ping across added host intents
+ #CASE104: Ping between all hosts
+ #CASE5: Reading state of ONOS
+ #CASE60: Initialize the upgrade.
+ #CASE61: Upgrade a minority of nodes PHASE 1
+ #CASE62: Transfer to new version. PHASE 2
+ #CASE63: Upgrade the rest of the nodes
+ #CASE64: Commit to the upgrade.
+ #CASE7: Check state after control plane failure
+ #CASE8: Compare topo
+ #CASE9: Link down
+ #CASE10: Link up
+ #CASE11: Switch down
+ #CASE12: Switch up
+ #CASE13: Clean up
+ #CASE14: Start election app on all onos nodes
+ #CASE15: Check that Leadership Election is still functional
+ #CASE16: Install Distributed Primitives app
+ #CASE17: Check for basic functionality with distributed primitives
+ <testcases>1,102,8,104,[5,14,15,16,17]*1,[60,8,61,8,62,8,63,8,64,8,7,104,15,17],8,7,104,15,17,9,8,104,10,8,104,[11,8,104,12,8,104]*0,13</testcases>
+
+ <GRAPH>
+ <nodeCluster>VM</nodeCluster>
+ <builds>20</builds>
+ </GRAPH>
+
+ <apps></apps>
+ <ONOS_Configuration>
+ <org.onosproject.store.flow.impl.ECFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.ECFlowRuleStore>
+ </ONOS_Configuration>
+ <ONOS_Logging>
+ <org.onosproject.events>TRACE</org.onosproject.events>
+ <org.onosproject.segmentrouting>DEBUG</org.onosproject.segmentrouting>
+ <org.onosproject.driver.pipeline>DEBUG</org.onosproject.driver.pipeline>
+ <org.onosproject.store.group.impl>DEBUG</org.onosproject.store.group.impl>
+ <org.onosproject.net.flowobjective.impl>DEBUG</org.onosproject.net.flowobjective.impl>
+ </ONOS_Logging>
+ <ENV>
+ <cellName>HA</cellName>
+ <appString>events,drivers,openflow,segmentrouting,netcfghostprovider</appString>
+ </ENV>
+ <GIT>
+ <pull>False</pull>
+ <branch>master</branch>
+ </GIT>
+ <num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
+
+ <topology>
+ <files>
+ <topo>~/TestON/tests/USECASE/SegmentRouting/dependencies/comcast_fabric.py</topo>
+ <dep1>~/TestON/tests/USECASE/SegmentRouting/dependencies/trellislib.py</dep1>
+ <dep2>~/TestON/tests/USECASE/SegmentRouting/dependencies/routinglib.py</dep2>
+ </files>
+ <topoFile>comcast_fabric.py</topoFile>
+ <args> --dhcp=0 --routers=0 --ipv6=0 --ipv4=1 </args>
+ <configPath>/USECASE/SegmentRouting/SRRouting/dependencies/json/</configPath>
+ <configName>COMCAST_CONFIG_ipv4=1_ipv6=0_dhcp=0_routers=0.json</configName>
+ </topology>
+ <CTRL>
+ <port1>6653</port1>
+ <port2>6653</port2>
+ <port3>6653</port3>
+ <port4>6653</port4>
+ <port5>6653</port5>
+ <port6>6653</port6>
+ <port7>6653</port7>
+ </CTRL>
+ <BACKUP>
+ <ENABLED> False </ENABLED>
+ <TESTONUSER>sdn</TESTONUSER>
+ <TESTONIP>10.128.30.9</TESTONIP>
+ </BACKUP>
+ <timers>
+ <NetCfg>5</NetCfg>
+ <SRSetup>60</SRSetup>
+ <LinkDiscovery>60</LinkDiscovery>
+ <SwitchDiscovery>60</SwitchDiscovery>
+ <gossip>5</gossip>
+ </timers>
+ <kill>
+ <linkSrc>spine102</linkSrc>
+ <linkDst>leaf1</linkDst>
+ <switch> spine101 </switch>
+ <dpid> 0000000000000101 </dpid>
+ <links> leaf1 leaf2 leaf2 leaf3 leaf3 leaf4 leaf4 leaf5 leaf5 </links>
+ </kill>
+ <MNtcpdump>
+ <intf>eth0</intf>
+ <port> </port>
+ <folder>~/packet_captures/</folder>
+ </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.params.intents b/TestON/tests/HA/HAupgrade/HAupgrade.params.intents
new file mode 100644
index 0000000..24dfa7e
--- /dev/null
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.params.intents
@@ -0,0 +1,111 @@
+<PARAMS>
+ #List of test cases:
+ #CASE1: Compile ONOS and push it to the test machines
+ #CASE2: Assign mastership to controllers
+ #CASE21: Assign mastership to controllers
+ #CASE102: Start Spine-Leaf Topology in Mininet
+ #CASE3: Assign intents
+ #CASE4: Ping across added host intents
+ #CASE104: Ping between all hosts
+ #CASE5: Reading state of ONOS
+ #CASE60: Initialize the upgrade.
+ #CASE61: Upgrade a minority of nodes PHASE 1
+ #CASE62: Transfer to new version. PHASE 2
+ #CASE63: Upgrade the rest of the nodes
+ #CASE64: Commit to the upgrade.
+ #CASE7: Check state after control plane failure
+ #CASE8: Compare topo
+ #CASE9: Link down
+ #CASE10: Link up
+ #CASE11: Switch down
+ #CASE12: Switch up
+ #CASE13: Clean up
+ #CASE14: Start election app on all onos nodes
+ #CASE15: Check that Leadership Election is still functional
+ #CASE16: Install Distributed Primitives app
+ #CASE17: Check for basic functionality with distributed primitives
+ <testcases>1,2,8,21,[3,4,5,14,15,16,17]*1,[60,8,61,8,62,8,63,8,64,8,7,4,15,17],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+
+ <GRAPH>
+ <nodeCluster>VM</nodeCluster>
+ <builds>20</builds>
+ </GRAPH>
+
+ <apps></apps>
+ <ONOS_Configuration>
+ <org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <useFlowObjectives>false</useFlowObjectives>
+ <defaultFlowObjectiveCompiler>org.onosproject.net.intent.impl.compiler.LinkCollectionIntentObjectiveCompiler</defaultFlowObjectiveCompiler>
+ </org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+ <org.onosproject.store.flow.impl.ECFlowRuleStore>
+ <backupCount>3</backupCount>
+ </org.onosproject.store.flow.impl.ECFlowRuleStore>
+ </ONOS_Configuration>
+ <ONOS_Logging>
+ <org.onosproject.events>TRACE</org.onosproject.events>
+ </ONOS_Logging>
+ <ENV>
+ <cellName>HA</cellName>
+ <appString>events,drivers,openflow,proxyarp,mobility</appString>
+ </ENV>
+ <GIT>
+ <pull>False</pull>
+ <branch>master</branch>
+ </GIT>
+ <num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
+
+ <CTRL>
+ <port1>6653</port1>
+ <port2>6653</port2>
+ <port3>6653</port3>
+ <port4>6653</port4>
+ <port5>6653</port5>
+ <port6>6653</port6>
+ <port7>6653</port7>
+ </CTRL>
+ <BACKUP>
+ <ENABLED> False </ENABLED>
+ <TESTONUSER>sdn</TESTONUSER>
+ <TESTONIP>10.128.30.9</TESTONIP>
+ </BACKUP>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ <gossip>5</gossip>
+ </timers>
+ <kill>
+ <linkSrc> s28 </linkSrc>
+ <linkDst> s3 </linkDst>
+ <switch> s5 </switch>
+ <dpid> 0000000000005000 </dpid>
+ <links> h5 s2 s1 s6 </links>
+ </kill>
+ <MNtcpdump>
+ <intf>eth0</intf>
+ <port> </port>
+ <folder>~/packet_captures/</folder>
+ </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.py b/TestON/tests/HA/HAupgrade/HAupgrade.py
index 2254bc3..8122044 100644
--- a/TestON/tests/HA/HAupgrade/HAupgrade.py
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.py
@@ -30,8 +30,11 @@
CASE3: Assign intents
CASE4: Ping across added host intents
CASE5: Reading state of ONOS
-CASE61: The Failure inducing case.
-CASE62: The Failure recovery case.
+CASE60: Initialize the upgrade.
+CASE61: Upgrade a minority of nodes PHASE 1
+CASE62: Transfer to new version. PHASE 2
+CASE63: Upgrade the rest of the nodes
+CASE64: Commit to the upgrade.
CASE7: Check state after control plane failure
CASE8: Compare topo
CASE9: Link s3-s28 down
@@ -84,25 +87,57 @@
main.HA = HA()
cellName = main.params[ 'ENV' ][ 'cellName' ]
main.apps = main.params[ 'ENV' ][ 'appString' ]
- stepResult = main.testSetUp.envSetup()
+ stepResult = main.testSetUp.envSetup( includeCaseDesc=False )
except Exception as e:
main.testSetUp.envSetupException( e )
main.testSetUp.evnSetupConclusion( stepResult )
- main.HA.generateGraph( "HAupgrade" )
+
+ applyFuncs = [ main.HA.copyBackupConfig ]
+ applyArgs = [ None ]
+ try:
+ if main.params[ 'topology' ][ 'topoFile' ]:
+ main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
+ else:
+ applyFuncs.append( main.HA.startingMininet )
+ applyArgs.append( None )
+ except (KeyError, IndexError):
+ applyFuncs.append( main.HA.startingMininet )
+ applyArgs.append( None )
main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
- extraApply=[ main.HA.startingMininet,
- main.HA.copyBackupConfig ],
- extraClean=main.HA.cleanUpGenPartition )
+ extraApply=applyFuncs,
+ applyArgs=applyArgs,
+ extraClean=main.HA.cleanUpGenPartition,
+ includeCaseDesc=False )
main.HA.initialSetUp( serviceClean=True )
+ main.step( 'Set logging levels' )
+ logging = True
+ try:
+ logs = main.params.get( 'ONOS_Logging', False )
+ if logs:
+ for namespace, level in logs.items():
+ for ctrl in main.Cluster.active():
+ ctrl.CLI.logSet( level, namespace )
+ except AttributeError:
+ logging = False
+ utilities.assert_equals( expect=True, actual=logging,
+ onpass="Set log levels",
+ onfail="Failed to set log levels" )
+
def CASE2( self, main ):
"""
Assign devices to controllers
"""
main.HA.assignDevices( main )
+ def CASE102( self, main ):
+ """
+ Set up Spine-Leaf fabric topology in Mininet
+ """
+ main.HA.startTopology( main )
+
def CASE21( self, main ):
"""
Assign mastership to controllers
@@ -121,6 +156,17 @@
"""
main.HA.pingAcrossHostIntent( main )
+ def CASE104( self, main ):
+ """
+ Ping Hosts
+ """
+ main.case( "Check connectivity" )
+ main.step( "Ping between all hosts" )
+ pingResult = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pingResult,
+ onpass="All Pings Passed",
+ onfail="Failed to ping between all hosts" )
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -215,6 +261,15 @@
main.kill.append( node )
main.HA.upgradeNodes( main )
+ main.step( "Checking ONOS nodes" )
+
+ nodeResults = utilities.retry( main.Cluster.nodesCheck,
+ False,
+ sleep=15,
+ attempts=5 )
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
def CASE64( self, main ):
"""
@@ -237,6 +292,15 @@
main.log.debug( status )
# TODO: check things here?
+ nodeResults = utilities.retry( main.Cluster.nodesCheck,
+ False,
+ sleep=15,
+ attempts=5 )
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+
+
def CASE7( self, main ):
"""
Check state after ONOS failure
@@ -279,7 +343,7 @@
leaderResult = main.FALSE
main.log.error(
"Inconsistent view of leader for the election test app" )
- # TODO: print the list
+ main.log.debug( leaderList )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -294,15 +358,19 @@
def CASE9( self, main ):
"""
- Link s3-s28 down
+ Link down
"""
- main.HA.linkDown( main )
+ src = main.params['kill']['linkSrc']
+ dst = main.params['kill']['linkDst']
+ main.HA.linkDown( main, src, dst )
def CASE10( self, main ):
"""
- Link s3-s28 up
+ Link up
"""
- main.HA.linkUp( main )
+ src = main.params['kill']['linkSrc']
+ dst = main.params['kill']['linkDst']
+ main.HA.linkUp( main, src, dst )
def CASE11( self, main ):
"""
@@ -328,7 +396,7 @@
def CASE14( self, main ):
"""
- start election app on all onos nodes
+ Start election app on all onos nodes
"""
main.HA.startElectionApp( main )