[ONOS-7408] Refactor HA suite to be able to run with trellis
- Better support for dual homed hosts
- Support parsing more types of flows from OVS of1.3 tables
- More specific error handling in Mininet driver
- Only check attachment points if that mapping is provided
- Minor refactoring of link up/down argument names for consistency
- Use list of hosts/switches in mn instead of hard coded ranges
- Add .params.fabric for testing with fabric
- Add .params.intents for testing with intents, classic/default version
of the tests
- Add support for setting karaf log levels after startup
- Fix malformed command in cell file if no OCN is supplied
- Add back CFG for the ECFlowRuleStore now that it is the default impl
- Check Network config after connecting mininet
TODO:
- Set log levels in ONOS service files so we can set logging during startup
- Make sure we process all treatments in flows. eg drop and
clear_treatment
- Does the topology come up the same each time?
- same port numbers, etc...
- Jenkinsfiles
- use .params.fabric for HA fabric tests
Notes:
- Uses Topology and config from the SegmentRouting tests
Change-Id: I08f08ba1d3d18f710f63a45b28ac3a2868a1a5cf
diff --git a/TestON/tests/HA/HAstopNodes/HAstopNodes.py b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
index 815ab52..bbb8db7 100644
--- a/TestON/tests/HA/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HA/HAstopNodes/HAstopNodes.py
@@ -83,25 +83,56 @@
main.HA = HA()
cellName = main.params[ 'ENV' ][ 'cellName' ]
main.apps = main.params[ 'ENV' ][ 'appString' ]
- stepResult = main.testSetUp.envSetup()
+ stepResult = main.testSetUp.envSetup( includeCaseDesc=False )
except Exception as e:
main.testSetUp.envSetupException( e )
main.testSetUp.evnSetupConclusion( stepResult )
- main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
- extraApply=[ main.HA.startingMininet,
- main.HA.customizeOnosGenPartitions,
- main.HA.copyBackupConfig ],
- extraClean=main.HA.cleanUpGenPartition )
+ applyFuncs = [ main.HA.customizeOnosGenPartitions, main.HA.copyBackupConfig ]
+ applyArgs = [ None, None ]
+ try:
+ if main.params[ 'topology' ][ 'topoFile' ]:
+ main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' )
+ else:
+ applyFuncs.append( main.HA.startingMininet )
+ applyArgs.append( None )
+ except (KeyError, IndexError):
+ applyFuncs.append( main.HA.startingMininet )
+ applyArgs.append( None )
+ main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
+ extraApply=applyFuncs,
+ applyArgs=applyArgs,
+ extraClean=main.HA.cleanUpGenPartition,
+ includeCaseDesc=False )
main.HA.initialSetUp( serviceClean=True )
+ main.step( 'Set logging levels' )
+ logging = True
+ try:
+ logs = main.params.get( 'ONOS_Logging', False )
+ if logs:
+ for namespace, level in logs.items():
+ for ctrl in main.Cluster.active():
+ ctrl.CLI.logSet( level, namespace )
+ except AttributeError:
+ logging = False
+ utilities.assert_equals( expect=True, actual=logging,
+ onpass="Set log levels",
+ onfail="Failed to set log levels" )
+
def CASE2( self, main ):
"""
Assign devices to controllers
"""
main.HA.assignDevices( main )
+ def CASE102( self, main ):
+ """
+ Set up Spine-Leaf fabric topology in Mininet
+ """
+ main.HA.startTopology( main )
+
def CASE21( self, main ):
"""
Assign mastership to controllers
@@ -120,6 +151,17 @@
"""
main.HA.pingAcrossHostIntent( main )
+ def CASE104( self, main ):
+ """
+ Ping Hosts
+ """
+ main.case( "Check connectivity" )
+ main.step( "Ping between all hosts" )
+ pingResult = main.Mininet1.pingall()
+ utilities.assert_equals( expect=main.TRUE, actual=pingResult,
+ onpass="All Pings Passed",
+ onfail="Failed to ping between all hosts" )
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -162,7 +204,7 @@
nodeResults = utilities.retry( main.Cluster.nodesCheck,
False,
sleep=15,
- attempts=5 )
+ attempts=50 )
utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
@@ -181,6 +223,11 @@
The bring up stopped nodes
"""
main.HA.bringUpStoppedNodes( main )
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( main.Cluster.nodesCheck,
+ False,
+ sleep=15,
+ attempts=50 )
def CASE7( self, main ):
"""
@@ -224,7 +271,7 @@
leaderResult = main.FALSE
main.log.error(
"Inconsistent view of leader for the election test app" )
- # TODO: print the list
+ main.log.debug( leaderList )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -239,15 +286,19 @@
def CASE9( self, main ):
"""
- Link s3-s28 down
+ Link down
"""
- main.HA.linkDown( main )
+ src = main.params['kill']['linkSrc']
+ dst = main.params['kill']['linkDst']
+ main.HA.linkDown( main, src, dst )
def CASE10( self, main ):
"""
- Link s3-s28 up
+ Link up
"""
- main.HA.linkUp( main )
+ src = main.params['kill']['linkSrc']
+ dst = main.params['kill']['linkDst']
+ main.HA.linkUp( main, src, dst )
def CASE11( self, main ):
"""
@@ -267,13 +318,11 @@
"""
Clean up
"""
- main.HAlabels.append( "Restart" )
- main.HAdata.append( str( main.restartTime ) )
main.HA.cleanUp( main )
def CASE14( self, main ):
"""
- start election app on all onos nodes
+ Start election app on all onos nodes
"""
main.HA.startElectionApp( main )