Refactor HA tests
Move more code into functions
Retry some functions
Rename class file
Enable FlowObjective intent compiler
Cleanup flowTableComp and doen't short circuit in Mininet driver
Change-Id: I2a404b32469d137370e512a1c44f06dd8e9df23b
diff --git a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index 6c2b01f..de14bf9 100644
--- a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -70,8 +70,8 @@
main.numCtrls = int( main.ONOSbench.maxNodes )
try:
- from tests.HAsanity.dependencies.Counters import Counters
- main.Counters = Counters()
+ from tests.HAsanity.dependencies.HA import HA
+ main.HA = HA()
except Exception as e:
main.log.exception( e )
main.cleanup()
@@ -241,38 +241,12 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
@@ -919,6 +893,7 @@
"functionality and check the state of " +\
"the intent"
+ onosCli = main.CLIs[ main.activeNodes[0] ]
main.step( "Check Intent state" )
installedCheck = True
# Print the intent states
@@ -948,7 +923,6 @@
"INSTALLED state" )
main.step( "Ping across added host intents" )
- onosCli = main.CLIs[ main.activeNodes[0] ]
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -1601,8 +1575,9 @@
for i in range( 28 ):
main.log.info( "Checking flow table on s" + str( i + 1 ) )
tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
- FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
- if FlowTables == main.FALSE:
+ curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
+ FlowTables = FlowTables and curSwitch
+ if curSwitch == main.FALSE:
main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
utilities.assert_equals(
expect=main.TRUE,
@@ -1865,6 +1840,20 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
+
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+ if not nodeResults:
+ for i in main.activeNodes:
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ main.CLIs[i].name,
+ main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
def CASE9( self, main ):
"""
@@ -2152,26 +2141,8 @@
main.step( "Check that each node shows the same leader and candidates" )
failMessage = "Nodes have different leaderboards"
- def consistentLeaderboards( nodes ):
- TOPIC = 'org.onosproject.election'
- # FIXME: use threads
- #FIXME: should we retry outside the function?
- for n in range( 5 ): # Retry in case election is still happening
- leaderList = []
- # Get all leaderboards
- for cli in nodes:
- leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
- # Compare leaderboards
- result = all( i == leaderList[0] for i in leaderList ) and\
- leaderList is not None
- main.log.debug( leaderList )
- main.log.warn( result )
- if result:
- return ( result, leaderList )
- time.sleep(5) #TODO: paramerterize
- main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
- sameResult, oldLeaders = consistentLeaderboards( activeCLIs )
+ sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
if sameResult:
oldLeader = oldLeaders[ 0 ][ 0 ]
main.log.warn( oldLeader )
@@ -2207,7 +2178,7 @@
main.step( "Check that a new node was elected leader" )
failMessage = "Nodes have different leaders"
# Get new leaders and candidates
- newLeaderResult, newLeaders = consistentLeaderboards( activeCLIs )
+ newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
if newLeaders[ 0 ][ 0 ] == 'none':
main.log.error( "No leader was elected on at least 1 node" )
if not expectNoLeader:
@@ -2275,7 +2246,7 @@
# Get new leaders and candidates
reRunLeaders = []
time.sleep( 5 ) # Paremterize
- positionResult, reRunLeaders = consistentLeaderboards( activeCLIs )
+ positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
# Check that the re-elected node is last on the candidate List
if oldLeader != reRunLeaders[ 0 ][ -1 ]:
@@ -2419,7 +2390,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
@@ -2519,7 +2490,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",