Harden HA tests against bad ONOS output
- also update gen partitions files
Change-Id: I9fbed42971650aa7feb050aafa66a37b1818804c
diff --git a/TestON/tests/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HAclusterRestart/HAclusterRestart.py
index 79d575b..a745e06 100644
--- a/TestON/tests/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HAclusterRestart/HAclusterRestart.py
@@ -2149,15 +2149,13 @@
main.case( "Compare ONOS Topology view to Mininet topology" )
main.caseExplanation = "Compare topology objects between Mininet" +\
" and ONOS"
-
- main.step( "Comparing ONOS topology to MN" )
topoResult = main.FALSE
elapsed = 0
count = 0
- main.step( "Collecting topology information from ONOS" )
+ main.step( "Comparing ONOS topology to MN topology" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
devicesResults = main.TRUE
linksResults = main.TRUE
hostsResults = main.TRUE
@@ -2193,6 +2191,7 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing hosts results" )
main.log.error( repr( t.result ) )
+ hosts.append( [] )
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
@@ -2383,6 +2382,11 @@
topoResult = ( devicesResults and linksResults
and hostsResults and ipResult and
hostAttachmentResults )
+ utilities.assert_equals( expect=True,
+ actual=topoResult,
+ onpass="ONOS topology matches Mininet",
+ onfail="ONOS topology don't match Mininet" )
+ # End of While loop to pull ONOS state
# Compare json objects for hosts and dataplane clusters
@@ -2515,26 +2519,23 @@
t.join()
nodesOutput.append( t.result )
ips = [ node.ip_address for node in main.nodes ]
+ ips.sort()
for i in nodesOutput:
try:
current = json.loads( i )
+ activeIps = []
+ currentResult = main.FALSE
for node in current:
- currentResult = main.FALSE
- if node['ip'] in ips: # node in nodes() output is in cell
- if node['state'] == 'ACTIVE':
- currentResult = main.TRUE
- else:
- main.log.error( "Error in ONOS node availability" )
- main.log.error(
- json.dumps( current,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- break
- nodeResults = nodeResults and currentResult
+ if node['state'] == 'ACTIVE':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = main.TRUE
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
+ currentResult = main.FALSE
+ nodeResults = nodeResults and currentResult
utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
diff --git a/TestON/tests/HAclusterRestart/dependencies/Counters.py b/TestON/tests/HAclusterRestart/dependencies/Counters.py
index 6614887..2dc95e1 100644
--- a/TestON/tests/HAclusterRestart/dependencies/Counters.py
+++ b/TestON/tests/HAclusterRestart/dependencies/Counters.py
@@ -11,56 +11,62 @@
nodes or main.FALSE
"""
import json
- correctResults = main.TRUE
- # Get onos counters results
- onosCountersRaw = []
- threads = []
- for i in range( main.numCtrls ):
- t = main.Thread( target=main.CLIs[i].counters,
- name="counters-" + str( i ) )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCountersRaw.append( t.result )
- onosCounters = []
- for i in range( main.numCtrls ):
- try:
- onosCounters.append( json.loads( onosCountersRaw[i] ) )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( i + 1 ) )
- main.log.warn( repr( onosCountersRaw[ i ] ) )
- return main.FALSE
+ try:
+ correctResults = main.TRUE
+ # Get onos counters results
+ onosCountersRaw = []
+ threads = []
+ for i in range( main.numCtrls ):
+ t = main.Thread( target=main.CLIs[i].counters,
+ name="counters-" + str( i ) )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ onosCountersRaw.append( t.result )
+ onosCounters = []
+ for i in range( main.numCtrls ):
+ try:
+ onosCounters.append( json.loads( onosCountersRaw[i] ) )
+ except ( ValueError, TypeError ):
+ main.log.error( "Could not parse counters response from ONOS" +
+ str( i + 1 ) )
+ main.log.warn( repr( onosCountersRaw[ i ] ) )
+ onosCounters.append( [] )
+ return main.FALSE
- testCounters = {}
- # make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
- # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
- # {"name":"TestON-Partitions","value":56} ]
- # }
- # NOTE: There is an assumtion that all nodes are active
- # based on the above for loops
- for controller in enumerate( onosCounters ):
- for dbType in controller[1]:
- for dbName, items in dbType.iteritems():
- for item in items:
- if 'TestON' in item['name']:
- node = 'ONOS' + str( controller[0] + 1 )
- try:
- testCounters[node].append( item )
- except KeyError:
- testCounters[node] = [ item ]
- # compare the counters on each node
- tmp = [ v == testCounters['ONOS1'] for k, v in testCounters.iteritems() ]
- if all( tmp ):
- consistent = main.TRUE
- else:
- consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters:\n" +
- testCounters )
- return ( onosCounters, consistent )
+ testCounters = {}
+ # make a list of all the "TestON-*" counters in ONOS
+ # lookes like a dict whose keys are the name of the ONOS node and values
+ # are a list of the counters. I.E.
+ # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
+ # {"name":"TestON-Partitions","value":56} ]
+ # }
+ # NOTE: There is an assumtion that all nodes are active
+ # based on the above for loops
+ for controller in enumerate( onosCounters ):
+ for dbType in controller[1]:
+ for dbName, items in dbType.iteritems():
+ for item in items:
+ if 'TestON' in item['name']:
+ node = 'ONOS' + str( controller[0] + 1 )
+ try:
+ testCounters[node].append( item )
+ except KeyError:
+ testCounters[node] = [ item ]
+ # compare the counters on each node
+ tmp = [ v == testCounters['ONOS1'] for k, v in testCounters.iteritems() ]
+ if all( tmp ):
+ consistent = main.TRUE
+ else:
+ consistent = main.FALSE
+ main.log.error( "ONOS nodes have different values for counters:\n" +
+ testCounters )
+ return ( onosCounters, consistent )
+ except Exception:
+ main.log.exception( "" )
+ main.cleanup()
+ main.exit()
def counterCheck( counterName, counterValue ):
"""