Harden HA tests against bad ONOS output
- also update gen partitions files
Change-Id: I9fbed42971650aa7feb050aafa66a37b1818804c
diff --git a/TestON/tests/HAkillNodes/HAkillNodes.py b/TestON/tests/HAkillNodes/HAkillNodes.py
index d26df2f..aa49941 100644
--- a/TestON/tests/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HAkillNodes/HAkillNodes.py
@@ -2146,15 +2146,13 @@
main.case( "Compare ONOS Topology view to Mininet topology" )
main.caseExplanation = "Compare topology objects between Mininet" +\
" and ONOS"
-
- main.step( "Comparing ONOS topology to MN" )
topoResult = main.FALSE
elapsed = 0
count = 0
- main.step( "Collecting topology information from ONOS" )
+ main.step( "Comparing ONOS topology to MN topology" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
devicesResults = main.TRUE
linksResults = main.TRUE
hostsResults = main.TRUE
@@ -2190,6 +2188,7 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing hosts results" )
main.log.error( repr( t.result ) )
+ hosts.append( [] )
for controller in range( 0, len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
for host in hosts[ controller ]:
@@ -2375,6 +2374,13 @@
hostsResults = hostsResults and currentHostsResult
hostAttachmentResults = hostAttachmentResults and\
hostAttachment
+ topoResult = devicesResults and linksResults and\
+ hostsResults and hostAttachmentResults
+ utilities.assert_equals( expect=True,
+ actual=topoResult,
+ onpass="ONOS topology matches Mininet",
+ onfail="ONOS topology don't match Mininet" )
+ # End of While loop to pull ONOS state
# Compare json objects for hosts and dataplane clusters
@@ -2507,26 +2513,23 @@
t.join()
nodesOutput.append( t.result )
ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
+ ips.sort()
for i in nodesOutput:
try:
current = json.loads( i )
+ activeIps = []
+ currentResult = main.FALSE
for node in current:
- currentResult = main.FALSE
- if node['ip'] in ips: # node in nodes() output is in cell
- if node['state'] == 'ACTIVE':
- currentResult = main.TRUE
- else:
- main.log.error( "Error in ONOS node availability" )
- main.log.error(
- json.dumps( current,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- break
- nodeResults = nodeResults and currentResult
+ if node['state'] == 'ACTIVE':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = main.TRUE
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
+ currentResult = main.FALSE
+ nodeResults = nodeResults and currentResult
utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
diff --git a/TestON/tests/HAkillNodes/dependencies/Counters.py b/TestON/tests/HAkillNodes/dependencies/Counters.py
index 455e3e7..f3833eb 100644
--- a/TestON/tests/HAkillNodes/dependencies/Counters.py
+++ b/TestON/tests/HAkillNodes/dependencies/Counters.py
@@ -32,6 +32,7 @@
main.log.error( "Could not parse counters response from ONOS" +
str( main.activeNodes[i] + 1 ) )
main.log.warn( repr( onosCountersRaw[ i ] ) )
+ onosCounters.append( [] )
return main.FALSE
testCounters = {}
diff --git a/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions b/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
index b73ed4a..cb0d826 100755
--- a/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
+++ b/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
@@ -1,16 +1,17 @@
#!/usr/bin/env python
-'''
+"""
Generate the partitions json file from the $OC* environment variables
Usage: onos-gen-partitions [output file]
If output file is not provided, the json is written to stdout.
-'''
+"""
from os import environ
from collections import deque, OrderedDict
import re
import json
import sys
+import hashlib
convert = lambda text: int(text) if text.isdigit() else text.lower()
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
@@ -54,8 +55,11 @@
vars = get_OC_vars()
nodes = get_nodes(vars)
partitions = generate_permutations2([v.get('id') for v in nodes], 3)
+ name = 0
+ for node in nodes:
+ name = name ^ hash(node['ip'])
data = {
- 'name': 'default',
+ 'name': name,
'nodes': nodes,
'partitions': partitions
}