Harden HA tests against bad ONOS output
- also update gen partitions files
Change-Id: I9fbed42971650aa7feb050aafa66a37b1818804c
diff --git a/TestON/tests/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HAclusterRestart/HAclusterRestart.py
index 79d575b..a745e06 100644
--- a/TestON/tests/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HAclusterRestart/HAclusterRestart.py
@@ -2149,15 +2149,13 @@
main.case( "Compare ONOS Topology view to Mininet topology" )
main.caseExplanation = "Compare topology objects between Mininet" +\
" and ONOS"
-
- main.step( "Comparing ONOS topology to MN" )
topoResult = main.FALSE
elapsed = 0
count = 0
- main.step( "Collecting topology information from ONOS" )
+ main.step( "Comparing ONOS topology to MN topology" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
devicesResults = main.TRUE
linksResults = main.TRUE
hostsResults = main.TRUE
@@ -2193,6 +2191,7 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing hosts results" )
main.log.error( repr( t.result ) )
+ hosts.append( [] )
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
@@ -2383,6 +2382,11 @@
topoResult = ( devicesResults and linksResults
and hostsResults and ipResult and
hostAttachmentResults )
+ utilities.assert_equals( expect=True,
+ actual=topoResult,
+ onpass="ONOS topology matches Mininet",
+ onfail="ONOS topology don't match Mininet" )
+ # End of While loop to pull ONOS state
# Compare json objects for hosts and dataplane clusters
@@ -2515,26 +2519,23 @@
t.join()
nodesOutput.append( t.result )
ips = [ node.ip_address for node in main.nodes ]
+ ips.sort()
for i in nodesOutput:
try:
current = json.loads( i )
+ activeIps = []
+ currentResult = main.FALSE
for node in current:
- currentResult = main.FALSE
- if node['ip'] in ips: # node in nodes() output is in cell
- if node['state'] == 'ACTIVE':
- currentResult = main.TRUE
- else:
- main.log.error( "Error in ONOS node availability" )
- main.log.error(
- json.dumps( current,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- break
- nodeResults = nodeResults and currentResult
+ if node['state'] == 'ACTIVE':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = main.TRUE
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
+ currentResult = main.FALSE
+ nodeResults = nodeResults and currentResult
utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
diff --git a/TestON/tests/HAclusterRestart/dependencies/Counters.py b/TestON/tests/HAclusterRestart/dependencies/Counters.py
index 6614887..2dc95e1 100644
--- a/TestON/tests/HAclusterRestart/dependencies/Counters.py
+++ b/TestON/tests/HAclusterRestart/dependencies/Counters.py
@@ -11,56 +11,62 @@
nodes or main.FALSE
"""
import json
- correctResults = main.TRUE
- # Get onos counters results
- onosCountersRaw = []
- threads = []
- for i in range( main.numCtrls ):
- t = main.Thread( target=main.CLIs[i].counters,
- name="counters-" + str( i ) )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCountersRaw.append( t.result )
- onosCounters = []
- for i in range( main.numCtrls ):
- try:
- onosCounters.append( json.loads( onosCountersRaw[i] ) )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( i + 1 ) )
- main.log.warn( repr( onosCountersRaw[ i ] ) )
- return main.FALSE
+ try:
+ correctResults = main.TRUE
+ # Get onos counters results
+ onosCountersRaw = []
+ threads = []
+ for i in range( main.numCtrls ):
+ t = main.Thread( target=main.CLIs[i].counters,
+ name="counters-" + str( i ) )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ onosCountersRaw.append( t.result )
+ onosCounters = []
+ for i in range( main.numCtrls ):
+ try:
+ onosCounters.append( json.loads( onosCountersRaw[i] ) )
+ except ( ValueError, TypeError ):
+ main.log.error( "Could not parse counters response from ONOS" +
+ str( i + 1 ) )
+ main.log.warn( repr( onosCountersRaw[ i ] ) )
+ onosCounters.append( [] )
+ return main.FALSE
- testCounters = {}
- # make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
- # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
- # {"name":"TestON-Partitions","value":56} ]
- # }
- # NOTE: There is an assumtion that all nodes are active
- # based on the above for loops
- for controller in enumerate( onosCounters ):
- for dbType in controller[1]:
- for dbName, items in dbType.iteritems():
- for item in items:
- if 'TestON' in item['name']:
- node = 'ONOS' + str( controller[0] + 1 )
- try:
- testCounters[node].append( item )
- except KeyError:
- testCounters[node] = [ item ]
- # compare the counters on each node
- tmp = [ v == testCounters['ONOS1'] for k, v in testCounters.iteritems() ]
- if all( tmp ):
- consistent = main.TRUE
- else:
- consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters:\n" +
- testCounters )
- return ( onosCounters, consistent )
+ testCounters = {}
+ # make a list of all the "TestON-*" counters in ONOS
+ # lookes like a dict whose keys are the name of the ONOS node and values
+ # are a list of the counters. I.E.
+ # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
+ # {"name":"TestON-Partitions","value":56} ]
+ # }
+ # NOTE: There is an assumtion that all nodes are active
+ # based on the above for loops
+ for controller in enumerate( onosCounters ):
+ for dbType in controller[1]:
+ for dbName, items in dbType.iteritems():
+ for item in items:
+ if 'TestON' in item['name']:
+ node = 'ONOS' + str( controller[0] + 1 )
+ try:
+ testCounters[node].append( item )
+ except KeyError:
+ testCounters[node] = [ item ]
+ # compare the counters on each node
+ tmp = [ v == testCounters['ONOS1'] for k, v in testCounters.iteritems() ]
+ if all( tmp ):
+ consistent = main.TRUE
+ else:
+ consistent = main.FALSE
+ main.log.error( "ONOS nodes have different values for counters:\n" +
+ testCounters )
+ return ( onosCounters, consistent )
+ except Exception:
+ main.log.exception( "" )
+ main.cleanup()
+ main.exit()
def counterCheck( counterName, counterValue ):
"""
diff --git a/TestON/tests/HAkillNodes/HAkillNodes.py b/TestON/tests/HAkillNodes/HAkillNodes.py
index d26df2f..aa49941 100644
--- a/TestON/tests/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HAkillNodes/HAkillNodes.py
@@ -2146,15 +2146,13 @@
main.case( "Compare ONOS Topology view to Mininet topology" )
main.caseExplanation = "Compare topology objects between Mininet" +\
" and ONOS"
-
- main.step( "Comparing ONOS topology to MN" )
topoResult = main.FALSE
elapsed = 0
count = 0
- main.step( "Collecting topology information from ONOS" )
+ main.step( "Comparing ONOS topology to MN topology" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
devicesResults = main.TRUE
linksResults = main.TRUE
hostsResults = main.TRUE
@@ -2190,6 +2188,7 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing hosts results" )
main.log.error( repr( t.result ) )
+ hosts.append( [] )
for controller in range( 0, len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
for host in hosts[ controller ]:
@@ -2375,6 +2374,13 @@
hostsResults = hostsResults and currentHostsResult
hostAttachmentResults = hostAttachmentResults and\
hostAttachment
+ topoResult = devicesResults and linksResults and\
+ hostsResults and hostAttachmentResults
+ utilities.assert_equals( expect=True,
+ actual=topoResult,
+ onpass="ONOS topology matches Mininet",
+ onfail="ONOS topology don't match Mininet" )
+ # End of While loop to pull ONOS state
# Compare json objects for hosts and dataplane clusters
@@ -2507,26 +2513,23 @@
t.join()
nodesOutput.append( t.result )
ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
+ ips.sort()
for i in nodesOutput:
try:
current = json.loads( i )
+ activeIps = []
+ currentResult = main.FALSE
for node in current:
- currentResult = main.FALSE
- if node['ip'] in ips: # node in nodes() output is in cell
- if node['state'] == 'ACTIVE':
- currentResult = main.TRUE
- else:
- main.log.error( "Error in ONOS node availability" )
- main.log.error(
- json.dumps( current,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- break
- nodeResults = nodeResults and currentResult
+ if node['state'] == 'ACTIVE':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = main.TRUE
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
+ currentResult = main.FALSE
+ nodeResults = nodeResults and currentResult
utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
diff --git a/TestON/tests/HAkillNodes/dependencies/Counters.py b/TestON/tests/HAkillNodes/dependencies/Counters.py
index 455e3e7..f3833eb 100644
--- a/TestON/tests/HAkillNodes/dependencies/Counters.py
+++ b/TestON/tests/HAkillNodes/dependencies/Counters.py
@@ -32,6 +32,7 @@
main.log.error( "Could not parse counters response from ONOS" +
str( main.activeNodes[i] + 1 ) )
main.log.warn( repr( onosCountersRaw[ i ] ) )
+ onosCounters.append( [] )
return main.FALSE
testCounters = {}
diff --git a/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions b/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
index b73ed4a..cb0d826 100755
--- a/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
+++ b/TestON/tests/HAkillNodes/dependencies/onos-gen-partitions
@@ -1,16 +1,17 @@
#!/usr/bin/env python
-'''
+"""
Generate the partitions json file from the $OC* environment variables
Usage: onos-gen-partitions [output file]
If output file is not provided, the json is written to stdout.
-'''
+"""
from os import environ
from collections import deque, OrderedDict
import re
import json
import sys
+import hashlib
convert = lambda text: int(text) if text.isdigit() else text.lower()
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
@@ -54,8 +55,11 @@
vars = get_OC_vars()
nodes = get_nodes(vars)
partitions = generate_permutations2([v.get('id') for v in nodes], 3)
+ name = 0
+ for node in nodes:
+ name = name ^ hash(node['ip'])
data = {
- 'name': 'default',
+ 'name': name,
'nodes': nodes,
'partitions': partitions
}
diff --git a/TestON/tests/HAsanity/HAsanity.py b/TestON/tests/HAsanity/HAsanity.py
index 7de57b5..859d582 100644
--- a/TestON/tests/HAsanity/HAsanity.py
+++ b/TestON/tests/HAsanity/HAsanity.py
@@ -2042,15 +2042,13 @@
main.case( "Compare ONOS Topology view to Mininet topology" )
main.caseExplanation = "Compare topology objects between Mininet" +\
" and ONOS"
-
- main.step( "Comparing ONOS topology to MN" )
topoResult = main.FALSE
elapsed = 0
count = 0
- main.step( "Collecting topology information from ONOS" )
+ main.step( "Comparing ONOS topology to MN topology" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
devicesResults = main.TRUE
linksResults = main.TRUE
hostsResults = main.TRUE
@@ -2086,6 +2084,7 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing hosts results" )
main.log.error( repr( t.result ) )
+ hosts.append( [] )
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
@@ -2274,6 +2273,11 @@
topoResult = ( devicesResults and linksResults
and hostsResults and ipResult and
hostAttachmentResults )
+ utilities.assert_equals( expect=True,
+ actual=topoResult,
+ onpass="ONOS topology matches Mininet",
+ onfail="ONOS topology don't match Mininet" )
+ # End of While loop to pull ONOS state
# Compare json objects for hosts and dataplane clusters
@@ -2413,26 +2417,23 @@
t.join()
nodesOutput.append( t.result )
ips = [ node.ip_address for node in main.nodes ]
+ ips.sort()
for i in nodesOutput:
try:
current = json.loads( i )
+ activeIps = []
+ currentResult = main.FALSE
for node in current:
- currentResult = main.FALSE
- if node['ip'] in ips: # node in nodes() output is in cell
- if node['state'] == 'ACTIVE':
- currentResult = main.TRUE
- else:
- main.log.error( "Error in ONOS node availability" )
- main.log.error(
- json.dumps( current,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- break
- nodeResults = nodeResults and currentResult
+ if node['state'] == 'ACTIVE':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = main.TRUE
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
+ currentResult = main.FALSE
+ nodeResults = nodeResults and currentResult
utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
diff --git a/TestON/tests/HAsanity/dependencies/Counters.py b/TestON/tests/HAsanity/dependencies/Counters.py
index 6614887..2dc95e1 100644
--- a/TestON/tests/HAsanity/dependencies/Counters.py
+++ b/TestON/tests/HAsanity/dependencies/Counters.py
@@ -11,56 +11,62 @@
nodes or main.FALSE
"""
import json
- correctResults = main.TRUE
- # Get onos counters results
- onosCountersRaw = []
- threads = []
- for i in range( main.numCtrls ):
- t = main.Thread( target=main.CLIs[i].counters,
- name="counters-" + str( i ) )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCountersRaw.append( t.result )
- onosCounters = []
- for i in range( main.numCtrls ):
- try:
- onosCounters.append( json.loads( onosCountersRaw[i] ) )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( i + 1 ) )
- main.log.warn( repr( onosCountersRaw[ i ] ) )
- return main.FALSE
+ try:
+ correctResults = main.TRUE
+ # Get onos counters results
+ onosCountersRaw = []
+ threads = []
+ for i in range( main.numCtrls ):
+ t = main.Thread( target=main.CLIs[i].counters,
+ name="counters-" + str( i ) )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ onosCountersRaw.append( t.result )
+ onosCounters = []
+ for i in range( main.numCtrls ):
+ try:
+ onosCounters.append( json.loads( onosCountersRaw[i] ) )
+ except ( ValueError, TypeError ):
+ main.log.error( "Could not parse counters response from ONOS" +
+ str( i + 1 ) )
+ main.log.warn( repr( onosCountersRaw[ i ] ) )
+ onosCounters.append( [] )
+ return main.FALSE
- testCounters = {}
- # make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
- # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
- # {"name":"TestON-Partitions","value":56} ]
- # }
- # NOTE: There is an assumtion that all nodes are active
- # based on the above for loops
- for controller in enumerate( onosCounters ):
- for dbType in controller[1]:
- for dbName, items in dbType.iteritems():
- for item in items:
- if 'TestON' in item['name']:
- node = 'ONOS' + str( controller[0] + 1 )
- try:
- testCounters[node].append( item )
- except KeyError:
- testCounters[node] = [ item ]
- # compare the counters on each node
- tmp = [ v == testCounters['ONOS1'] for k, v in testCounters.iteritems() ]
- if all( tmp ):
- consistent = main.TRUE
- else:
- consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters:\n" +
- testCounters )
- return ( onosCounters, consistent )
+ testCounters = {}
+ # make a list of all the "TestON-*" counters in ONOS
+ # lookes like a dict whose keys are the name of the ONOS node and values
+ # are a list of the counters. I.E.
+ # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
+ # {"name":"TestON-Partitions","value":56} ]
+ # }
+ # NOTE: There is an assumtion that all nodes are active
+ # based on the above for loops
+ for controller in enumerate( onosCounters ):
+ for dbType in controller[1]:
+ for dbName, items in dbType.iteritems():
+ for item in items:
+ if 'TestON' in item['name']:
+ node = 'ONOS' + str( controller[0] + 1 )
+ try:
+ testCounters[node].append( item )
+ except KeyError:
+ testCounters[node] = [ item ]
+ # compare the counters on each node
+ tmp = [ v == testCounters['ONOS1'] for k, v in testCounters.iteritems() ]
+ if all( tmp ):
+ consistent = main.TRUE
+ else:
+ consistent = main.FALSE
+ main.log.error( "ONOS nodes have different values for counters:\n" +
+ testCounters )
+ return ( onosCounters, consistent )
+ except Exception:
+ main.log.exception( "" )
+ main.cleanup()
+ main.exit()
def counterCheck( counterName, counterValue ):
"""
diff --git a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index 52bf2f9..728b9ab 100644
--- a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -1427,15 +1427,13 @@
main.case( "Compare ONOS Topology view to Mininet topology" )
main.caseExplanation = "Compare topology objects between Mininet" +\
" and ONOS"
-
- main.step( "Comparing ONOS topology to MN" )
topoResult = main.FALSE
elapsed = 0
count = 0
- main.step( "Collecting topology information from ONOS" )
+ main.step( "Comparing ONOS topology to MN topology" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
devicesResults = main.TRUE
linksResults = main.TRUE
hostsResults = main.TRUE
diff --git a/TestON/tests/HAsingleInstanceRestart/dependencies/Counters.py b/TestON/tests/HAsingleInstanceRestart/dependencies/Counters.py
index 6614887..2dc95e1 100644
--- a/TestON/tests/HAsingleInstanceRestart/dependencies/Counters.py
+++ b/TestON/tests/HAsingleInstanceRestart/dependencies/Counters.py
@@ -11,56 +11,62 @@
nodes or main.FALSE
"""
import json
- correctResults = main.TRUE
- # Get onos counters results
- onosCountersRaw = []
- threads = []
- for i in range( main.numCtrls ):
- t = main.Thread( target=main.CLIs[i].counters,
- name="counters-" + str( i ) )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCountersRaw.append( t.result )
- onosCounters = []
- for i in range( main.numCtrls ):
- try:
- onosCounters.append( json.loads( onosCountersRaw[i] ) )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( i + 1 ) )
- main.log.warn( repr( onosCountersRaw[ i ] ) )
- return main.FALSE
+ try:
+ correctResults = main.TRUE
+ # Get onos counters results
+ onosCountersRaw = []
+ threads = []
+ for i in range( main.numCtrls ):
+ t = main.Thread( target=main.CLIs[i].counters,
+ name="counters-" + str( i ) )
+ threads.append( t )
+ t.start()
+ for t in threads:
+ t.join()
+ onosCountersRaw.append( t.result )
+ onosCounters = []
+ for i in range( main.numCtrls ):
+ try:
+ onosCounters.append( json.loads( onosCountersRaw[i] ) )
+ except ( ValueError, TypeError ):
+ main.log.error( "Could not parse counters response from ONOS" +
+ str( i + 1 ) )
+ main.log.warn( repr( onosCountersRaw[ i ] ) )
+ onosCounters.append( [] )
+ return main.FALSE
- testCounters = {}
- # make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
- # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
- # {"name":"TestON-Partitions","value":56} ]
- # }
- # NOTE: There is an assumtion that all nodes are active
- # based on the above for loops
- for controller in enumerate( onosCounters ):
- for dbType in controller[1]:
- for dbName, items in dbType.iteritems():
- for item in items:
- if 'TestON' in item['name']:
- node = 'ONOS' + str( controller[0] + 1 )
- try:
- testCounters[node].append( item )
- except KeyError:
- testCounters[node] = [ item ]
- # compare the counters on each node
- tmp = [ v == testCounters['ONOS1'] for k, v in testCounters.iteritems() ]
- if all( tmp ):
- consistent = main.TRUE
- else:
- consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters:\n" +
- testCounters )
- return ( onosCounters, consistent )
+ testCounters = {}
+ # make a list of all the "TestON-*" counters in ONOS
+ # lookes like a dict whose keys are the name of the ONOS node and values
+ # are a list of the counters. I.E.
+ # { "ONOS1": [ {"name":"TestON-inMemory","value":56},
+ # {"name":"TestON-Partitions","value":56} ]
+ # }
+ # NOTE: There is an assumtion that all nodes are active
+ # based on the above for loops
+ for controller in enumerate( onosCounters ):
+ for dbType in controller[1]:
+ for dbName, items in dbType.iteritems():
+ for item in items:
+ if 'TestON' in item['name']:
+ node = 'ONOS' + str( controller[0] + 1 )
+ try:
+ testCounters[node].append( item )
+ except KeyError:
+ testCounters[node] = [ item ]
+ # compare the counters on each node
+ tmp = [ v == testCounters['ONOS1'] for k, v in testCounters.iteritems() ]
+ if all( tmp ):
+ consistent = main.TRUE
+ else:
+ consistent = main.FALSE
+ main.log.error( "ONOS nodes have different values for counters:\n" +
+ testCounters )
+ return ( onosCounters, consistent )
+ except Exception:
+ main.log.exception( "" )
+ main.cleanup()
+ main.exit()
def counterCheck( counterName, counterValue ):
"""
diff --git a/TestON/tests/HAstopNodes/HAstopNodes.py b/TestON/tests/HAstopNodes/HAstopNodes.py
index ce3bb6c..47689af 100644
--- a/TestON/tests/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HAstopNodes/HAstopNodes.py
@@ -2135,15 +2135,13 @@
main.case( "Compare ONOS Topology view to Mininet topology" )
main.caseExplanation = "Compare topology objects between Mininet" +\
" and ONOS"
-
- main.step( "Comparing ONOS topology to MN" )
topoResult = main.FALSE
elapsed = 0
count = 0
- main.step( "Collecting topology information from ONOS" )
+ main.step( "Comparing ONOS topology to MN topology" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
devicesResults = main.TRUE
linksResults = main.TRUE
hostsResults = main.TRUE
@@ -2179,6 +2177,7 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing hosts results" )
main.log.error( repr( t.result ) )
+ hosts.append( [] )
for controller in range( 0, len( hosts ) ):
controllerStr = str( main.activeNodes[controller] + 1 )
for host in hosts[ controller ]:
@@ -2364,6 +2363,13 @@
hostsResults = hostsResults and currentHostsResult
hostAttachmentResults = hostAttachmentResults and\
hostAttachment
+ topoResult = devicesResults and linksResults and\
+ hostsResults and hostAttachmentResults
+ utilities.assert_equals( expect=True,
+ actual=topoResult,
+ onpass="ONOS topology matches Mininet",
+ onfail="ONOS topology don't match Mininet" )
+ # End of While loop to pull ONOS state
# Compare json objects for hosts and dataplane clusters
@@ -2496,26 +2502,23 @@
t.join()
nodesOutput.append( t.result )
ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
+ ips.sort()
for i in nodesOutput:
try:
current = json.loads( i )
+ activeIps = []
+ currentResult = main.FALSE
for node in current:
- currentResult = main.FALSE
- if node['ip'] in ips: # node in nodes() output is in cell
- if node['state'] == 'ACTIVE':
- currentResult = main.TRUE
- else:
- main.log.error( "Error in ONOS node availability" )
- main.log.error(
- json.dumps( current,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- break
- nodeResults = nodeResults and currentResult
+ if node['state'] == 'ACTIVE':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = main.TRUE
except ( ValueError, TypeError ):
main.log.error( "Error parsing nodes output" )
main.log.warn( repr( i ) )
+ currentResult = main.FALSE
+ nodeResults = nodeResults and currentResult
utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
diff --git a/TestON/tests/HAstopNodes/dependencies/Counters.py b/TestON/tests/HAstopNodes/dependencies/Counters.py
index 455e3e7..f3833eb 100644
--- a/TestON/tests/HAstopNodes/dependencies/Counters.py
+++ b/TestON/tests/HAstopNodes/dependencies/Counters.py
@@ -32,6 +32,7 @@
main.log.error( "Could not parse counters response from ONOS" +
str( main.activeNodes[i] + 1 ) )
main.log.warn( repr( onosCountersRaw[ i ] ) )
+ onosCounters.append( [] )
return main.FALSE
testCounters = {}
diff --git a/TestON/tests/HAstopNodes/dependencies/onos-gen-partitions b/TestON/tests/HAstopNodes/dependencies/onos-gen-partitions
index b73ed4a..cb0d826 100755
--- a/TestON/tests/HAstopNodes/dependencies/onos-gen-partitions
+++ b/TestON/tests/HAstopNodes/dependencies/onos-gen-partitions
@@ -1,16 +1,17 @@
#!/usr/bin/env python
-'''
+"""
Generate the partitions json file from the $OC* environment variables
Usage: onos-gen-partitions [output file]
If output file is not provided, the json is written to stdout.
-'''
+"""
from os import environ
from collections import deque, OrderedDict
import re
import json
import sys
+import hashlib
convert = lambda text: int(text) if text.isdigit() else text.lower()
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
@@ -54,8 +55,11 @@
vars = get_OC_vars()
nodes = get_nodes(vars)
partitions = generate_permutations2([v.get('id') for v in nodes], 3)
+ name = 0
+ for node in nodes:
+ name = name ^ hash(node['ip'])
data = {
- 'name': 'default',
+ 'name': name,
'nodes': nodes,
'partitions': partitions
}