Refactor HA tests
Move more code into functions
Retry some functions
Rename class file
Enable FlowObjective intent compiler
Cleanup flowTableComp and doen't short circuit in Mininet driver
Change-Id: I2a404b32469d137370e512a1c44f06dd8e9df23b
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index a34741a..0cfd804 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -1927,28 +1927,28 @@
def flowTableComp( self, flowTable1, flowTable2 ):
# This function compares the selctors and treatments of each flow
try:
+ returnValue = main.TRUE
if len(flowTable1) != len(flowTable2):
main.log.warn( "Flow table lengths do not match" )
- return main.FALSE
+ returnValue = main.FALSE
dFields = ["n_bytes", "cookie", "n_packets", "duration"]
for flow1, flow2 in zip(flowTable1, flowTable2):
for field in dFields:
try:
flow1.pop( field )
- except KeyError as e:
- main.log.warn( e )
- main.log.debug( flow1 )
+ except KeyError:
+ pass
try:
flow2.pop( field )
- except KeyError as e:
- main.log.warn( e )
- main.log.debug( flow2 )
+ except KeyError:
+ pass
for i in range( len(flowTable1) ):
if flowTable1[i] not in flowTable2:
main.log.warn( "Flow tables do not match:" )
main.log.warn( "Old flow:\n{}\n not in new flow table".format( flowTable1[i] ) )
- return main.FALSE
- return main.TRUE
+ returnValue = main.FALSE
+ break
+ return returnValue
except Exception:
main.log.exception( "Uncaught exception!" )
main.cleanup()
diff --git a/TestON/tests/HAclusterRestart/HAclusterRestart.params b/TestON/tests/HAclusterRestart/HAclusterRestart.params
index d67878a..95bbd87 100644
--- a/TestON/tests/HAclusterRestart/HAclusterRestart.params
+++ b/TestON/tests/HAclusterRestart/HAclusterRestart.params
@@ -22,7 +22,7 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>false</useFlowObjectives>
+ <useFlowObjectives>true</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
</ONOS_Configuration>
<ENV>
diff --git a/TestON/tests/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HAclusterRestart/HAclusterRestart.py
index cb40b4f..2e840a1 100644
--- a/TestON/tests/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HAclusterRestart/HAclusterRestart.py
@@ -93,8 +93,8 @@
ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
try:
- from tests.HAsanity.dependencies.Counters import Counters
- main.Counters = Counters()
+ from tests.HAsanity.dependencies.HA import HA
+ main.HA = HA()
except Exception as e:
main.log.exception( e )
main.cleanup()
@@ -251,38 +251,12 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
@@ -971,6 +945,7 @@
"functionality and check the state of " +\
"the intent"
+ onosCli = main.CLIs[ main.activeNodes[0] ]
main.step( "Check Intent state" )
installedCheck = False
loopCount = 0
@@ -1006,7 +981,6 @@
"INSTALLED state" )
main.step( "Ping across added host intents" )
- onosCli = main.CLIs[ main.activeNodes[0] ]
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -2158,8 +2132,9 @@
for i in range( 28 ):
main.log.info( "Checking flow table on s" + str( i + 1 ) )
tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
- FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
- if FlowTables == main.FALSE:
+ curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
+ FlowTables = FlowTables and curSwitch
+ if curSwitch == main.FALSE:
main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
utilities.assert_equals(
expect=main.TRUE,
@@ -2634,45 +2609,19 @@
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
if not nodeResults:
- for cli in main.CLIs:
+ for i in main.activeNodes:
main.log.debug( "{} components not ACTIVE: \n{}".format(
- cli.name,
- cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.CLIs[i].name,
+ main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
def CASE9( self, main ):
"""
@@ -2990,26 +2939,8 @@
main.step( "Check that each node shows the same leader and candidates" )
failMessage = "Nodes have different leaderboards"
- def consistentLeaderboards( nodes ):
- TOPIC = 'org.onosproject.election'
- # FIXME: use threads
- #FIXME: should we retry outside the function?
- for n in range( 5 ): # Retry in case election is still happening
- leaderList = []
- # Get all leaderboards
- for cli in nodes:
- leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
- # Compare leaderboards
- result = all( i == leaderList[0] for i in leaderList ) and\
- leaderList is not None
- main.log.debug( leaderList )
- main.log.warn( result )
- if result:
- return ( result, leaderList )
- time.sleep(5) #TODO: paramerterize
- main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
- sameResult, oldLeaders = consistentLeaderboards( activeCLIs )
+ sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
if sameResult:
oldLeader = oldLeaders[ 0 ][ 0 ]
main.log.warn( oldLeader )
@@ -3045,7 +2976,7 @@
main.step( "Check that a new node was elected leader" )
failMessage = "Nodes have different leaders"
# Get new leaders and candidates
- newLeaderResult, newLeaders = consistentLeaderboards( activeCLIs )
+ newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
if newLeaders[ 0 ][ 0 ] == 'none':
main.log.error( "No leader was elected on at least 1 node" )
if not expectNoLeader:
@@ -3113,7 +3044,7 @@
# Get new leaders and candidates
reRunLeaders = []
time.sleep( 5 ) # Paremterize
- positionResult, reRunLeaders = consistentLeaderboards( activeCLIs )
+ positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
# Check that the re-elected node is last on the candidate List
if oldLeader != reRunLeaders[ 0 ][ -1 ]:
@@ -3257,7 +3188,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
@@ -3357,7 +3288,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
diff --git a/TestON/tests/HAclusterRestart/dependencies/Counters.py b/TestON/tests/HAclusterRestart/dependencies/HA.py
similarity index 60%
rename from TestON/tests/HAclusterRestart/dependencies/Counters.py
rename to TestON/tests/HAclusterRestart/dependencies/HA.py
index 192b919..e00d290 100644
--- a/TestON/tests/HAclusterRestart/dependencies/Counters.py
+++ b/TestON/tests/HAclusterRestart/dependencies/HA.py
@@ -1,6 +1,7 @@
import json
+import time
-class Counters():
+class HA():
def __init__( self ):
self.default = ''
@@ -10,12 +11,12 @@
Checks that TestON counters are consistent across all nodes.
Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
+ - onosCounters is the parsed json output of the counters command on
+ all nodes
+ - consistent is main.TRUE if all "TestON" counters are consitent across
+ all nodes or main.FALSE
"""
try:
- correctResults = main.TRUE
# Get onos counters results
onosCountersRaw = []
threads = []
@@ -42,8 +43,8 @@
testCounters = {}
# make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
+ # lookes like a dict whose keys are the name of the ONOS node and
+ # values are a list of the counters. I.E.
# { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
# }
# NOTE: There is an assumtion that all nodes are active
@@ -86,20 +87,75 @@
onosValue = None
try:
onosValue = current.get( counterName )
- except AttributeError, e:
+ except AttributeError:
node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
+ main.log.exception( "ONOS" + node + " counters result " +
+ "is not as expected" )
correctResults = main.FALSE
if onosValue == counterValue:
main.log.info( counterName + " counter value is correct" )
else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
+ main.log.error( counterName +
+ " counter value is incorrect," +
+ " expected value: " + str( counterValue ) +
+ " current value: " + str( onosValue ) )
correctResults = main.FALSE
return consistent and correctResults
except Exception:
main.log.exception( "" )
main.cleanup()
main.exit()
+
+ def consistentLeaderboards( self, nodes ):
+ TOPIC = 'org.onosproject.election'
+ # FIXME: use threads
+ #FIXME: should we retry outside the function?
+ for n in range( 5 ): # Retry in case election is still happening
+ leaderList = []
+ # Get all leaderboards
+ for cli in nodes:
+ leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
+ # Compare leaderboards
+ result = all( i == leaderList[0] for i in leaderList ) and\
+ leaderList is not None
+ main.log.debug( leaderList )
+ main.log.warn( result )
+ if result:
+ return ( result, leaderList )
+ time.sleep(5) # TODO: paramerterize
+ main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
+ return ( result, leaderList )
+
+ def nodesCheck( self, nodes ):
+ nodesOutput = []
+ results = True
+ threads = []
+ for i in nodes:
+ t = main.Thread( target=main.CLIs[i].nodes,
+ name="nodes-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ nodesOutput.append( t.result )
+ ips = [ main.nodes[node].ip_address for node in nodes ]
+ ips.sort()
+ for i in nodesOutput:
+ try:
+ current = json.loads( i )
+ activeIps = []
+ currentResult = False
+ for node in current:
+ if node['state'] == 'READY':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = True
+ except ( ValueError, TypeError ):
+ main.log.error( "Error parsing nodes output" )
+ main.log.warn( repr( i ) )
+ currentResult = False
+ results = results and currentResult
+ return results
diff --git a/TestON/tests/HAfullNetPartition/HAfullNetPartition.params b/TestON/tests/HAfullNetPartition/HAfullNetPartition.params
index 4dc5a5e..5461243 100644
--- a/TestON/tests/HAfullNetPartition/HAfullNetPartition.params
+++ b/TestON/tests/HAfullNetPartition/HAfullNetPartition.params
@@ -24,7 +24,7 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>false</useFlowObjectives>
+ <useFlowObjectives>true</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
</ONOS_Configuration>
<ENV>
diff --git a/TestON/tests/HAfullNetPartition/HAfullNetPartition.py b/TestON/tests/HAfullNetPartition/HAfullNetPartition.py
index 12d8bbb..6ae0c7b 100644
--- a/TestON/tests/HAfullNetPartition/HAfullNetPartition.py
+++ b/TestON/tests/HAfullNetPartition/HAfullNetPartition.py
@@ -95,8 +95,8 @@
ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
try:
- from tests.HAsanity.dependencies.Counters import Counters
- main.Counters = Counters()
+ from tests.HAsanity.dependencies.HA import HA
+ main.HA = HA()
except Exception as e:
main.log.exception( e )
main.cleanup()
@@ -276,38 +276,12 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
@@ -973,6 +947,7 @@
"functionality and check the state of " +\
"the intent"
+ onosCli = main.CLIs[ main.activeNodes[0] ]
main.step( "Check Intent state" )
installedCheck = False
loopCount = 0
@@ -1008,7 +983,6 @@
"INSTALLED state" )
main.step( "Ping across added host intents" )
- onosCli = main.CLIs[ main.activeNodes[0] ]
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -2140,8 +2114,9 @@
for i in range( 28 ):
main.log.info( "Checking flow table on s" + str( i + 1 ) )
tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
- FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
- if FlowTables == main.FALSE:
+ curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
+ FlowTables = FlowTables and curSwitch
+ if curSwitch == main.FALSE:
main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
utilities.assert_equals(
expect=main.TRUE,
@@ -2620,45 +2595,19 @@
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
if not nodeResults:
- for cli in main.CLIs:
+ for i in main.activeNodes:
main.log.debug( "{} components not ACTIVE: \n{}".format(
- cli.name,
- cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.CLIs[i].name,
+ main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
def CASE9( self, main ):
"""
@@ -2978,26 +2927,8 @@
main.step( "Check that each node shows the same leader and candidates" )
failMessage = "Nodes have different leaderboards"
- def consistentLeaderboards( nodes ):
- TOPIC = 'org.onosproject.election'
- # FIXME: use threads
- #FIXME: should we retry outside the function?
- for n in range( 5 ): # Retry in case election is still happening
- leaderList = []
- # Get all leaderboards
- for cli in nodes:
- leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
- # Compare leaderboards
- result = all( i == leaderList[0] for i in leaderList ) and\
- leaderList is not None
- main.log.debug( leaderList )
- main.log.warn( result )
- if result:
- return ( result, leaderList )
- time.sleep(5) #TODO: paramerterize
- main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
- sameResult, oldLeaders = consistentLeaderboards( activeCLIs )
+ sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
if sameResult:
oldLeader = oldLeaders[ 0 ][ 0 ]
main.log.warn( oldLeader )
@@ -3033,7 +2964,7 @@
main.step( "Check that a new node was elected leader" )
failMessage = "Nodes have different leaders"
# Get new leaders and candidates
- newLeaderResult, newLeaders = consistentLeaderboards( activeCLIs )
+ newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
if newLeaders[ 0 ][ 0 ] == 'none':
main.log.error( "No leader was elected on at least 1 node" )
if not expectNoLeader:
@@ -3101,7 +3032,7 @@
# Get new leaders and candidates
reRunLeaders = []
time.sleep( 5 ) # Paremterize
- positionResult, reRunLeaders = consistentLeaderboards( activeCLIs )
+ positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
# Check that the re-elected node is last on the candidate List
if oldLeader != reRunLeaders[ 0 ][ -1 ]:
@@ -3245,7 +3176,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
@@ -3345,7 +3276,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
diff --git a/TestON/tests/HAfullNetPartition/dependencies/Counters.py b/TestON/tests/HAfullNetPartition/dependencies/Counters.py
deleted file mode 100644
index 192b919..0000000
--- a/TestON/tests/HAfullNetPartition/dependencies/Counters.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import json
-
-class Counters():
-
- def __init__( self ):
- self.default = ''
-
- def consistentCheck( self ):
- """
- Checks that TestON counters are consistent across all nodes.
-
- Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results
- onosCountersRaw = []
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=utilities.retry,
- name="counters-" + str( i ),
- args=[ main.CLIs[i].counters, [ None ] ],
- kwargs= { 'sleep': 5, 'attempts': 5,
- 'randomTime': True } )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCountersRaw.append( t.result )
- onosCounters = []
- for i in range( len( main.activeNodes ) ):
- try:
- onosCounters.append( json.loads( onosCountersRaw[i] ) )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( main.activeNodes[i] + 1 ) )
- main.log.warn( repr( onosCountersRaw[ i ] ) )
- onosCounters.append( [] )
-
- testCounters = {}
- # make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
- # { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
- # }
- # NOTE: There is an assumtion that all nodes are active
- # based on the above for loops
- for controller in enumerate( onosCounters ):
- for key, value in controller[1].iteritems():
- if 'TestON' in key:
- node = 'ONOS' + str( controller[0] + 1 )
- try:
- testCounters[node].append( { key: value } )
- except KeyError:
- testCounters[node] = [ { key: value } ]
- # compare the counters on each node
- firstV = testCounters.values()[0]
- tmp = [ v == firstV for k, v in testCounters.iteritems() ]
- if all( tmp ):
- consistent = main.TRUE
- else:
- consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters:\n" +
- testCounters )
- return ( onosCounters, consistent )
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
-
- def counterCheck( self, counterName, counterValue ):
- """
- Checks that TestON counters are consistent across all nodes and that
- specified counter is in ONOS with the given value
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results and consistentCheck
- onosCounters, consistent = self.consistentCheck()
- # Check for correct values
- for i in range( len( main.activeNodes ) ):
- current = onosCounters[i]
- onosValue = None
- try:
- onosValue = current.get( counterName )
- except AttributeError, e:
- node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
- correctResults = main.FALSE
- if onosValue == counterValue:
- main.log.info( counterName + " counter value is correct" )
- else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
- correctResults = main.FALSE
- return consistent and correctResults
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
diff --git a/TestON/tests/HAclusterRestart/dependencies/Counters.py b/TestON/tests/HAfullNetPartition/dependencies/HA.py
similarity index 60%
copy from TestON/tests/HAclusterRestart/dependencies/Counters.py
copy to TestON/tests/HAfullNetPartition/dependencies/HA.py
index 192b919..e00d290 100644
--- a/TestON/tests/HAclusterRestart/dependencies/Counters.py
+++ b/TestON/tests/HAfullNetPartition/dependencies/HA.py
@@ -1,6 +1,7 @@
import json
+import time
-class Counters():
+class HA():
def __init__( self ):
self.default = ''
@@ -10,12 +11,12 @@
Checks that TestON counters are consistent across all nodes.
Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
+ - onosCounters is the parsed json output of the counters command on
+ all nodes
+ - consistent is main.TRUE if all "TestON" counters are consitent across
+ all nodes or main.FALSE
"""
try:
- correctResults = main.TRUE
# Get onos counters results
onosCountersRaw = []
threads = []
@@ -42,8 +43,8 @@
testCounters = {}
# make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
+ # lookes like a dict whose keys are the name of the ONOS node and
+ # values are a list of the counters. I.E.
# { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
# }
# NOTE: There is an assumtion that all nodes are active
@@ -86,20 +87,75 @@
onosValue = None
try:
onosValue = current.get( counterName )
- except AttributeError, e:
+ except AttributeError:
node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
+ main.log.exception( "ONOS" + node + " counters result " +
+ "is not as expected" )
correctResults = main.FALSE
if onosValue == counterValue:
main.log.info( counterName + " counter value is correct" )
else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
+ main.log.error( counterName +
+ " counter value is incorrect," +
+ " expected value: " + str( counterValue ) +
+ " current value: " + str( onosValue ) )
correctResults = main.FALSE
return consistent and correctResults
except Exception:
main.log.exception( "" )
main.cleanup()
main.exit()
+
+ def consistentLeaderboards( self, nodes ):
+ TOPIC = 'org.onosproject.election'
+ # FIXME: use threads
+ #FIXME: should we retry outside the function?
+ for n in range( 5 ): # Retry in case election is still happening
+ leaderList = []
+ # Get all leaderboards
+ for cli in nodes:
+ leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
+ # Compare leaderboards
+ result = all( i == leaderList[0] for i in leaderList ) and\
+ leaderList is not None
+ main.log.debug( leaderList )
+ main.log.warn( result )
+ if result:
+ return ( result, leaderList )
+ time.sleep(5) # TODO: paramerterize
+ main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
+ return ( result, leaderList )
+
+ def nodesCheck( self, nodes ):
+ nodesOutput = []
+ results = True
+ threads = []
+ for i in nodes:
+ t = main.Thread( target=main.CLIs[i].nodes,
+ name="nodes-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ nodesOutput.append( t.result )
+ ips = [ main.nodes[node].ip_address for node in nodes ]
+ ips.sort()
+ for i in nodesOutput:
+ try:
+ current = json.loads( i )
+ activeIps = []
+ currentResult = False
+ for node in current:
+ if node['state'] == 'READY':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = True
+ except ( ValueError, TypeError ):
+ main.log.error( "Error parsing nodes output" )
+ main.log.warn( repr( i ) )
+ currentResult = False
+ results = results and currentResult
+ return results
diff --git a/TestON/tests/HAkillNodes/HAkillNodes.params b/TestON/tests/HAkillNodes/HAkillNodes.params
index 1296300..134f5f2 100644
--- a/TestON/tests/HAkillNodes/HAkillNodes.params
+++ b/TestON/tests/HAkillNodes/HAkillNodes.params
@@ -24,7 +24,7 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>false</useFlowObjectives>
+ <useFlowObjectives>true</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
</ONOS_Configuration>
<ENV>
diff --git a/TestON/tests/HAkillNodes/HAkillNodes.py b/TestON/tests/HAkillNodes/HAkillNodes.py
index 92d2fd5..4d75d8d 100644
--- a/TestON/tests/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HAkillNodes/HAkillNodes.py
@@ -95,8 +95,8 @@
ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
try:
- from tests.HAsanity.dependencies.Counters import Counters
- main.Counters = Counters()
+ from tests.HAsanity.dependencies.HA import HA
+ main.HA = HA()
except Exception as e:
main.log.exception( e )
main.cleanup()
@@ -287,38 +287,12 @@
handle.expect( "\$" )
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
@@ -994,6 +968,7 @@
"functionality and check the state of " +\
"the intent"
+ onosCli = main.CLIs[ main.activeNodes[0] ]
main.step( "Check Intent state" )
installedCheck = False
loopCount = 0
@@ -1029,7 +1004,6 @@
"INSTALLED state" )
main.step( "Ping across added host intents" )
- onosCli = main.CLIs[ main.activeNodes[0] ]
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -1226,8 +1200,7 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
# Print flowrules
- node = main.activeNodes[0]
- main.log.debug( main.CLIs[node].flows( jsonFormat=False ) )
+ main.log.debug( onosCli.flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
# the wait is above
PingResult = main.TRUE
@@ -2176,8 +2149,9 @@
for i in range( 28 ):
main.log.info( "Checking flow table on s" + str( i + 1 ) )
tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
- FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
- if FlowTables == main.FALSE:
+ curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
+ FlowTables = FlowTables and curSwitch
+ if curSwitch == main.FALSE:
main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
utilities.assert_equals(
expect=main.TRUE,
@@ -2656,45 +2630,19 @@
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
if not nodeResults:
- for cli in main.CLIs:
+ for i in main.activeNodes:
main.log.debug( "{} components not ACTIVE: \n{}".format(
- cli.name,
- cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.CLIs[i].name,
+ main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
def CASE9( self, main ):
"""
@@ -3014,26 +2962,8 @@
main.step( "Check that each node shows the same leader and candidates" )
failMessage = "Nodes have different leaderboards"
- def consistentLeaderboards( nodes ):
- TOPIC = 'org.onosproject.election'
- # FIXME: use threads
- #FIXME: should we retry outside the function?
- for n in range( 5 ): # Retry in case election is still happening
- leaderList = []
- # Get all leaderboards
- for cli in nodes:
- leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
- # Compare leaderboards
- result = all( i == leaderList[0] for i in leaderList ) and\
- leaderList is not None
- main.log.debug( leaderList )
- main.log.warn( result )
- if result:
- return ( result, leaderList )
- time.sleep(5) #TODO: paramerterize
- main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
- sameResult, oldLeaders = consistentLeaderboards( activeCLIs )
+ sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
if sameResult:
oldLeader = oldLeaders[ 0 ][ 0 ]
main.log.warn( oldLeader )
@@ -3069,7 +2999,7 @@
main.step( "Check that a new node was elected leader" )
failMessage = "Nodes have different leaders"
# Get new leaders and candidates
- newLeaderResult, newLeaders = consistentLeaderboards( activeCLIs )
+ newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
if newLeaders[ 0 ][ 0 ] == 'none':
main.log.error( "No leader was elected on at least 1 node" )
if not expectNoLeader:
@@ -3137,7 +3067,7 @@
# Get new leaders and candidates
reRunLeaders = []
time.sleep( 5 ) # Paremterize
- positionResult, reRunLeaders = consistentLeaderboards( activeCLIs )
+ positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
# Check that the re-elected node is last on the candidate List
if oldLeader != reRunLeaders[ 0 ][ -1 ]:
@@ -3281,7 +3211,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
@@ -3381,7 +3311,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
diff --git a/TestON/tests/HAkillNodes/dependencies/Counters.py b/TestON/tests/HAkillNodes/dependencies/Counters.py
deleted file mode 100644
index 192b919..0000000
--- a/TestON/tests/HAkillNodes/dependencies/Counters.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import json
-
-class Counters():
-
- def __init__( self ):
- self.default = ''
-
- def consistentCheck( self ):
- """
- Checks that TestON counters are consistent across all nodes.
-
- Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results
- onosCountersRaw = []
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=utilities.retry,
- name="counters-" + str( i ),
- args=[ main.CLIs[i].counters, [ None ] ],
- kwargs= { 'sleep': 5, 'attempts': 5,
- 'randomTime': True } )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCountersRaw.append( t.result )
- onosCounters = []
- for i in range( len( main.activeNodes ) ):
- try:
- onosCounters.append( json.loads( onosCountersRaw[i] ) )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( main.activeNodes[i] + 1 ) )
- main.log.warn( repr( onosCountersRaw[ i ] ) )
- onosCounters.append( [] )
-
- testCounters = {}
- # make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
- # { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
- # }
- # NOTE: There is an assumtion that all nodes are active
- # based on the above for loops
- for controller in enumerate( onosCounters ):
- for key, value in controller[1].iteritems():
- if 'TestON' in key:
- node = 'ONOS' + str( controller[0] + 1 )
- try:
- testCounters[node].append( { key: value } )
- except KeyError:
- testCounters[node] = [ { key: value } ]
- # compare the counters on each node
- firstV = testCounters.values()[0]
- tmp = [ v == firstV for k, v in testCounters.iteritems() ]
- if all( tmp ):
- consistent = main.TRUE
- else:
- consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters:\n" +
- testCounters )
- return ( onosCounters, consistent )
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
-
- def counterCheck( self, counterName, counterValue ):
- """
- Checks that TestON counters are consistent across all nodes and that
- specified counter is in ONOS with the given value
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results and consistentCheck
- onosCounters, consistent = self.consistentCheck()
- # Check for correct values
- for i in range( len( main.activeNodes ) ):
- current = onosCounters[i]
- onosValue = None
- try:
- onosValue = current.get( counterName )
- except AttributeError, e:
- node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
- correctResults = main.FALSE
- if onosValue == counterValue:
- main.log.info( counterName + " counter value is correct" )
- else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
- correctResults = main.FALSE
- return consistent and correctResults
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
diff --git a/TestON/tests/HAclusterRestart/dependencies/Counters.py b/TestON/tests/HAkillNodes/dependencies/HA.py
similarity index 60%
copy from TestON/tests/HAclusterRestart/dependencies/Counters.py
copy to TestON/tests/HAkillNodes/dependencies/HA.py
index 192b919..e00d290 100644
--- a/TestON/tests/HAclusterRestart/dependencies/Counters.py
+++ b/TestON/tests/HAkillNodes/dependencies/HA.py
@@ -1,6 +1,7 @@
import json
+import time
-class Counters():
+class HA():
def __init__( self ):
self.default = ''
@@ -10,12 +11,12 @@
Checks that TestON counters are consistent across all nodes.
Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
+ - onosCounters is the parsed json output of the counters command on
+ all nodes
+ - consistent is main.TRUE if all "TestON" counters are consitent across
+ all nodes or main.FALSE
"""
try:
- correctResults = main.TRUE
# Get onos counters results
onosCountersRaw = []
threads = []
@@ -42,8 +43,8 @@
testCounters = {}
# make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
+ # lookes like a dict whose keys are the name of the ONOS node and
+ # values are a list of the counters. I.E.
# { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
# }
# NOTE: There is an assumtion that all nodes are active
@@ -86,20 +87,75 @@
onosValue = None
try:
onosValue = current.get( counterName )
- except AttributeError, e:
+ except AttributeError:
node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
+ main.log.exception( "ONOS" + node + " counters result " +
+ "is not as expected" )
correctResults = main.FALSE
if onosValue == counterValue:
main.log.info( counterName + " counter value is correct" )
else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
+ main.log.error( counterName +
+ " counter value is incorrect," +
+ " expected value: " + str( counterValue ) +
+ " current value: " + str( onosValue ) )
correctResults = main.FALSE
return consistent and correctResults
except Exception:
main.log.exception( "" )
main.cleanup()
main.exit()
+
+ def consistentLeaderboards( self, nodes ):
+ TOPIC = 'org.onosproject.election'
+ # FIXME: use threads
+ #FIXME: should we retry outside the function?
+ for n in range( 5 ): # Retry in case election is still happening
+ leaderList = []
+ # Get all leaderboards
+ for cli in nodes:
+ leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
+ # Compare leaderboards
+ result = all( i == leaderList[0] for i in leaderList ) and\
+ leaderList is not None
+ main.log.debug( leaderList )
+ main.log.warn( result )
+ if result:
+ return ( result, leaderList )
+ time.sleep(5) # TODO: paramerterize
+ main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
+ return ( result, leaderList )
+
+ def nodesCheck( self, nodes ):
+ nodesOutput = []
+ results = True
+ threads = []
+ for i in nodes:
+ t = main.Thread( target=main.CLIs[i].nodes,
+ name="nodes-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ nodesOutput.append( t.result )
+ ips = [ main.nodes[node].ip_address for node in nodes ]
+ ips.sort()
+ for i in nodesOutput:
+ try:
+ current = json.loads( i )
+ activeIps = []
+ currentResult = False
+ for node in current:
+ if node['state'] == 'READY':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = True
+ except ( ValueError, TypeError ):
+ main.log.error( "Error parsing nodes output" )
+ main.log.warn( repr( i ) )
+ currentResult = False
+ results = results and currentResult
+ return results
diff --git a/TestON/tests/HAsanity/HAsanity.params b/TestON/tests/HAsanity/HAsanity.params
index a3f123b..4b52fb1 100644
--- a/TestON/tests/HAsanity/HAsanity.params
+++ b/TestON/tests/HAsanity/HAsanity.params
@@ -23,7 +23,7 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>false</useFlowObjectives>
+ <useFlowObjectives>true</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
</ONOS_Configuration>
<ENV>
diff --git a/TestON/tests/HAsanity/HAsanity.py b/TestON/tests/HAsanity/HAsanity.py
index 0295b3b..499b23a 100644
--- a/TestON/tests/HAsanity/HAsanity.py
+++ b/TestON/tests/HAsanity/HAsanity.py
@@ -94,8 +94,8 @@
ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
try:
- from tests.HAsanity.dependencies.Counters import Counters
- main.Counters = Counters()
+ from tests.HAsanity.dependencies.HA import HA
+ main.HA = HA()
except Exception as e:
main.log.exception( e )
main.cleanup()
@@ -252,38 +252,12 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
@@ -959,6 +933,7 @@
"functionality and check the state of " +\
"the intent"
+ onosCli = main.CLIs[ main.activeNodes[0] ]
main.step( "Check Intent state" )
installedCheck = False
loopCount = 0
@@ -994,7 +969,6 @@
"INSTALLED state" )
main.step( "Ping across added host intents" )
- onosCli = main.CLIs[ main.activeNodes[0] ]
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -2075,8 +2049,9 @@
for i in range( 28 ):
main.log.info( "Checking flow table on s" + str( i + 1 ) )
tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
- FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
- if FlowTables == main.FALSE:
+ curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
+ FlowTables = FlowTables and curSwitch
+ if curSwitch == main.FALSE:
main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
utilities.assert_equals(
expect=main.TRUE,
@@ -2556,45 +2531,19 @@
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
if not nodeResults:
- for cli in main.CLIs:
+ for i in main.activeNodes:
main.log.debug( "{} components not ACTIVE: \n{}".format(
- cli.name,
- cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.CLIs[i].name,
+ main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
def CASE9( self, main ):
"""
@@ -2914,26 +2863,8 @@
main.step( "Check that each node shows the same leader and candidates" )
failMessage = "Nodes have different leaderboards"
- def consistentLeaderboards( nodes ):
- TOPIC = 'org.onosproject.election'
- # FIXME: use threads
- #FIXME: should we retry outside the function?
- for n in range( 5 ): # Retry in case election is still happening
- leaderList = []
- # Get all leaderboards
- for cli in nodes:
- leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
- # Compare leaderboards
- result = all( i == leaderList[0] for i in leaderList ) and\
- leaderList is not None
- main.log.debug( leaderList )
- main.log.warn( result )
- if result:
- return ( result, leaderList )
- time.sleep(5) #TODO: paramerterize
- main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
- sameResult, oldLeaders = consistentLeaderboards( activeCLIs )
+ sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
if sameResult:
oldLeader = oldLeaders[ 0 ][ 0 ]
main.log.warn( oldLeader )
@@ -2969,7 +2900,7 @@
main.step( "Check that a new node was elected leader" )
failMessage = "Nodes have different leaders"
# Get new leaders and candidates
- newLeaderResult, newLeaders = consistentLeaderboards( activeCLIs )
+ newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
if newLeaders[ 0 ][ 0 ] == 'none':
main.log.error( "No leader was elected on at least 1 node" )
if not expectNoLeader:
@@ -3037,7 +2968,7 @@
# Get new leaders and candidates
reRunLeaders = []
time.sleep( 5 ) # Paremterize
- positionResult, reRunLeaders = consistentLeaderboards( activeCLIs )
+ positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
# Check that the re-elected node is last on the candidate List
if oldLeader != reRunLeaders[ 0 ][ -1 ]:
@@ -3181,7 +3112,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
@@ -3281,7 +3212,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
diff --git a/TestON/tests/HAsanity/dependencies/Counters.py b/TestON/tests/HAsanity/dependencies/Counters.py
deleted file mode 100644
index 192b919..0000000
--- a/TestON/tests/HAsanity/dependencies/Counters.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import json
-
-class Counters():
-
- def __init__( self ):
- self.default = ''
-
- def consistentCheck( self ):
- """
- Checks that TestON counters are consistent across all nodes.
-
- Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results
- onosCountersRaw = []
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=utilities.retry,
- name="counters-" + str( i ),
- args=[ main.CLIs[i].counters, [ None ] ],
- kwargs= { 'sleep': 5, 'attempts': 5,
- 'randomTime': True } )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCountersRaw.append( t.result )
- onosCounters = []
- for i in range( len( main.activeNodes ) ):
- try:
- onosCounters.append( json.loads( onosCountersRaw[i] ) )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( main.activeNodes[i] + 1 ) )
- main.log.warn( repr( onosCountersRaw[ i ] ) )
- onosCounters.append( [] )
-
- testCounters = {}
- # make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
- # { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
- # }
- # NOTE: There is an assumtion that all nodes are active
- # based on the above for loops
- for controller in enumerate( onosCounters ):
- for key, value in controller[1].iteritems():
- if 'TestON' in key:
- node = 'ONOS' + str( controller[0] + 1 )
- try:
- testCounters[node].append( { key: value } )
- except KeyError:
- testCounters[node] = [ { key: value } ]
- # compare the counters on each node
- firstV = testCounters.values()[0]
- tmp = [ v == firstV for k, v in testCounters.iteritems() ]
- if all( tmp ):
- consistent = main.TRUE
- else:
- consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters:\n" +
- testCounters )
- return ( onosCounters, consistent )
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
-
- def counterCheck( self, counterName, counterValue ):
- """
- Checks that TestON counters are consistent across all nodes and that
- specified counter is in ONOS with the given value
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results and consistentCheck
- onosCounters, consistent = self.consistentCheck()
- # Check for correct values
- for i in range( len( main.activeNodes ) ):
- current = onosCounters[i]
- onosValue = None
- try:
- onosValue = current.get( counterName )
- except AttributeError, e:
- node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
- correctResults = main.FALSE
- if onosValue == counterValue:
- main.log.info( counterName + " counter value is correct" )
- else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
- correctResults = main.FALSE
- return consistent and correctResults
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
diff --git a/TestON/tests/HAclusterRestart/dependencies/Counters.py b/TestON/tests/HAsanity/dependencies/HA.py
similarity index 60%
copy from TestON/tests/HAclusterRestart/dependencies/Counters.py
copy to TestON/tests/HAsanity/dependencies/HA.py
index 192b919..e00d290 100644
--- a/TestON/tests/HAclusterRestart/dependencies/Counters.py
+++ b/TestON/tests/HAsanity/dependencies/HA.py
@@ -1,6 +1,7 @@
import json
+import time
-class Counters():
+class HA():
def __init__( self ):
self.default = ''
@@ -10,12 +11,12 @@
Checks that TestON counters are consistent across all nodes.
Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
+ - onosCounters is the parsed json output of the counters command on
+ all nodes
+ - consistent is main.TRUE if all "TestON" counters are consitent across
+ all nodes or main.FALSE
"""
try:
- correctResults = main.TRUE
# Get onos counters results
onosCountersRaw = []
threads = []
@@ -42,8 +43,8 @@
testCounters = {}
# make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
+ # lookes like a dict whose keys are the name of the ONOS node and
+ # values are a list of the counters. I.E.
# { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
# }
# NOTE: There is an assumtion that all nodes are active
@@ -86,20 +87,75 @@
onosValue = None
try:
onosValue = current.get( counterName )
- except AttributeError, e:
+ except AttributeError:
node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
+ main.log.exception( "ONOS" + node + " counters result " +
+ "is not as expected" )
correctResults = main.FALSE
if onosValue == counterValue:
main.log.info( counterName + " counter value is correct" )
else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
+ main.log.error( counterName +
+ " counter value is incorrect," +
+ " expected value: " + str( counterValue ) +
+ " current value: " + str( onosValue ) )
correctResults = main.FALSE
return consistent and correctResults
except Exception:
main.log.exception( "" )
main.cleanup()
main.exit()
+
+ def consistentLeaderboards( self, nodes ):
+ TOPIC = 'org.onosproject.election'
+ # FIXME: use threads
+ #FIXME: should we retry outside the function?
+ for n in range( 5 ): # Retry in case election is still happening
+ leaderList = []
+ # Get all leaderboards
+ for cli in nodes:
+ leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
+ # Compare leaderboards
+ result = all( i == leaderList[0] for i in leaderList ) and\
+ leaderList is not None
+ main.log.debug( leaderList )
+ main.log.warn( result )
+ if result:
+ return ( result, leaderList )
+ time.sleep(5) # TODO: paramerterize
+ main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
+ return ( result, leaderList )
+
+ def nodesCheck( self, nodes ):
+ nodesOutput = []
+ results = True
+ threads = []
+ for i in nodes:
+ t = main.Thread( target=main.CLIs[i].nodes,
+ name="nodes-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ nodesOutput.append( t.result )
+ ips = [ main.nodes[node].ip_address for node in nodes ]
+ ips.sort()
+ for i in nodesOutput:
+ try:
+ current = json.loads( i )
+ activeIps = []
+ currentResult = False
+ for node in current:
+ if node['state'] == 'READY':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = True
+ except ( ValueError, TypeError ):
+ main.log.error( "Error parsing nodes output" )
+ main.log.warn( repr( i ) )
+ currentResult = False
+ results = results and currentResult
+ return results
diff --git a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.params b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.params
index c3bcc23..33943ff 100644
--- a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.params
+++ b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.params
@@ -19,7 +19,7 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>false</useFlowObjectives>
+ <useFlowObjectives>true</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
</ONOS_Configuration>
<ENV>
diff --git a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index 6c2b01f..de14bf9 100644
--- a/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -70,8 +70,8 @@
main.numCtrls = int( main.ONOSbench.maxNodes )
try:
- from tests.HAsanity.dependencies.Counters import Counters
- main.Counters = Counters()
+ from tests.HAsanity.dependencies.HA import HA
+ main.HA = HA()
except Exception as e:
main.log.exception( e )
main.cleanup()
@@ -241,38 +241,12 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
@@ -919,6 +893,7 @@
"functionality and check the state of " +\
"the intent"
+ onosCli = main.CLIs[ main.activeNodes[0] ]
main.step( "Check Intent state" )
installedCheck = True
# Print the intent states
@@ -948,7 +923,6 @@
"INSTALLED state" )
main.step( "Ping across added host intents" )
- onosCli = main.CLIs[ main.activeNodes[0] ]
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -1601,8 +1575,9 @@
for i in range( 28 ):
main.log.info( "Checking flow table on s" + str( i + 1 ) )
tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
- FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
- if FlowTables == main.FALSE:
+ curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
+ FlowTables = FlowTables and curSwitch
+ if curSwitch == main.FALSE:
main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
utilities.assert_equals(
expect=main.TRUE,
@@ -1865,6 +1840,20 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
+
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+ if not nodeResults:
+ for i in main.activeNodes:
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ main.CLIs[i].name,
+ main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
def CASE9( self, main ):
"""
@@ -2152,26 +2141,8 @@
main.step( "Check that each node shows the same leader and candidates" )
failMessage = "Nodes have different leaderboards"
- def consistentLeaderboards( nodes ):
- TOPIC = 'org.onosproject.election'
- # FIXME: use threads
- #FIXME: should we retry outside the function?
- for n in range( 5 ): # Retry in case election is still happening
- leaderList = []
- # Get all leaderboards
- for cli in nodes:
- leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
- # Compare leaderboards
- result = all( i == leaderList[0] for i in leaderList ) and\
- leaderList is not None
- main.log.debug( leaderList )
- main.log.warn( result )
- if result:
- return ( result, leaderList )
- time.sleep(5) #TODO: paramerterize
- main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
- sameResult, oldLeaders = consistentLeaderboards( activeCLIs )
+ sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
if sameResult:
oldLeader = oldLeaders[ 0 ][ 0 ]
main.log.warn( oldLeader )
@@ -2207,7 +2178,7 @@
main.step( "Check that a new node was elected leader" )
failMessage = "Nodes have different leaders"
# Get new leaders and candidates
- newLeaderResult, newLeaders = consistentLeaderboards( activeCLIs )
+ newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
if newLeaders[ 0 ][ 0 ] == 'none':
main.log.error( "No leader was elected on at least 1 node" )
if not expectNoLeader:
@@ -2275,7 +2246,7 @@
# Get new leaders and candidates
reRunLeaders = []
time.sleep( 5 ) # Paremterize
- positionResult, reRunLeaders = consistentLeaderboards( activeCLIs )
+ positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
# Check that the re-elected node is last on the candidate List
if oldLeader != reRunLeaders[ 0 ][ -1 ]:
@@ -2419,7 +2390,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
@@ -2519,7 +2490,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
diff --git a/TestON/tests/HAsingleInstanceRestart/dependencies/Counters.py b/TestON/tests/HAsingleInstanceRestart/dependencies/Counters.py
deleted file mode 100644
index 192b919..0000000
--- a/TestON/tests/HAsingleInstanceRestart/dependencies/Counters.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import json
-
-class Counters():
-
- def __init__( self ):
- self.default = ''
-
- def consistentCheck( self ):
- """
- Checks that TestON counters are consistent across all nodes.
-
- Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results
- onosCountersRaw = []
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=utilities.retry,
- name="counters-" + str( i ),
- args=[ main.CLIs[i].counters, [ None ] ],
- kwargs= { 'sleep': 5, 'attempts': 5,
- 'randomTime': True } )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCountersRaw.append( t.result )
- onosCounters = []
- for i in range( len( main.activeNodes ) ):
- try:
- onosCounters.append( json.loads( onosCountersRaw[i] ) )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( main.activeNodes[i] + 1 ) )
- main.log.warn( repr( onosCountersRaw[ i ] ) )
- onosCounters.append( [] )
-
- testCounters = {}
- # make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
- # { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
- # }
- # NOTE: There is an assumtion that all nodes are active
- # based on the above for loops
- for controller in enumerate( onosCounters ):
- for key, value in controller[1].iteritems():
- if 'TestON' in key:
- node = 'ONOS' + str( controller[0] + 1 )
- try:
- testCounters[node].append( { key: value } )
- except KeyError:
- testCounters[node] = [ { key: value } ]
- # compare the counters on each node
- firstV = testCounters.values()[0]
- tmp = [ v == firstV for k, v in testCounters.iteritems() ]
- if all( tmp ):
- consistent = main.TRUE
- else:
- consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters:\n" +
- testCounters )
- return ( onosCounters, consistent )
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
-
- def counterCheck( self, counterName, counterValue ):
- """
- Checks that TestON counters are consistent across all nodes and that
- specified counter is in ONOS with the given value
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results and consistentCheck
- onosCounters, consistent = self.consistentCheck()
- # Check for correct values
- for i in range( len( main.activeNodes ) ):
- current = onosCounters[i]
- onosValue = None
- try:
- onosValue = current.get( counterName )
- except AttributeError, e:
- node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
- correctResults = main.FALSE
- if onosValue == counterValue:
- main.log.info( counterName + " counter value is correct" )
- else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
- correctResults = main.FALSE
- return consistent and correctResults
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
diff --git a/TestON/tests/HAclusterRestart/dependencies/Counters.py b/TestON/tests/HAsingleInstanceRestart/dependencies/HA.py
similarity index 60%
copy from TestON/tests/HAclusterRestart/dependencies/Counters.py
copy to TestON/tests/HAsingleInstanceRestart/dependencies/HA.py
index 192b919..e00d290 100644
--- a/TestON/tests/HAclusterRestart/dependencies/Counters.py
+++ b/TestON/tests/HAsingleInstanceRestart/dependencies/HA.py
@@ -1,6 +1,7 @@
import json
+import time
-class Counters():
+class HA():
def __init__( self ):
self.default = ''
@@ -10,12 +11,12 @@
Checks that TestON counters are consistent across all nodes.
Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
+ - onosCounters is the parsed json output of the counters command on
+ all nodes
+ - consistent is main.TRUE if all "TestON" counters are consitent across
+ all nodes or main.FALSE
"""
try:
- correctResults = main.TRUE
# Get onos counters results
onosCountersRaw = []
threads = []
@@ -42,8 +43,8 @@
testCounters = {}
# make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
+ # lookes like a dict whose keys are the name of the ONOS node and
+ # values are a list of the counters. I.E.
# { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
# }
# NOTE: There is an assumtion that all nodes are active
@@ -86,20 +87,75 @@
onosValue = None
try:
onosValue = current.get( counterName )
- except AttributeError, e:
+ except AttributeError:
node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
+ main.log.exception( "ONOS" + node + " counters result " +
+ "is not as expected" )
correctResults = main.FALSE
if onosValue == counterValue:
main.log.info( counterName + " counter value is correct" )
else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
+ main.log.error( counterName +
+ " counter value is incorrect," +
+ " expected value: " + str( counterValue ) +
+ " current value: " + str( onosValue ) )
correctResults = main.FALSE
return consistent and correctResults
except Exception:
main.log.exception( "" )
main.cleanup()
main.exit()
+
+ def consistentLeaderboards( self, nodes ):
+ TOPIC = 'org.onosproject.election'
+ # FIXME: use threads
+ #FIXME: should we retry outside the function?
+ for n in range( 5 ): # Retry in case election is still happening
+ leaderList = []
+ # Get all leaderboards
+ for cli in nodes:
+ leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
+ # Compare leaderboards
+ result = all( i == leaderList[0] for i in leaderList ) and\
+ leaderList is not None
+ main.log.debug( leaderList )
+ main.log.warn( result )
+ if result:
+ return ( result, leaderList )
+ time.sleep(5) # TODO: paramerterize
+ main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
+ return ( result, leaderList )
+
+ def nodesCheck( self, nodes ):
+ nodesOutput = []
+ results = True
+ threads = []
+ for i in nodes:
+ t = main.Thread( target=main.CLIs[i].nodes,
+ name="nodes-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ nodesOutput.append( t.result )
+ ips = [ main.nodes[node].ip_address for node in nodes ]
+ ips.sort()
+ for i in nodesOutput:
+ try:
+ current = json.loads( i )
+ activeIps = []
+ currentResult = False
+ for node in current:
+ if node['state'] == 'READY':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = True
+ except ( ValueError, TypeError ):
+ main.log.error( "Error parsing nodes output" )
+ main.log.warn( repr( i ) )
+ currentResult = False
+ results = results and currentResult
+ return results
diff --git a/TestON/tests/HAstopNodes/HAstopNodes.params b/TestON/tests/HAstopNodes/HAstopNodes.params
index 1296300..134f5f2 100644
--- a/TestON/tests/HAstopNodes/HAstopNodes.params
+++ b/TestON/tests/HAstopNodes/HAstopNodes.params
@@ -24,7 +24,7 @@
<apps></apps>
<ONOS_Configuration>
<org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
- <useFlowObjectives>false</useFlowObjectives>
+ <useFlowObjectives>true</useFlowObjectives>
</org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
</ONOS_Configuration>
<ENV>
diff --git a/TestON/tests/HAstopNodes/HAstopNodes.py b/TestON/tests/HAstopNodes/HAstopNodes.py
index a09a3fc..6a594b2 100644
--- a/TestON/tests/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HAstopNodes/HAstopNodes.py
@@ -95,8 +95,8 @@
ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
try:
- from tests.HAsanity.dependencies.Counters import Counters
- main.Counters = Counters()
+ from tests.HAsanity.dependencies.HA import HA
+ main.HA = HA()
except Exception as e:
main.log.exception( e )
main.cleanup()
@@ -276,38 +276,12 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
@@ -973,6 +947,7 @@
"functionality and check the state of " +\
"the intent"
+ onosCli = main.CLIs[ main.activeNodes[0] ]
main.step( "Check Intent state" )
installedCheck = False
loopCount = 0
@@ -1008,7 +983,6 @@
"INSTALLED state" )
main.step( "Ping across added host intents" )
- onosCli = main.CLIs[ main.activeNodes[0] ]
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -1205,8 +1179,7 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
# Print flowrules
- node = main.activeNodes[0]
- main.log.debug( main.CLIs[node].flows( jsonFormat=False ) )
+ main.log.debug( onosCli.flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
# the wait is above
PingResult = main.TRUE
@@ -2153,8 +2126,9 @@
for i in range( 28 ):
main.log.info( "Checking flow table on s" + str( i + 1 ) )
tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
- FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
- if FlowTables == main.FALSE:
+ curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
+ FlowTables = FlowTables and curSwitch
+ if curSwitch == main.FALSE:
main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
utilities.assert_equals(
expect=main.TRUE,
@@ -2633,45 +2607,19 @@
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
- nodesOutput = []
- nodeResults = main.TRUE
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=main.CLIs[i].nodes,
- name="nodes-" + str( i ),
- args=[ ] )
- threads.append( t )
- t.start()
+ nodeResults = utilities.retry( main.HA.nodesCheck,
+ False,
+ args=[main.activeNodes],
+ attempts=5 )
- for t in threads:
- t.join()
- nodesOutput.append( t.result )
- ips = [ main.nodes[node].ip_address for node in main.activeNodes ]
- ips.sort()
- for i in nodesOutput:
- try:
- current = json.loads( i )
- activeIps = []
- currentResult = main.FALSE
- for node in current:
- if node['state'] == 'READY':
- activeIps.append( node['ip'] )
- activeIps.sort()
- if ips == activeIps:
- currentResult = main.TRUE
- except ( ValueError, TypeError ):
- main.log.error( "Error parsing nodes output" )
- main.log.warn( repr( i ) )
- currentResult = main.FALSE
- nodeResults = nodeResults and currentResult
- utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
+ utilities.assert_equals( expect=True, actual=nodeResults,
onpass="Nodes check successful",
onfail="Nodes check NOT successful" )
if not nodeResults:
- for cli in main.CLIs:
+ for i in main.activeNodes:
main.log.debug( "{} components not ACTIVE: \n{}".format(
- cli.name,
- cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.CLIs[i].name,
+ main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
def CASE9( self, main ):
"""
@@ -2991,26 +2939,8 @@
main.step( "Check that each node shows the same leader and candidates" )
failMessage = "Nodes have different leaderboards"
- def consistentLeaderboards( nodes ):
- TOPIC = 'org.onosproject.election'
- # FIXME: use threads
- #FIXME: should we retry outside the function?
- for n in range( 5 ): # Retry in case election is still happening
- leaderList = []
- # Get all leaderboards
- for cli in nodes:
- leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
- # Compare leaderboards
- result = all( i == leaderList[0] for i in leaderList ) and\
- leaderList is not None
- main.log.debug( leaderList )
- main.log.warn( result )
- if result:
- return ( result, leaderList )
- time.sleep(5) #TODO: paramerterize
- main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
- sameResult, oldLeaders = consistentLeaderboards( activeCLIs )
+ sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
if sameResult:
oldLeader = oldLeaders[ 0 ][ 0 ]
main.log.warn( oldLeader )
@@ -3046,7 +2976,7 @@
main.step( "Check that a new node was elected leader" )
failMessage = "Nodes have different leaders"
# Get new leaders and candidates
- newLeaderResult, newLeaders = consistentLeaderboards( activeCLIs )
+ newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
if newLeaders[ 0 ][ 0 ] == 'none':
main.log.error( "No leader was elected on at least 1 node" )
if not expectNoLeader:
@@ -3114,7 +3044,7 @@
# Get new leaders and candidates
reRunLeaders = []
time.sleep( 5 ) # Paremterize
- positionResult, reRunLeaders = consistentLeaderboards( activeCLIs )
+ positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
# Check that the re-elected node is last on the candidate List
if oldLeader != reRunLeaders[ 0 ][ -1 ]:
@@ -3258,7 +3188,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
@@ -3358,7 +3288,7 @@
" counter" )
main.step( "Counters we added have the correct values" )
- incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
+ incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
utilities.assert_equals( expect=main.TRUE,
actual=incrementCheck,
onpass="Added counters are correct",
diff --git a/TestON/tests/HAstopNodes/dependencies/Counters.py b/TestON/tests/HAstopNodes/dependencies/Counters.py
deleted file mode 100644
index 192b919..0000000
--- a/TestON/tests/HAstopNodes/dependencies/Counters.py
+++ /dev/null
@@ -1,105 +0,0 @@
-import json
-
-class Counters():
-
- def __init__( self ):
- self.default = ''
-
- def consistentCheck( self ):
- """
- Checks that TestON counters are consistent across all nodes.
-
- Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results
- onosCountersRaw = []
- threads = []
- for i in main.activeNodes:
- t = main.Thread( target=utilities.retry,
- name="counters-" + str( i ),
- args=[ main.CLIs[i].counters, [ None ] ],
- kwargs= { 'sleep': 5, 'attempts': 5,
- 'randomTime': True } )
- threads.append( t )
- t.start()
- for t in threads:
- t.join()
- onosCountersRaw.append( t.result )
- onosCounters = []
- for i in range( len( main.activeNodes ) ):
- try:
- onosCounters.append( json.loads( onosCountersRaw[i] ) )
- except ( ValueError, TypeError ):
- main.log.error( "Could not parse counters response from ONOS" +
- str( main.activeNodes[i] + 1 ) )
- main.log.warn( repr( onosCountersRaw[ i ] ) )
- onosCounters.append( [] )
-
- testCounters = {}
- # make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
- # { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
- # }
- # NOTE: There is an assumtion that all nodes are active
- # based on the above for loops
- for controller in enumerate( onosCounters ):
- for key, value in controller[1].iteritems():
- if 'TestON' in key:
- node = 'ONOS' + str( controller[0] + 1 )
- try:
- testCounters[node].append( { key: value } )
- except KeyError:
- testCounters[node] = [ { key: value } ]
- # compare the counters on each node
- firstV = testCounters.values()[0]
- tmp = [ v == firstV for k, v in testCounters.iteritems() ]
- if all( tmp ):
- consistent = main.TRUE
- else:
- consistent = main.FALSE
- main.log.error( "ONOS nodes have different values for counters:\n" +
- testCounters )
- return ( onosCounters, consistent )
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
-
- def counterCheck( self, counterName, counterValue ):
- """
- Checks that TestON counters are consistent across all nodes and that
- specified counter is in ONOS with the given value
- """
- try:
- correctResults = main.TRUE
- # Get onos counters results and consistentCheck
- onosCounters, consistent = self.consistentCheck()
- # Check for correct values
- for i in range( len( main.activeNodes ) ):
- current = onosCounters[i]
- onosValue = None
- try:
- onosValue = current.get( counterName )
- except AttributeError, e:
- node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
- correctResults = main.FALSE
- if onosValue == counterValue:
- main.log.info( counterName + " counter value is correct" )
- else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
- correctResults = main.FALSE
- return consistent and correctResults
- except Exception:
- main.log.exception( "" )
- main.cleanup()
- main.exit()
diff --git a/TestON/tests/HAclusterRestart/dependencies/Counters.py b/TestON/tests/HAstopNodes/dependencies/HA.py
similarity index 60%
copy from TestON/tests/HAclusterRestart/dependencies/Counters.py
copy to TestON/tests/HAstopNodes/dependencies/HA.py
index 192b919..e00d290 100644
--- a/TestON/tests/HAclusterRestart/dependencies/Counters.py
+++ b/TestON/tests/HAstopNodes/dependencies/HA.py
@@ -1,6 +1,7 @@
import json
+import time
-class Counters():
+class HA():
def __init__( self ):
self.default = ''
@@ -10,12 +11,12 @@
Checks that TestON counters are consistent across all nodes.
Returns the tuple (onosCounters, consistent)
- - onosCounters is the parsed json output of the counters command on all nodes
- - consistent is main.TRUE if all "TestON" counters are consitent across all
- nodes or main.FALSE
+ - onosCounters is the parsed json output of the counters command on
+ all nodes
+ - consistent is main.TRUE if all "TestON" counters are consitent across
+ all nodes or main.FALSE
"""
try:
- correctResults = main.TRUE
# Get onos counters results
onosCountersRaw = []
threads = []
@@ -42,8 +43,8 @@
testCounters = {}
# make a list of all the "TestON-*" counters in ONOS
- # lookes like a dict whose keys are the name of the ONOS node and values
- # are a list of the counters. I.E.
+ # lookes like a dict whose keys are the name of the ONOS node and
+ # values are a list of the counters. I.E.
# { "ONOS1": [ { "name":"TestON-Partitions","value":56} ]
# }
# NOTE: There is an assumtion that all nodes are active
@@ -86,20 +87,75 @@
onosValue = None
try:
onosValue = current.get( counterName )
- except AttributeError, e:
+ except AttributeError:
node = str( main.activeNodes[i] + 1 )
- main.log.error( "ONOS" + node + " counters result " +
- "is not as expected" )
+ main.log.exception( "ONOS" + node + " counters result " +
+ "is not as expected" )
correctResults = main.FALSE
if onosValue == counterValue:
main.log.info( counterName + " counter value is correct" )
else:
- main.log.error( counterName + " counter value is incorrect," +
- " expected value: " + str( counterValue )
- + " current value: " + str( onosValue ) )
+ main.log.error( counterName +
+ " counter value is incorrect," +
+ " expected value: " + str( counterValue ) +
+ " current value: " + str( onosValue ) )
correctResults = main.FALSE
return consistent and correctResults
except Exception:
main.log.exception( "" )
main.cleanup()
main.exit()
+
+ def consistentLeaderboards( self, nodes ):
+ TOPIC = 'org.onosproject.election'
+ # FIXME: use threads
+ #FIXME: should we retry outside the function?
+ for n in range( 5 ): # Retry in case election is still happening
+ leaderList = []
+ # Get all leaderboards
+ for cli in nodes:
+ leaderList.append( cli.specificLeaderCandidate( TOPIC ) )
+ # Compare leaderboards
+ result = all( i == leaderList[0] for i in leaderList ) and\
+ leaderList is not None
+ main.log.debug( leaderList )
+ main.log.warn( result )
+ if result:
+ return ( result, leaderList )
+ time.sleep(5) # TODO: paramerterize
+ main.log.error( "Inconsistent leaderboards:" + str( leaderList ) )
+ return ( result, leaderList )
+
+ def nodesCheck( self, nodes ):
+ nodesOutput = []
+ results = True
+ threads = []
+ for i in nodes:
+ t = main.Thread( target=main.CLIs[i].nodes,
+ name="nodes-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ nodesOutput.append( t.result )
+ ips = [ main.nodes[node].ip_address for node in nodes ]
+ ips.sort()
+ for i in nodesOutput:
+ try:
+ current = json.loads( i )
+ activeIps = []
+ currentResult = False
+ for node in current:
+ if node['state'] == 'READY':
+ activeIps.append( node['ip'] )
+ activeIps.sort()
+ if ips == activeIps:
+ currentResult = True
+ except ( ValueError, TypeError ):
+ main.log.error( "Error parsing nodes output" )
+ main.log.warn( repr( i ) )
+ currentResult = False
+ results = results and currentResult
+ return results