Clean up HATests
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index bb69fcc..50bf06f 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -46,7 +46,7 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
+ main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
main.caseExplaination = "Setup the test environment including " +\
@@ -95,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -126,15 +126,15 @@
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean install" )
cleanInstallResult = main.TRUE
- if gitPullResult == main.TRUE:
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
utilities.assert_equals( expect=main.TRUE,
actual=cleanInstallResult,
onpass="MCI successful",
@@ -145,10 +145,12 @@
# Plot Name = Plot-HA, only can be used if multiple plots
# index = The number of the graph under plot name
job = "HAClusterRestart"
+ plotName = "Plot-HA"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
- '/plot/getPlot?index=0&width=500&height=300"' +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
'noborder="0" width="500" height="300" scrolling="yes" ' +\
'seamless="seamless"></iframe>\n'
graphs += ']]></ac:plain-text-body>\n'
@@ -177,7 +179,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
@@ -204,12 +206,13 @@
onpass="ONOS cli startup successful",
onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
appCheck = main.TRUE
@@ -292,8 +295,6 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -538,7 +539,6 @@
pass # intent submitted is in onos
else:
intentAddResult = False
- # FIXME: DEBUG
if intentAddResult:
intentStop = time.time()
else:
@@ -639,7 +639,15 @@
main.log.debug( "Intents in " + cli.name + ": " +
str( sorted( onosIds ) ) )
if sorted( ids ) != sorted( intentIds ):
+ # IF intents are missing
correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ correct = False
+ break
if correct:
break
else:
@@ -777,7 +785,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -789,9 +797,6 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
@@ -799,31 +804,37 @@
onfail="Intents have not been installed correctly, pings failed." )
main.step( "Check Intent state" )
- installedCheck = True
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ installedCheck = False
count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- # Print states
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
+ while not installedCheck and count < 40:
+ installedCheck = True
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
utilities.assert_equals( expect=True, actual=installedCheck,
onpass="Intents are all INSTALLED",
- onfail="Intents are not all in " +\
+ onfail="Intents are not all in " +
"INSTALLED state" )
main.step( "Check leadership of topics" )
@@ -894,11 +905,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
-
+ main.log.info( "Waiting 60 seconds to see if the state of " +
+ "intents change" )
+ time.sleep( 60 )
if not installedCheck:
- main.log.info( "Waiting 60 seconds to see if the state of " +
- "intents change" )
- time.sleep( 60 )
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
@@ -974,9 +984,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- main.log.debug( CLIs[0].flows( jsonFormat=False ) )
-
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
+ # the wait is above
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -989,7 +1000,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -1068,7 +1079,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1082,7 +1093,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1130,7 +1141,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1143,11 +1154,11 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1231,7 +1242,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1255,7 +1266,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1383,6 +1394,14 @@
for t in threads:
t.join()
ports.append( t.result )
+ try:
+ # FIXME: DEBUG
+ main.log.debug( json.dumps( json.loads( t.result ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except:
+ main.log.debug( repr( t.result ) )
links = []
threads = []
for i in range( numControllers ):
@@ -1418,14 +1437,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1461,12 +1480,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1568,6 +1587,15 @@
iCounterValue = 0
main.case( "Restart entire ONOS cluster" )
+ # FIXME: DEBUG
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ # FIXME: DEBUG
+
main.step( "Killing ONOS nodes" )
killResults = main.TRUE
killTime = time.time()
@@ -1584,7 +1612,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
@@ -1608,25 +1636,25 @@
utilities.assert_equals( expect=main.TRUE, actual=cliResults,
onpass="ONOS cli started",
onfail="ONOS clis did not restart" )
-
+
# Grab the time of restart so we chan check how long the gossip
# protocol has had time to work
main.restartTime = time.time() - killTime
main.log.debug( "Restart time: " + str( main.restartTime ) )
- '''
+
# FIXME: revisit test plan for election with madan
# Rerun for election on restarted nodes
- run1 = CLIs[0].electionTestRun()
- run2 = CLIs[1].electionTestRun()
- run3 = CLIs[2].electionTestRun()
- ...
- ...
- runResults = run1 and run2 and run3
+ runResults = main.TRUE
+ for cli in CLIs:
+ run = CLIs[0].electionTestRun()
+ if run != main.TRUE:
+ main.log.error( "Error running for election on " + cli.name )
+ runResults = runResults and run
utilities.assert_equals( expect=main.TRUE, actual=runResults,
onpass="Reran for election",
onfail="Failed to rerun for election" )
- '''
- # TODO: Make this configurable
+
+ # TODO: Make this configurable
time.sleep( 60 )
main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
@@ -1683,7 +1711,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1697,7 +1725,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1773,7 +1801,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1786,7 +1814,7 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
@@ -1854,22 +1882,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1898,8 +1945,8 @@
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables" )
- main.step( "Check the continuous pings to ensure that no packets " +
- "were dropped during component failure" )
+ # main.step( "Check the continuous pings to ensure that no packets " +
+ # "were dropped during component failure" )
main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
main.params[ 'TESTONIP' ] )
LossInPings = main.FALSE
@@ -1917,12 +1964,13 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
- utilities.assert_equals(
- expect=main.FALSE,
- actual=LossInPings,
- onpass="No Loss of connectivity",
- onfail="Loss of dataplane connectivity detected" )
+ main.log.info( "No loss of dataplane connectivity" )
+ # utilities.assert_equals(
+ # expect=main.FALSE,
+ # actual=LossInPings,
+ # onpass="No Loss of connectivity",
+ # onfail="Loss of dataplane connectivity detected" )
+
# NOTE: Since intents are not persisted with IntnentStore,
# we expect loss in dataplane connectivity
LossInPings = main.FALSE
@@ -1936,14 +1984,13 @@
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.report( cli.name +
- " shows no leader for the election-app was" +
- " elected after the old one died" )
+ main.log.error( cli.name +
+ " shows no leader for the election-app." )
leaderResult = main.FALSE
if len( set( leaderList ) ) != 1:
leaderResult = main.FALSE
@@ -2003,11 +2050,12 @@
main.step( "Collecting topology information from ONOS" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and elapsed < 120:
count += 1
if count > 1:
# TODO: Deprecate STS usage
MNTopo = TestONTopology( main.Mininet1, ctrls )
+ time.sleep( 1 )
cliStart = time.time()
devices = []
threads = []
@@ -2058,6 +2106,14 @@
for t in threads:
t.join()
ports.append( t.result )
+ try:
+ # FIXME: DEBUG
+ main.log.debug( json.dumps( json.loads( t.result ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except:
+ main.log.debug( repr( t.result ) )
links = []
threads = []
for i in range( numControllers ):
@@ -2085,6 +2141,7 @@
elapsed = time.time() - startTime
cliTime = time.time() - cliStart
+ print "Elapsed time: " + str( elapsed )
print "CLI time: " + str( cliTime )
for controller in range( numControllers ):
@@ -2157,13 +2214,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2184,13 +2241,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2291,7 +2348,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2320,7 +2376,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2348,14 +2403,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2394,11 +2448,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2447,9 +2499,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2505,7 +2555,7 @@
for node in nodes:
print colors[ 'purple' ] + "Checking logs for errors on " + \
node.name + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( node.ip_address )
+ print main.ONOSbench.checkLogs( node.ip_address, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
@@ -2513,8 +2563,9 @@
try:
timerLog = open( main.logdir + "/Timers.csv", 'w')
# Overwrite with empty line and close
- timerLog.write( "Restart\n" )
- timerLog.write( str( main.restartTime ) )
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
timerLog.close()
except NameError, e:
main.log.exception(e)
@@ -2546,7 +2597,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2582,7 +2633,6 @@
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
# NOTE: Need to re-run since being a canidate is not persistant
main.step( "Run for election on each node" )
@@ -2593,7 +2643,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2647,14 +2697,14 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
@@ -2671,10 +2721,10 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
utilities.assert_equals(
@@ -2714,6 +2764,7 @@
"""
Install Distributed Primitives app
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -2743,7 +2794,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
- time.sleep (5 ) # To allow all nodes to activate
+ time.sleep( 5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2803,7 +2854,7 @@
# Check that counter incremented numController times
pCounterResults = True
for i in addedPValues:
- tmpResult = i in pCounters
+ tmpResult = i in pCounters
pCounterResults = pCounterResults and tmpResult
if not tmpResult:
main.log.error( str( i ) + " is not in partitioned "
@@ -2873,28 +2924,41 @@
main.step( "Counters we added have the correct values" )
correctResults = main.TRUE
for i in range( numControllers ):
- current = onosCounters[i]
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
try:
- pValue = current.get( pCounterName )
- iValue = current.get( iCounterName )
- if pValue == pCounterValue:
- main.log.info( "Partitioned counter value is correct" )
- else:
- main.log.error( "Partitioned counter value is incorrect," +
- " expected value: " + str( pCounterValue )
- + " current value: " + str( pValue ) )
- correctResults = main.FALSE
- if iValue == iCounterValue:
- main.log.info( "In memory counter value is correct" )
- else:
- main.log.error( "In memory counter value is incorrect, " +
- "expected value: " + str( iCounterValue ) +
- " current value: " + str( iValue ) )
- correctResults = main.FALSE
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
except AttributeError, e:
main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
"is not as expected" )
correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=correctResults,
onpass="Added counters are correct",