Update HATests
- Clean up some core code( no functional changes)
- add retry to cli functions using the distributed primitives app
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
index d027c36..f057592 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
@@ -16,7 +16,7 @@
#CASE15: Check that Leadership Election is still functional
#1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13
#extra hosts test 1,2,8,11,8,12,8
- <testcases>1,2,8,3,4,5,14,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,15,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index 0f68a02..7194a3b 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -49,6 +49,9 @@
main.log.report( "ONOS Single node cluster restart " +
"HA test - initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -92,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -115,26 +118,27 @@
onpass="Mininet Started",
onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
# values of 1 or 3 are good
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
-
- main.step( "Using mvn clean and install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
- utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if gitPullResult == main.TRUE:
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
# GRAPHS
# NOTE: important params here:
# job = name of Jenkins job
@@ -171,8 +175,6 @@
onos1Isup = main.ONOSbench.isup( ONOS1Ip )
if onos1Isup:
break
- if not onos1Isup:
- main.log.report( "ONOS1 didn't start!" )
utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
onpass="ONOS startup successful",
onfail="ONOS startup failed" )
@@ -191,18 +193,7 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
- appCheck = main.TRUE
- threads = []
- for i in range( numControllers ):
- t = main.Thread( target=CLIs[i].appToIDCheck,
- name="appToIDCheck-" + str( i ),
- args=[] )
- threads.append( t )
- t.start()
-
- for t in threads:
- t.join()
- appCheck = appCheck and t.result
+ appCheck = main.ONOScli1.appToIDCheck()
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
@@ -210,11 +201,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onos1Isup and cliResults )
-
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -234,8 +222,10 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device."
main.step( "Assign switches to controllers" )
for i in range( 1, 29 ):
@@ -255,7 +245,7 @@
else:
mastershipCheck = main.FALSE
if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
+ main.log.info( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -340,13 +330,14 @@
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- # FIXME: we must reinstall intents until we have a persistant
- # datastore!
- main.log.report( "Adding host intents" )
+ # NOTE: we must reinstall intents until we have a persistant intent
+ # datastore!
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
main.step( "Install reactive forwarding app" )
@@ -355,6 +346,7 @@
onpass="Install fwd successful",
onfail="Install fwd failed" )
+ main.step( "Check app ids" )
appCheck = main.ONOScli1.appToIDCheck()
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
@@ -363,6 +355,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -385,6 +379,8 @@
utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
onpass="Uninstall fwd successful",
onfail="Uninstall fwd failed" )
+
+ main.step( "Check app ids" )
appCheck2 = main.ONOScli1.appToIDCheck()
if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
@@ -393,7 +389,7 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -435,7 +431,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -530,13 +529,11 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
correct = True
@@ -667,9 +664,11 @@
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -695,7 +694,7 @@
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
if PingResult == main.TRUE:
- main.log.report(
+ main.log.info(
"Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
@@ -703,83 +702,102 @@
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +\
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -862,6 +880,37 @@
main.log.error( repr( pendingMap ) )
main.log.debug( main.ONOScli1.flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.report(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -873,7 +922,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -961,7 +1009,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -1074,7 +1122,9 @@
iCounterValue = 0
main.log.report( "Restart ONOS node" )
- main.log.case( "Restart ONOS node" )
+ main.case( "Restart ONOS node" )
+ main.caseExplaination = "Killing ONOS process and restart cli " +\
+ "sessions once onos is up."
main.step( "Killing ONOS processes" )
killResult = main.ONOSbench.onosKill( ONOS1Ip )
start = time.time()
@@ -1105,6 +1155,8 @@
main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
str( elapsed ) )
time.sleep( 5 )
+ # rerun on election apps
+ main.ONOScli1.electionTestRun()
def CASE7( self, main ):
"""
@@ -1115,7 +1167,6 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
main.case( "Running ONOS Constant State Tests" )
-
main.step( "Check that each switch has a master" )
# Assert that each device has a master
rolesNotNull = main.ONOScli1.rolesNotNull()
@@ -1162,8 +1213,6 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -1207,9 +1256,9 @@
sameIntents = main.TRUE
if intentState and intentState == ONOS1Intents:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
- # what the aceptable states are
+ # what the acceptable states are
else:
try:
main.log.warn( "ONOS1 intents: " )
@@ -1241,14 +1290,13 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables" )
+ main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leader = ONOS1Ip
@@ -1264,7 +1312,7 @@
# all is well
pass
elif leaderN == main.FALSE:
- # error in response
+ # error in response
main.log.report( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
@@ -1275,25 +1323,12 @@
str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
onpass="Leadership election passed",
onfail="Something went wrong with Leadership election" )
- result = ( mastershipCheck and intentCheck and FlowTables and
- rolesNotNull and leaderResult )
- result = int( result )
- if result == main.TRUE:
- main.log.report( "Constant State Tests Passed" )
- utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Constant State Tests Passed",
- onfail="Constant state tests failed" )
-
def CASE8( self, main ):
"""
Compare topo
@@ -1309,15 +1344,24 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- node = main.ONOS1
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ node = main.ONOS1
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1344,7 +1388,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -1445,8 +1489,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
def CASE9( self, main ):
"""
@@ -1662,10 +1704,17 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- leaderResult = main.TRUE
main.case("Start Leadership Election app")
main.step( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.ONOScli1.electionTestRun()
# check for leader
leader = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
@@ -1674,32 +1723,27 @@
pass
elif leader is None:
# No leader elected
- main.log.report( "No leader was elected" )
+ main.log.error( "No leader was elected" )
leaderResult = main.FALSE
elif leader == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with electionTestLeader" +
+ main.log.error( "Something is wrong with electionTestLeader" +
" function, check the error logs" )
leaderResult = main.FALSE
else:
# error in response
- main.log.report(
+ main.log.error(
"Unexpected response from electionTestLeader function:'" +
str( leader ) +
"'" )
leaderResult = main.FALSE
-
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
def CASE15( self, main ):
"""
@@ -1726,13 +1770,15 @@
oldLeader = None
else:
main.log.error( "Leader election --- why am I HERE?!?")
+ leaderResult = main.FALSE
+ oldLeader = None
if oldLeader:
withdrawResult = oldLeader.electionTestWithdraw()
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
leaderN = main.ONOScli1.electionTestLeader()
@@ -1750,9 +1796,7 @@
elif leaderN is None:
main.log.info(
"There is no leader after the app withdrew from election" )
- if leaderResult:
- main.log.report( "Leadership election tests passed( There is no " +
- "leader after the old leader resigned )" )
+ leaderResult = main.TRUE
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1770,20 +1814,21 @@
actual=runResult,
onpass="App re-ran for election",
onfail="App failed to run for election" )
- leader = main.ONOScli1.electionTestLeader()
+
+ main.step( "Node became leader when it ran for election" )
+ afterRun = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
- if leader == ONOS1Ip:
- leaderResult = main.TRUE
+ if afterRun == ONOS1Ip:
+ afterResult = main.TRUE
else:
- leaderResult = main.FALSE
- # TODO: assert on run and withdraw results?
+ afterResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
- actual=leaderResult,
- onpass="Leadership election passed",
- onfail="ONOS1's election app was not leader after it re-ran " +
- "for election" )
+ actual=afterResult,
+ onpass="Old leader successfully re-ran for election",
+ onfail="Something went wrong with Leadership election after " +
+ "the old leader re-ran for election" )
def CASE16( self, main ):
"""
@@ -1861,11 +1906,13 @@
main.step( "Increment and get a default counter on each node" )
pCounters = []
threads = []
+ addedPValues = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
name="counterIncrement-" + str( i ),
args=[ pCounterName ] )
pCounterValue += 1
+ addedPValues.append( pCounterValue )
threads.append( t )
t.start()
@@ -1874,8 +1921,12 @@
pCounters.append( t.result )
# Check that counter incremented numController times
pCounterResults = True
- for i in range( numControllers ):
- pCounterResults and ( i + 1 ) in pCounters
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=pCounterResults,
onpass="Default counter incremented",
@@ -1884,6 +1935,7 @@
main.step( "Increment and get an in memory counter on each node" )
iCounters = []
+ addedIValues = []
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
@@ -1891,6 +1943,7 @@
args=[ iCounterName ],
kwargs={ "inMemory": True } )
iCounterValue += 1
+ addedIValues.append( iCounterValue )
threads.append( t )
t.start()
@@ -1899,8 +1952,12 @@
iCounters.append( t.result )
# Check that counter incremented numController times
iCounterResults = True
- for i in range( numControllers ):
- iCounterResults and ( i + 1 ) in iCounters
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=iCounterResults,
onpass="In memory counter incremented",