Leadership Election HA Tests
- minor bug fixes
diff --git a/TestON/dependencies/rotate.sh b/TestON/dependencies/rotate.sh
index c685a14..0f879b6 100755
--- a/TestON/dependencies/rotate.sh
+++ b/TestON/dependencies/rotate.sh
@@ -51,4 +51,4 @@
#Maybe this should be an argument? pack-and-rotate supports that
nr_max=20
-pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap ${root_dir}/${base_name}*.log " ${nr_max}
+pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap ${root_dir}/${base_name}*.log" ${nr_max}
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 358945f..6d999e4 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1853,17 +1853,21 @@
node_search = re.search("The\scurrent\sleader\sfor\sthe\sElection\sapp\sis\s(?P<node>.+)\.", response)
if node_search:
node = node_search.group('node')
- main.log.info("Election-test-leader found " + node + " as the leader")
+ main.log.info("Election-test-leader on "+str(self.name)+" found " + node + " as the leader")
return node
#no leader
null_search = re.search("There\sis\scurrently\sno\sleader\selected\sfor\sthe\sElection\sapp", response)
if null_search:
- main.log.info("Election-test-leader found no leader")
+ main.log.info("Election-test-leader found no leader on " + self.name )
return None
#error
- main.log.error("Error in election_test_leader: unexpected response")
- main.log.error( repr(response) )
- return main.FALSE
+ if re.search("Command\snot\sfound", response):
+ main.log.error("Election app is not loaded on " + self.name)
+ return main.FALSE
+ else:
+ main.log.error("Error in election_test_leader: unexpected response")
+ main.log.error( repr(response) )
+ return main.FALSE
except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
@@ -1892,12 +1896,16 @@
#success
search = re.search("Entering\sleadership\selections\sfor\sthe\sElection\sapp.", response)
if search:
- main.log.info("Entering leadership elections for the Election app.")
+ main.log.info(self.name + " entering leadership elections for the Election app.")
return main.TRUE
#error
- main.log.error("Error in election_test_run: unexpected response")
- main.log.error( repr(response) )
- return main.FALSE
+ if re.search("Command\snot\sfound", response):
+ main.log.error("Election app is not loaded on " + self.name)
+ return main.FALSE
+ else:
+ main.log.error("Error in election_test_run: unexpected response")
+ main.log.error( repr(response) )
+ return main.FALSE
except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
@@ -1927,12 +1935,16 @@
#success
search = re.search("Withdrawing\sfrom\sleadership\selections\sfor\sthe\sElection\sapp.", response)
if search:
- main.log.info("Withdrawing from leadership elections for the Election app.")
+ main.log.info(self.name + " withdrawing from leadership elections for the Election app.")
return main.TRUE
#error
- main.log.error("Error in election_test_withdraw: unexpected response")
- main.log.error( repr(response) )
- return main.FALSE
+ if re.search("Command\snot\sfound", response):
+ main.log.error("Election app is not loaded on " + self.name)
+ return main.FALSE
+ else:
+ main.log.error("Error in election_test_withdraw: unexpected response")
+ main.log.error( repr(response) )
+ return main.FALSE
except pexpect.EOF:
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
index 68f12a8..38a9b02 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -1,9 +1,10 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],8,3,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
<Git>False</Git>
+ <num_controllers> 7 </num_controllers>
<CTRL>
<ip1>10.128.30.11</ip1>
@@ -52,8 +53,8 @@
<target10>10.0.0.27</target10>
</PING>
<timers>
- <LinkDiscovery>2</LinkDiscovery>
- <SwitchDiscovery>2</SwitchDiscovery>
+ <LinkDiscovery>.2</LinkDiscovery>
+ <SwitchDiscovery>.2</SwitchDiscovery>
</timers>
<MNtcpdump>
<intf>eth0</intf>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 3e0a055..b9a07b9 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -16,6 +16,8 @@
CASE11: Switch down
CASE12: Switch up
CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
'''
class HATestClusterRestart:
@@ -61,6 +63,7 @@
global ONOS6_port
global ONOS7_ip
global ONOS7_port
+ global num_controllers
ONOS1_ip = main.params['CTRL']['ip1']
ONOS1_port = main.params['CTRL']['port1']
@@ -76,6 +79,7 @@
ONOS6_port = main.params['CTRL']['port6']
ONOS7_ip = main.params['CTRL']['ip7']
ONOS7_port = main.params['CTRL']['port7']
+ num_controllers = int(main.params['num_controllers'])
main.step("Applying cell variable to environment")
@@ -143,8 +147,7 @@
main.step("Checking if ONOS is up yet")
- #TODO: Refactor
- # check bundle:list?
+ #TODO check bundle:list?
for i in range(2):
onos1_isup = main.ONOSbench.isup(ONOS1_ip)
if not onos1_isup:
@@ -217,7 +220,7 @@
main.step("Assign switches to controllers")
for i in range (1,29):
- main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+ main.Mininet1.assign_sw_controller(sw=str(i),count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -467,7 +470,7 @@
Assign intents
"""
- #FIXME: This should only be called once when we have persistence
+ #FIXME: we must reinstall intents until we have a persistant datastore!
import time
import json
import re
@@ -503,6 +506,8 @@
main.ONOScli5.feature_uninstall("onos-app-fwd")
main.ONOScli6.feature_uninstall("onos-app-fwd")
main.ONOScli7.feature_uninstall("onos-app-fwd")
+ #timeout for fwd flows
+ time.sleep(10)
main.step("Add host intents")
#TODO: move the host numbers to params
@@ -513,11 +518,14 @@
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
- #NOTE: get host can return None
- #TODO: handle this
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
- tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ #NOTE: get host can return None
+ if host1_id and host2_id:
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ else:
+ main.log.error("Error, get_host() failed")
+ tmp_result = main.FALSE
intent_add_result = bool(intent_add_result and tmp_result)
utilities.assert_equals(expect=True, actual=intent_add_result,
onpass="Switch mastership correctly assigned",
@@ -921,7 +929,7 @@
devices_results = main.TRUE
ports_results = main.TRUE
links_results = main.TRUE
- for controller in range(7): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -1115,7 +1123,7 @@
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed")
#NOTE: we expect mastership to change on controller failure
- mastership_check = mastership_check #and consistent_mastership
+ mastership_check = consistent_mastership
@@ -1152,7 +1160,7 @@
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
else:
- main.log.warn("ONOS1 intents: ")
+ main.log.warn("ONOS1 intents: ")
print json.dumps(json.loads(ONOS1_intents),
sort_keys=True, indent=4, separators=(',', ': '))
main.log.warn("ONOS2 intents: ")
@@ -1177,7 +1185,7 @@
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
- #NOTE: Hazelcast has no durability, so intents are lost
+ #NOTE: Hazelcast has no durability, so intents are lost across system restarts
'''
main.step("Compare current intents with intents before the failure")
#NOTE: this requires case 5 to pass for intent_state to be set.
@@ -1187,6 +1195,12 @@
main.log.report("Intents are consistent with before failure")
#TODO: possibly the states have changed? we may need to figure out what the aceptable states are
else:
+ try:
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ except:
+ pass
same_intents = main.FALSE
utilities.assert_equals(expect = main.TRUE,actual=same_intents,
onpass="Intents are consistent with before failure",
@@ -1237,8 +1251,33 @@
Loss_In_Pings = main.FALSE
- #TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null
+ #Test of LeadershipElection
+ leader_list = []
+ leader_result = main.TRUE
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leaderN = node.election_test_leader()
+ leader_list.append(leaderN)
+ if leaderN == main.FALSE:
+ #error in response
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leaderN == None:
+ main.log.report("ONOS"+str(controller) + " shows no leader for the election-app was elected after the old one died")
+ leader_result = main.FALSE
+ if len( set( leader_list ) ) != 1:
+ leader_result = main.FALSE
+ main.log.error("Inconsistent view of leader for the election test app")
+ #TODO: print the list
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected after restart)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null\
+ and leader_result
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -1350,7 +1389,7 @@
cli_time = time.time() - cli_start
print "CLI time: " + str(cli_time)
- for controller in range(7): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -1461,7 +1500,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Turn off a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1483,7 +1522,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Restore a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1506,7 +1545,7 @@
#NOTE: You should probably run a topology check after this
import time
- switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Killing a switch to ensure it is discovered correctly"
main.log.report(description)
@@ -1534,6 +1573,8 @@
'''
#NOTE: You should probably run a topology check after this
import time
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1545,7 +1586,7 @@
main.Mininet1.add_link('s28', 's3')
main.Mininet1.add_link('s28', 's6')
main.Mininet1.add_link('s28', 'h28')
- main.Mininet1.assign_sw_controller(sw="28",count=7,
+ main.Mininet1.assign_sw_controller(sw="28",count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -1571,6 +1612,15 @@
'''
import os
import time
+ #printing colors to terminal
+ colors = {}
+ colors['cyan'] = '\033[96m'
+ colors['purple'] = '\033[95m'
+ colors['blue'] = '\033[94m'
+ colors['green'] = '\033[92m'
+ colors['yellow'] = '\033[93m'
+ colors['red'] = '\033[91m'
+ colors['end'] = '\033[0m'
description = "Test Cleanup"
main.log.report(description)
main.case(description)
@@ -1578,19 +1628,19 @@
main.Mininet2.stop_tcpdump()
main.step("Checking ONOS Logs for errors")
- print "Checking logs for errors on ONOS1:"
+ print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
print main.ONOSbench.check_logs(ONOS1_ip)
- print "Checking logs for errors on ONOS2:"
+ print colors['purple'] + "Checking logs for errors on ONOS2:" + colors['end']
print main.ONOSbench.check_logs(ONOS2_ip)
- print "Checking logs for errors on ONOS3:"
+ print colors['purple'] + "Checking logs for errors on ONOS3:" + colors['end']
print main.ONOSbench.check_logs(ONOS3_ip)
- print "Checking logs for errors on ONOS4:"
+ print colors['purple'] + "Checking logs for errors on ONOS4:" + colors['end']
print main.ONOSbench.check_logs(ONOS4_ip)
- print "Checking logs for errors on ONOS5:"
+ print colors['purple'] + "Checking logs for errors on ONOS5:" + colors['end']
print main.ONOSbench.check_logs(ONOS5_ip)
- print "Checking logs for errors on ONOS6:"
+ print colors['purple'] + "Checking logs for errors on ONOS6:" + colors['end']
print main.ONOSbench.check_logs(ONOS6_ip)
- print "Checking logs for errors on ONOS7:"
+ print colors['purple'] + "Checking logs for errors on ONOS7:" + colors['end']
print main.ONOSbench.check_logs(ONOS7_ip)
main.step("Copying MN pcap and ONOS log files to test station")
@@ -1670,3 +1720,139 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ leader_result = main.TRUE
+ #install app on onos 1
+ main.log.info("Install leadership election app")
+ main.ONOScli1.feature_install("onos-app-election")
+ #wait for election
+ #check for leader
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ #all is well
+ pass
+ elif leader == None:
+ #No leader elected
+ main.log.report("No leader was elected")
+ leader_result = main.FALSE
+ elif leader == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ else:
+ #error in response
+ main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+ leader_result = main.FALSE
+
+
+
+
+ #install on other nodes and check for leader.
+ #Should be onos1 and each app should show the same leader
+ for controller in range(2,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ node.feature_install("onos-app-election")
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == ONOS1_ip:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+ def CASE15 ( self, main ) :
+ '''
+ Check that Leadership Election is still functional
+ '''
+ leader_result = main.TRUE
+ description = "Check that Leadership Election is still functional"
+ main.log.report(description)
+ main.case(description)
+ main.step("Find current leader and withdraw")
+ leader = main.ONOScli1.election_test_leader()
+ #TODO: do some sanity checking on leader before using it
+ withdraw_result = main.FALSE
+ if leader == ONOS1_ip:
+ old_leader = getattr( main, "ONOScli1" )
+ elif leader == ONOS2_ip:
+ old_leader = getattr( main, "ONOScli2" )
+ elif leader == ONOS3_ip:
+ old_leader = getattr( main, "ONOScli3" )
+ elif leader == ONOS4_ip:
+ old_leader = getattr( main, "ONOScli4" )
+ elif leader == ONOS5_ip:
+ old_leader = getattr( main, "ONOScli5" )
+ elif leader == ONOS6_ip:
+ old_leader = getattr( main, "ONOScli6" )
+ elif leader == ONOS7_ip:
+ old_leader = getattr( main, "ONOScli7" )
+ elif leader == None or leader == main.FALSE:
+ main.log.report("Leader for the election app should be an ONOS node,"\
+ +"instead got '"+str(leader)+"'")
+ leader_result = main.FALSE
+ withdraw_result = old_leader.election_test_withdraw()
+
+
+ main.step("Make sure new leader is elected")
+ leader_list = []
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leader_list.append( node.election_test_leader() )
+ for leaderN in leader_list:
+ if leaderN == leader:
+ main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+ " as leader after they withdrew")
+ leader_result = main.FALSE
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ consistent_leader = main.FALSE
+ if len( set( leader_list ) ) == 1:
+ main.log.info("Each Election-app sees '"+str(leader_list[0])+"' as the leader")
+ consistent_leader = main.TRUE
+ else:
+ main.log.report("Inconsistent responses for leader of Election-app:")
+ for n in range(len(leader_list)):
+ main.log.report("ONOS" + str(n+1) + " response: " + str(leader_list[n]) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was elected when the old leader resigned)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ main.step("Run for election on old leader(just so everyone is in the hat)")
+ run_result = old_leader.election_test_run()
+ if consistent_leader == main.TRUE:
+ after_run = main.ONOScli1.election_test_leader()
+ #verify leader didn't just change
+ if after_run == leader_list[0]:
+ leader_result = main.TRUE
+ else:
+ leader_result = main.FALSE
+ #TODO: assert on run and withdraw results?
+
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election after the old leader re-ran for election")
+
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
index 5b97b66..78e0c83 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -1,9 +1,10 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],8,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
<Git>False</Git>
+ <num_controllers> 7 </num_controllers>
<CTRL>
<ip1>10.128.30.11</ip1>
@@ -52,8 +53,8 @@
<target10>10.0.0.27</target10>
</PING>
<timers>
- <LinkDiscovery>2</LinkDiscovery>
- <SwitchDiscovery>2</SwitchDiscovery>
+ <LinkDiscovery>.2</LinkDiscovery>
+ <SwitchDiscovery>.2</SwitchDiscovery>
</timers>
<MNtcpdump>
<intf>eth0</intf>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 3fbc29d..b9d2925 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -16,6 +16,8 @@
CASE11: Switch down
CASE12: Switch up
CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
'''
class HATestMinorityRestart:
@@ -61,6 +63,7 @@
global ONOS6_port
global ONOS7_ip
global ONOS7_port
+ global num_controllers
ONOS1_ip = main.params['CTRL']['ip1']
ONOS1_port = main.params['CTRL']['port1']
@@ -76,6 +79,7 @@
ONOS6_port = main.params['CTRL']['port6']
ONOS7_ip = main.params['CTRL']['ip7']
ONOS7_port = main.params['CTRL']['port7']
+ num_controllers = int(main.params['num_controllers'])
main.step("Applying cell variable to environment")
@@ -143,8 +147,7 @@
main.step("Checking if ONOS is up yet")
- #TODO: Refactor
- # check bundle:list?
+ #TODO check bundle:list?
for i in range(2):
onos1_isup = main.ONOSbench.isup(ONOS1_ip)
if not onos1_isup:
@@ -184,6 +187,7 @@
cli_results = cli_result1 and cli_result2 and cli_result3 and\
cli_result4 and cli_result5 and cli_result6 and cli_result7
+
main.step("Start Packet Capture MN")
main.Mininet2.start_tcpdump(
str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
@@ -217,7 +221,7 @@
main.step("Assign switches to controllers")
for i in range (1,29):
- main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+ main.Mininet1.assign_sw_controller(sw=str(i),count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -502,6 +506,8 @@
main.ONOScli5.feature_uninstall("onos-app-fwd")
main.ONOScli6.feature_uninstall("onos-app-fwd")
main.ONOScli7.feature_uninstall("onos-app-fwd")
+ #timeout for fwd flows
+ time.sleep(10)
main.step("Add host intents")
#TODO: move the host numbers to params
@@ -512,11 +518,14 @@
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
- #NOTE: get host can return None
- #TODO: handle this
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
- tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ #NOTE: get host can return None
+ if host1_id and host2_id:
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ else:
+ main.log.error("Error, get_host() failed")
+ tmp_result = main.FALSE
intent_add_result = bool(intent_add_result and tmp_result)
utilities.assert_equals(expect=True, actual=intent_add_result,
onpass="Switch mastership correctly assigned",
@@ -806,13 +815,17 @@
devices.append( main.ONOScli6.devices() )
devices.append( main.ONOScli7.devices() )
hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
+ hosts.append( json.loads( main.ONOScli1.hosts() ) )
+ hosts.append( json.loads( main.ONOScli2.hosts() ) )
+ hosts.append( json.loads( main.ONOScli3.hosts() ) )
+ hosts.append( json.loads( main.ONOScli4.hosts() ) )
+ hosts.append( json.loads( main.ONOScli5.hosts() ) )
+ hosts.append( json.loads( main.ONOScli6.hosts() ) )
+ hosts.append( json.loads( main.ONOScli7.hosts() ) )
+ for controller in range(0, len(hosts) ):
+ for host in hosts[controller]:
+ if host['ips'] == []:
+ main.log.error("DEBUG:Error with host ips on controller"+str(controller+1)+": " + str(host))
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -920,7 +933,7 @@
devices_results = main.TRUE
ports_results = main.TRUE
links_results = main.TRUE
- for controller in range(7): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -995,6 +1008,8 @@
cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
cli_results = cli_result1 and cli_result2 and cli_result3
+ main.log.info("Install leadership election app on restarted node")
+
case_results = main.TRUE and onos_isup_result and cli_results
utilities.assert_equals(expect=main.TRUE, actual=case_results,
onpass="ONOS restart successful",
@@ -1103,7 +1118,7 @@
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed")
#NOTE: we expect mastership to change on controller failure
- mastership_check = mastership_check #and consistent_mastership
+ mastership_check = consistent_mastership
@@ -1140,7 +1155,7 @@
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
else:
- main.log.warn("ONOS1 intents: ")
+ main.log.warn("ONOS1 intents: ")
print json.dumps(json.loads(ONOS1_intents),
sort_keys=True, indent=4, separators=(',', ': '))
main.log.warn("ONOS2 intents: ")
@@ -1165,7 +1180,7 @@
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
- #NOTE: Hazelcast has no durability, so intents are lost
+ #NOTE: Hazelcast has no durability, so intents are lost across system restarts
main.step("Compare current intents with intents before the failure")
#NOTE: this requires case 5 to pass for intent_state to be set.
# maybe we should stop the test if that fails?
@@ -1174,6 +1189,12 @@
main.log.report("Intents are consistent with before failure")
#TODO: possibly the states have changed? we may need to figure out what the aceptable states are
else:
+ try:
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ except:
+ pass
same_intents = main.FALSE
utilities.assert_equals(expect = main.TRUE,actual=same_intents,
onpass="Intents are consistent with before failure",
@@ -1221,8 +1242,36 @@
onfail="Loss of dataplane connectivity detected")
- #TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null
+ #Test of LeadershipElection
+ leader_list = []
+ leader_result = main.TRUE
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leaderN = node.election_test_leader()
+ leader_list.append(leaderN)
+ if leaderN == main.FALSE:
+ #error in response
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leaderN == None:
+ main.log.report("ONOS"+str(controller) + " shows no leader for the election-app was elected after the old one died")
+ leader_result = main.FALSE
+ elif leaderN == ONOS1_ip or leaderN == ONOS2_ip or leaderN == ONOS3_ip:
+ main.log.report("ONOS"+str(controller) + " shows "+str(leaderN)+" as leader for the election-app, but it was restarted")
+ leader_result = main.FALSE
+ if len( set( leader_list ) ) != 1:
+ leader_result = main.FALSE
+ main.log.error("Inconsistent view of leader for the election test app")
+ #TODO: print the list
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null\
+ and leader_result
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -1282,13 +1331,18 @@
devices.append( main.ONOScli6.devices() )
devices.append( main.ONOScli7.devices() )
hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
+ hosts.append( json.loads( main.ONOScli1.hosts() ) )
+ hosts.append( json.loads( main.ONOScli2.hosts() ) )
+ hosts.append( json.loads( main.ONOScli3.hosts() ) )
+ hosts.append( json.loads( main.ONOScli4.hosts() ) )
+ hosts.append( json.loads( main.ONOScli5.hosts() ) )
+ hosts.append( json.loads( main.ONOScli6.hosts() ) )
+ hosts.append( json.loads( main.ONOScli7.hosts() ) )
+ for controller in range(0, len(hosts) ):
+ for host in hosts[controller]:
+ host
+ if host['ips'] == []:
+ main.log.error("DEBUG:Error with host ips on controller"+str(controller+1)+": " + str(host))
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -1334,7 +1388,7 @@
cli_time = time.time() - cli_start
print "CLI time: " + str(cli_time)
- for controller in range(7): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -1447,7 +1501,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Turn off a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1469,7 +1523,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Restore a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1492,7 +1546,7 @@
#NOTE: You should probably run a topology check after this
import time
- switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Killing a switch to ensure it is discovered correctly"
main.log.report(description)
@@ -1520,6 +1574,8 @@
'''
#NOTE: You should probably run a topology check after this
import time
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1531,7 +1587,7 @@
main.Mininet1.add_link('s28', 's3')
main.Mininet1.add_link('s28', 's6')
main.Mininet1.add_link('s28', 'h28')
- main.Mininet1.assign_sw_controller(sw="28",count=7,
+ main.Mininet1.assign_sw_controller(sw="28",count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -1557,6 +1613,15 @@
'''
import os
import time
+ #printing colors to terminal
+ colors = {}
+ colors['cyan'] = '\033[96m'
+ colors['purple'] = '\033[95m'
+ colors['blue'] = '\033[94m'
+ colors['green'] = '\033[92m'
+ colors['yellow'] = '\033[93m'
+ colors['red'] = '\033[91m'
+ colors['end'] = '\033[0m'
description = "Test Cleanup"
main.log.report(description)
main.case(description)
@@ -1564,19 +1629,19 @@
main.Mininet2.stop_tcpdump()
main.step("Checking ONOS Logs for errors")
- print "Checking logs for errors on ONOS1:"
+ print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
print main.ONOSbench.check_logs(ONOS1_ip)
- print "Checking logs for errors on ONOS2:"
+ print colors['purple'] + "Checking logs for errors on ONOS2:" + colors['end']
print main.ONOSbench.check_logs(ONOS2_ip)
- print "Checking logs for errors on ONOS3:"
+ print colors['purple'] + "Checking logs for errors on ONOS3:" + colors['end']
print main.ONOSbench.check_logs(ONOS3_ip)
- print "Checking logs for errors on ONOS4:"
+ print colors['purple'] + "Checking logs for errors on ONOS4:" + colors['end']
print main.ONOSbench.check_logs(ONOS4_ip)
- print "Checking logs for errors on ONOS5:"
+ print colors['purple'] + "Checking logs for errors on ONOS5:" + colors['end']
print main.ONOSbench.check_logs(ONOS5_ip)
- print "Checking logs for errors on ONOS6:"
+ print colors['purple'] + "Checking logs for errors on ONOS6:" + colors['end']
print main.ONOSbench.check_logs(ONOS6_ip)
- print "Checking logs for errors on ONOS7:"
+ print colors['purple'] + "Checking logs for errors on ONOS7:" + colors['end']
print main.ONOSbench.check_logs(ONOS7_ip)
main.step("Copying MN pcap and ONOS log files to test station")
@@ -1656,3 +1721,139 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ leader_result = main.TRUE
+ #install app on onos 1
+ main.log.info("Install leadership election app")
+ main.ONOScli1.feature_install("onos-app-election")
+ #wait for election
+ #check for leader
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ #all is well
+ pass
+ elif leader == None:
+ #No leader elected
+ main.log.report("No leader was elected")
+ leader_result = main.FALSE
+ elif leader == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ else:
+ #error in response
+ main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+ leader_result = main.FALSE
+
+
+
+
+ #install on other nodes and check for leader.
+ #Should be onos1 and each app should show the same leader
+ for controller in range(2,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ node.feature_install("onos-app-election")
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == ONOS1_ip:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+ def CASE15 ( self, main ) :
+ '''
+ Check that Leadership Election is still functional
+ '''
+ leader_result = main.TRUE
+ description = "Check that Leadership Election is still functional"
+ main.log.report(description)
+ main.case(description)
+ main.step("Find current leader and withdraw")
+ leader = main.ONOScli1.election_test_leader()
+ #TODO: do some sanity checking on leader before using it
+ withdraw_result = main.FALSE
+ if leader == ONOS1_ip:
+ old_leader = getattr( main, "ONOScli1" )
+ elif leader == ONOS2_ip:
+ old_leader = getattr( main, "ONOScli2" )
+ elif leader == ONOS3_ip:
+ old_leader = getattr( main, "ONOScli3" )
+ elif leader == ONOS4_ip:
+ old_leader = getattr( main, "ONOScli4" )
+ elif leader == ONOS5_ip:
+ old_leader = getattr( main, "ONOScli5" )
+ elif leader == ONOS6_ip:
+ old_leader = getattr( main, "ONOScli6" )
+ elif leader == ONOS7_ip:
+ old_leader = getattr( main, "ONOScli7" )
+ elif leader == None or leader == main.FALSE:
+ main.log.report("Leader for the election app should be an ONOS node,"\
+ +"instead got '"+str(leader)+"'")
+ leader_result = main.FALSE
+ withdraw_result = old_leader.election_test_withdraw()
+
+
+ main.step("Make sure new leader is elected")
+ leader_list = []
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leader_list.append( node.election_test_leader() )
+ for leaderN in leader_list:
+ if leaderN == leader:
+ main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+ " as leader after they withdrew")
+ leader_result = main.FALSE
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ consistent_leader = main.FALSE
+ if len( set( leader_list ) ) == 1:
+ main.log.info("Each Election-app sees '"+str(leader_list[0])+"' as the leader")
+ consistent_leader = main.TRUE
+ else:
+ main.log.report("Inconsistent responses for leader of Election-app:")
+ for n in range(len(leader_list)):
+ main.log.report("ONOS" + str(n+1) + " response: " + str(leader_list[n]) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was elected when the old leader resigned)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ main.step("Run for election on old leader(just so everyone is in the hat)")
+ run_result = old_leader.election_test_run()
+ if consistent_leader == main.TRUE:
+ after_run = main.ONOScli1.election_test_leader()
+ #verify leader didn't just change
+ if after_run == leader_list[0]:
+ leader_result = main.TRUE
+ else:
+ leader_result = main.FALSE
+ #TODO: assert on run and withdraw results?
+
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election after the old leader re-ran for election")
+
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index 5259a47..78e0c83 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -1,9 +1,10 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],8,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
- <Git>True</Git>
+ <Git>False</Git>
+ <num_controllers> 7 </num_controllers>
<CTRL>
<ip1>10.128.30.11</ip1>
@@ -52,8 +53,8 @@
<target10>10.0.0.27</target10>
</PING>
<timers>
- <LinkDiscovery>2</LinkDiscovery>
- <SwitchDiscovery>2</SwitchDiscovery>
+ <LinkDiscovery>.2</LinkDiscovery>
+ <SwitchDiscovery>.2</SwitchDiscovery>
</timers>
<MNtcpdump>
<intf>eth0</intf>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 4e5b463..7399925 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -17,6 +17,8 @@
CASE11: Switch down
CASE12: Switch up
CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
'''
class HATestSanity:
@@ -63,6 +65,7 @@
global ONOS6_port
global ONOS7_ip
global ONOS7_port
+ global num_controllers
ONOS1_ip = main.params['CTRL']['ip1']
ONOS1_port = main.params['CTRL']['port1']
@@ -78,6 +81,7 @@
ONOS6_port = main.params['CTRL']['port6']
ONOS7_ip = main.params['CTRL']['ip7']
ONOS7_port = main.params['CTRL']['port7']
+ num_controllers = int(main.params['num_controllers'])
main.step("Applying cell variable to environment")
@@ -145,8 +149,7 @@
main.step("Checking if ONOS is up yet")
- #TODO: Refactor
- # check bundle:list?
+ #TODO check bundle:list?
for i in range(2):
onos1_isup = main.ONOSbench.isup(ONOS1_ip)
if not onos1_isup:
@@ -173,7 +176,6 @@
and onos4_isup and onos5_isup and onos6_isup and onos7_isup
if onos_isup_result == main.TRUE:
break
- # TODO: if it becomes an issue, we can retry this step a few times
cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
@@ -219,7 +221,7 @@
main.step("Assign switches to controllers")
for i in range (1,29):
- main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+ main.Mininet1.assign_sw_controller(sw=str(i),count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -504,6 +506,8 @@
main.ONOScli5.feature_uninstall("onos-app-fwd")
main.ONOScli6.feature_uninstall("onos-app-fwd")
main.ONOScli7.feature_uninstall("onos-app-fwd")
+ #timeout for fwd flows
+ time.sleep(10)
main.step("Add host intents")
#TODO: move the host numbers to params
@@ -514,11 +518,14 @@
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
- #NOTE: get host can return None
- #TODO: handle this
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
- tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ #NOTE: get host can return None
+ if host1_id and host2_id:
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ else:
+ main.log.error("Error, get_host() failed")
+ tmp_result = main.FALSE
intent_add_result = bool(intent_add_result and tmp_result)
utilities.assert_equals(expect=True, actual=intent_add_result,
onpass="Switch mastership correctly assigned",
@@ -922,7 +929,7 @@
devices_results = main.TRUE
ports_results = main.TRUE
links_results = main.TRUE
- for controller in range(7): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -1195,9 +1202,35 @@
onpass="No Loss of connectivity",
onfail="Loss of dataplane connectivity detected")
+ #Test of LeadershipElection
+ #NOTE: this only works for the sanity test. In case of failures, leader will likely change
+ leader = ONOS1_ip
+ leader_result = main.TRUE
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == leader:
+ #all is well
+ #NOTE: In failure scenario, this could be a new node, maybe check != ONOS1
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
- #TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null
+
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null\
+ and leader_result
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -1309,7 +1342,7 @@
cli_time = time.time() - cli_start
print "CLI time: " + str(cli_time)
- for controller in range(7): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -1420,7 +1453,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Turn off a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1442,7 +1475,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Restore a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1465,7 +1498,7 @@
#NOTE: You should probably run a topology check after this
import time
- switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Killing a switch to ensure it is discovered correctly"
main.log.report(description)
@@ -1493,6 +1526,8 @@
'''
#NOTE: You should probably run a topology check after this
import time
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1504,7 +1539,7 @@
main.Mininet1.add_link('s28', 's3')
main.Mininet1.add_link('s28', 's6')
main.Mininet1.add_link('s28', 'h28')
- main.Mininet1.assign_sw_controller(sw="28",count=7,
+ main.Mininet1.assign_sw_controller(sw="28",count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -1530,6 +1565,15 @@
'''
import os
import time
+ #printing colors to terminal
+ colors = {}
+ colors['cyan'] = '\033[96m'
+ colors['purple'] = '\033[95m'
+ colors['blue'] = '\033[94m'
+ colors['green'] = '\033[92m'
+ colors['yellow'] = '\033[93m'
+ colors['red'] = '\033[91m'
+ colors['end'] = '\033[0m'
description = "Test Cleanup"
main.log.report(description)
main.case(description)
@@ -1537,19 +1581,19 @@
main.Mininet2.stop_tcpdump()
main.step("Checking ONOS Logs for errors")
- print "Checking logs for errors on ONOS1:"
+ print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
print main.ONOSbench.check_logs(ONOS1_ip)
- print "Checking logs for errors on ONOS2:"
+ print colors['purple'] + "Checking logs for errors on ONOS2:" + colors['end']
print main.ONOSbench.check_logs(ONOS2_ip)
- print "Checking logs for errors on ONOS3:"
+ print colors['purple'] + "Checking logs for errors on ONOS3:" + colors['end']
print main.ONOSbench.check_logs(ONOS3_ip)
- print "Checking logs for errors on ONOS4:"
+ print colors['purple'] + "Checking logs for errors on ONOS4:" + colors['end']
print main.ONOSbench.check_logs(ONOS4_ip)
- print "Checking logs for errors on ONOS5:"
+ print colors['purple'] + "Checking logs for errors on ONOS5:" + colors['end']
print main.ONOSbench.check_logs(ONOS5_ip)
- print "Checking logs for errors on ONOS6:"
+ print colors['purple'] + "Checking logs for errors on ONOS6:" + colors['end']
print main.ONOSbench.check_logs(ONOS6_ip)
- print "Checking logs for errors on ONOS7:"
+ print colors['purple'] + "Checking logs for errors on ONOS7:" + colors['end']
print main.ONOSbench.check_logs(ONOS7_ip)
main.step("Copying MN pcap and ONOS log files to test station")
@@ -1629,22 +1673,139 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+
def CASE14 ( self, main ) :
'''
start election app on all onos nodes
'''
+ leader_result = main.TRUE
#install app on onos 1
+ main.log.info("Install leadership election app")
+ main.ONOScli1.feature_install("onos-app-election")
#wait for election
#check for leader
- #install on other nodes
- #check for leader. Should be onos1 and each app shows same leader
- #
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ #all is well
+ pass
+ elif leader == None:
+ #No leader elected
+ main.log.report("No leader was elected")
+ leader_result = main.FALSE
+ elif leader == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ else:
+ #error in response
+ main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+ leader_result = main.FALSE
+ #install on other nodes and check for leader.
+ #Should be onos1 and each app should show the same leader
+ for controller in range(2,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ node.feature_install("onos-app-election")
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == ONOS1_ip:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
- #Next Case
- #add to reboot case?
- #check for conistent leader
- #new leader should have been elected
+ def CASE15 ( self, main ) :
+ '''
+ Check that Leadership Election is still functional
+ '''
+ leader_result = main.TRUE
+ description = "Check that Leadership Election is still functional"
+ main.log.report(description)
+ main.case(description)
+ main.step("Find current leader and withdraw")
+ leader = main.ONOScli1.election_test_leader()
+ #TODO: do some sanity checking on leader before using it
+ withdraw_result = main.FALSE
+ if leader == ONOS1_ip:
+ old_leader = getattr( main, "ONOScli1" )
+ elif leader == ONOS2_ip:
+ old_leader = getattr( main, "ONOScli2" )
+ elif leader == ONOS3_ip:
+ old_leader = getattr( main, "ONOScli3" )
+ elif leader == ONOS4_ip:
+ old_leader = getattr( main, "ONOScli4" )
+ elif leader == ONOS5_ip:
+ old_leader = getattr( main, "ONOScli5" )
+ elif leader == ONOS6_ip:
+ old_leader = getattr( main, "ONOScli6" )
+ elif leader == ONOS7_ip:
+ old_leader = getattr( main, "ONOScli7" )
+ elif leader == None or leader == main.FALSE:
+ main.log.report("Leader for the election app should be an ONOS node,"\
+ +"instead got '"+str(leader)+"'")
+ leader_result = main.FALSE
+ withdraw_result = old_leader.election_test_withdraw()
+
+
+ main.step("Make sure new leader is elected")
+ leader_list = []
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leader_list.append( node.election_test_leader() )
+ for leaderN in leader_list:
+ if leaderN == leader:
+ main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+ " as leader after they withdrew")
+ leader_result = main.FALSE
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ consistent_leader = main.FALSE
+ if len( set( leader_list ) ) == 1:
+ main.log.info("Each Election-app sees '"+str(leader_list[0])+"' as the leader")
+ consistent_leader = main.TRUE
+ else:
+ main.log.report("Inconsistent responses for leader of Election-app:")
+ for n in range(len(leader_list)):
+ main.log.report("ONOS" + str(n+1) + " response: " + str(leader_list[n]) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was elected when the old leader resigned)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ main.step("Run for election on old leader(just so everyone is in the hat)")
+ run_result = old_leader.election_test_run()
+ if consistent_leader == main.TRUE:
+ after_run = main.ONOScli1.election_test_leader()
+ #verify leader didn't just change
+ if after_run == leader_list[0]:
+ leader_result = main.TRUE
+ else:
+ leader_result = main.FALSE
+ #TODO: assert on run and withdraw results?
+
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election after the old leader re-ran for election")
+
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
index 68f12a8..95a75d7 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
@@ -1,9 +1,10 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],8,3,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
<Git>False</Git>
+ <num_controllers> 1 </num_controllers>
<CTRL>
<ip1>10.128.30.11</ip1>
@@ -52,8 +53,8 @@
<target10>10.0.0.27</target10>
</PING>
<timers>
- <LinkDiscovery>2</LinkDiscovery>
- <SwitchDiscovery>2</SwitchDiscovery>
+ <LinkDiscovery>.2</LinkDiscovery>
+ <SwitchDiscovery>.2</SwitchDiscovery>
</timers>
<MNtcpdump>
<intf>eth0</intf>
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
index a82bbd5..6a6fe23 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
@@ -16,6 +16,8 @@
CASE11: Switch down
CASE12: Switch up
CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
'''
class SingleInstanceHATestRestart:
@@ -61,6 +63,7 @@
global ONOS6_port
global ONOS7_ip
global ONOS7_port
+ global num_controllers
ONOS1_ip = main.params['CTRL']['ip1']
ONOS1_port = main.params['CTRL']['port1']
@@ -76,6 +79,7 @@
ONOS6_port = main.params['CTRL']['port6']
ONOS7_ip = main.params['CTRL']['ip7']
ONOS7_port = main.params['CTRL']['port7']
+ num_controllers = int(main.params['num_controllers'])
main.step("Applying cell variable to environment")
@@ -123,8 +127,7 @@
main.step("Checking if ONOS is up yet")
- #TODO: Refactor
- # check bundle:list?
+ #TODO check bundle:list?
for i in range(2):
onos1_isup = main.ONOSbench.isup(ONOS1_ip)
if onos1_isup:
@@ -197,7 +200,7 @@
Assign intents
"""
- #FIXME: Only call this once when using a persistent datastore!!!!
+ #FIXME: we must reinstall intents until we have a persistant datastore!
import time
import json
import re
@@ -221,6 +224,8 @@
#uninstall onos-app-fwd
main.log.info("Uninstall reactive forwarding app")
main.ONOScli1.feature_uninstall("onos-app-fwd")
+ #timeout for fwd flows
+ time.sleep(10)
main.step("Add host intents")
#TODO: move the host numbers to params
@@ -231,11 +236,14 @@
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
- #NOTE: get host can return None
- #TODO: handle this
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
- tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ #NOTE: get host can return None
+ if host1_id and host2_id:
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ else:
+ main.log.error("Error, get_host() failed")
+ tmp_result = main.FALSE
intent_add_result = bool(intent_add_result and tmp_result)
utilities.assert_equals(expect=True, actual=intent_add_result,
onpass="Switch mastership correctly assigned",
@@ -368,7 +376,7 @@
devices_results = main.TRUE
ports_results = main.TRUE
links_results = main.TRUE
- for controller in range(1): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -544,8 +552,36 @@
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables")
- #TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and roles_not_null
+
+ #Test of LeadershipElection
+
+ leader = ONOS1_ip
+ leader_result = main.TRUE
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ #NOTE even though we restarted ONOS, it is the only one so onos 1 must be leader
+ if leaderN == leader:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ result = mastership_check and intent_check and Flow_Tables and roles_not_null\
+ and leader_result
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -604,7 +640,7 @@
elapsed = time.time() - start_time
print "CLI time: " + str(time.time() - cli_start)
- for controller in range(1): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -651,7 +687,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Turn off a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -673,7 +709,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Restore a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -696,7 +732,7 @@
#NOTE: You should probably run a topology check after this
import time
- switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Killing a switch to ensure it is discovered correctly"
main.log.report(description)
@@ -724,6 +760,8 @@
'''
#NOTE: You should probably run a topology check after this
import time
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -755,6 +793,15 @@
'''
import os
import time
+ #printing colors to terminal
+ colors = {}
+ colors['cyan'] = '\033[96m'
+ colors['purple'] = '\033[95m'
+ colors['blue'] = '\033[94m'
+ colors['green'] = '\033[92m'
+ colors['yellow'] = '\033[93m'
+ colors['red'] = '\033[91m'
+ colors['end'] = '\033[0m'
description = "Test Cleanup"
main.log.report(description)
main.case(description)
@@ -762,7 +809,7 @@
main.Mininet2.stop_tcpdump()
main.step("Checking ONOS Logs for errors")
- print "Checking logs for errors on ONOS1:"
+ print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
print main.ONOSbench.check_logs(ONOS1_ip)
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
@@ -807,3 +854,117 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ leader_result = main.TRUE
+ #install app on onos 1
+ main.log.info("Install leadership election app")
+ main.ONOScli1.feature_install("onos-app-election")
+ #wait for election
+ #check for leader
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ #all is well
+ pass
+ elif leader == None:
+ #No leader elected
+ main.log.report("No leader was elected")
+ leader_result = main.FALSE
+ elif leader == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ else:
+ #error in response
+ main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+ leader_result = main.FALSE
+
+
+
+
+ #install on other nodes and check for leader.
+ #Should be onos1 and each app should show the same leader
+ for controller in range(2,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ node.feature_install("onos-app-election")
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == ONOS1_ip:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+ def CASE15 ( self, main ) :
+ '''
+ Check that Leadership Election is still functional
+ '''
+ leader_result = main.TRUE
+ description = "Check that Leadership Election is still functional"
+ main.log.report(description)
+ main.case(description)
+ main.step("Find current leader and withdraw")
+ leader = main.ONOScli1.election_test_leader()
+ #TODO: do some sanity checking on leader before using it
+ withdraw_result = main.FALSE
+ if leader == ONOS1_ip:
+ old_leader = getattr( main, "ONOScli1" )
+ elif leader == None or leader == main.FALSE:
+ main.log.report("Leader for the election app should be an ONOS node,"\
+ +"instead got '"+str(leader)+"'")
+ leader_result = main.FALSE
+ withdraw_result = old_leader.election_test_withdraw()
+
+
+ main.step("Make sure new leader is elected")
+ leader_list = []
+ leaderN = main.ONOScli1.election_test_leader()
+ if leaderN == leader:
+ main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+ " as leader after they withdrew")
+ leader_result = main.FALSE
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leaderN == None:
+ main.log.info("There is no leader after the app withdrew from election")
+ if leader_result:
+ main.log.report("Leadership election tests passed(There is no leader after the old leader resigned)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ main.step("Run for election on old leader(just so everyone is in the hat)")
+ run_result = old_leader.election_test_run()
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ leader_result = main.TRUE
+ else:
+ leader_result = main.FALSE
+ #TODO: assert on run and withdraw results?
+
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="ONOS1's election app was not leader after it re-ran for election")
+