Rename single instance HA test
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
new file mode 100644
index 0000000..51ac0f6
--- /dev/null
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -0,0 +1,970 @@
+'''
+Description: This test is to determine if a single
+ instance ONOS 'cluster' can handle a restart
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE6: The Failure case. Since this is the Sanity test, we do nothing.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
+'''
+class HATestSingleInstanceRestart:
+
+ def __init__(self) :
+ self.default = ''
+
+ def CASE1(self,main) :
+ '''
+ CASE1 is to compile ONOS and push it to the test machines
+
+ Startup sequence:
+ git pull
+ mvn clean install
+ onos-package
+ cell <name>
+ onos-verify-cell
+ NOTE: temporary - onos-remove-raft-logs
+ onos-install -f
+ onos-wait-for-start
+ '''
+ import time
+ main.log.report("ONOS Single node cluster restart HA test - initialization")
+ main.case("Setting up test environment")
+
+ # load some vairables from the params file
+ PULL_CODE = False
+ if main.params['Git'] == 'True':
+ PULL_CODE = True
+ cell_name = main.params['ENV']['cellName']
+
+ #set global variables
+ global ONOS1_ip
+ global ONOS1_port
+ global ONOS2_ip
+ global ONOS2_port
+ global ONOS3_ip
+ global ONOS3_port
+ global ONOS4_ip
+ global ONOS4_port
+ global ONOS5_ip
+ global ONOS5_port
+ global ONOS6_ip
+ global ONOS6_port
+ global ONOS7_ip
+ global ONOS7_port
+ global num_controllers
+
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS1_port = main.params['CTRL']['port1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS2_port = main.params['CTRL']['port2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS3_port = main.params['CTRL']['port3']
+ ONOS4_ip = main.params['CTRL']['ip4']
+ ONOS4_port = main.params['CTRL']['port4']
+ ONOS5_ip = main.params['CTRL']['ip5']
+ ONOS5_port = main.params['CTRL']['port5']
+ ONOS6_ip = main.params['CTRL']['ip6']
+ ONOS6_port = main.params['CTRL']['port6']
+ ONOS7_ip = main.params['CTRL']['ip7']
+ ONOS7_port = main.params['CTRL']['port7']
+ num_controllers = int(main.params['num_controllers'])
+
+
+ main.step("Applying cell variable to environment")
+ cell_result = main.ONOSbench.set_cell(cell_name)
+ verify_result = main.ONOSbench.verify_cell()
+
+ #FIXME:this is short term fix
+ main.log.report("Removing raft logs")
+ main.ONOSbench.onos_remove_raft_logs()
+ main.log.report("Uninstalling ONOS")
+ main.ONOSbench.onos_uninstall(ONOS1_ip)
+ main.ONOSbench.onos_uninstall(ONOS2_ip)
+ main.ONOSbench.onos_uninstall(ONOS3_ip)
+ main.ONOSbench.onos_uninstall(ONOS4_ip)
+ main.ONOSbench.onos_uninstall(ONOS5_ip)
+ main.ONOSbench.onos_uninstall(ONOS6_ip)
+ main.ONOSbench.onos_uninstall(ONOS7_ip)
+
+ clean_install_result = main.TRUE
+ git_pull_result = main.TRUE
+
+ main.step("Compiling the latest version of ONOS")
+ if PULL_CODE:
+ main.step("Git checkout and pull master")
+ main.ONOSbench.git_checkout("master")
+ git_pull_result = main.ONOSbench.git_pull()
+
+ main.step("Using mvn clean & install")
+ clean_install_result = main.TRUE
+ if git_pull_result == main.TRUE:
+ clean_install_result = main.ONOSbench.clean_install()
+ else:
+ main.log.warn("Did not pull new code so skipping mvn "+ \
+ "clean install")
+ main.ONOSbench.get_version(report=True)
+
+ cell_result = main.ONOSbench.set_cell("SingleHA")
+ verify_result = main.ONOSbench.verify_cell()
+ main.step("Creating ONOS package")
+ package_result = main.ONOSbench.onos_package()
+
+ main.step("Installing ONOS package")
+ onos1_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS1_ip)
+
+
+ main.step("Checking if ONOS is up yet")
+ #TODO check bundle:list?
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if onos1_isup:
+ break
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+
+ # TODO: if it becomes an issue, we can retry this step a few times
+
+
+ cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
+
+ main.step("Start Packet Capture MN")
+ main.Mininet2.start_tcpdump(
+ str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+ intf = main.params['MNtcpdump']['intf'],
+ port = main.params['MNtcpdump']['port'])
+
+
+ case1_result = (clean_install_result and package_result and
+ cell_result and verify_result and onos1_install_result and
+ onos1_isup and cli_result)
+
+ utilities.assert_equals(expect=main.TRUE, actual=case1_result,
+ onpass="Test startup successful",
+ onfail="Test startup NOT successful")
+
+
+ if case1_result==main.FALSE:
+ main.cleanup()
+ main.exit()
+
+ def CASE2(self,main) :
+ '''
+ Assign mastership to controllers
+ '''
+ import time
+ import json
+ import re
+
+ main.log.report("Assigning switches to controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign switches to controllers")
+
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),
+ ip1=ONOS1_ip,port1=ONOS1_port)
+
+ mastership_check = main.TRUE
+ for i in range (1,29):
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ try:
+ main.log.info(str(response))
+ except:
+ main.log.info(repr(response))
+ if re.search("tcp:"+ONOS1_ip,response):
+ mastership_check = mastership_check and main.TRUE
+ else:
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Switch mastership assigned correctly")
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership assigned correctly",
+ onfail="Switches not assigned correctly to controllers")
+
+
+
+ def CASE3(self,main) :
+ """
+ Assign intents
+
+ """
+ #FIXME: we must reinstall intents until we have a persistant datastore!
+ import time
+ import json
+ import re
+ main.log.report("Adding host intents")
+ main.case("Adding host Intents")
+
+ main.step("Discovering Hosts( Via pingall for now)")
+ #FIXME: Once we have a host discovery mechanism, use that instead
+
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+
+ #REACTIVE FWD test
+ ping_result = main.FALSE
+ time1 = time.time()
+ ping_result = main.Mininet1.pingall()
+ time2 = time.time()
+ main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
+
+ #uninstall onos-app-fwd
+ main.log.info("Uninstall reactive forwarding app")
+ main.ONOScli1.feature_uninstall("onos-app-fwd")
+ #timeout for fwd flows
+ time.sleep(10)
+
+ main.step("Add host intents")
+ #TODO: move the host numbers to params
+ import json
+ intents_json= json.loads(main.ONOScli1.hosts())
+ intent_add_result = True
+ for i in range(8,18):
+ main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+ host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+ host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+ host1_id = main.ONOScli1.get_host(host1)['id']
+ host2_id = main.ONOScli1.get_host(host2)['id']
+ #NOTE: get host can return None
+ if host1_id and host2_id:
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ else:
+ main.log.error("Error, get_host() failed")
+ tmp_result = main.FALSE
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
+ #TODO Check if intents all exist in datastore
+ #NOTE: Do we need to print this once the test is working?
+ #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
+ # sort_keys=True, indent=4, separators=(',', ': ') ) )
+
+ def CASE4(self,main) :
+ """
+ Ping across added host intents
+ """
+ description = " Ping across added host intents"
+ main.log.report(description)
+ main.case(description)
+ Ping_Result = main.TRUE
+ for i in range(8,18):
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ Ping_Result = Ping_Result and ping
+ if ping==main.FALSE:
+ main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
+ elif ping==main.TRUE:
+ main.log.info("Ping test passed!")
+ Ping_Result = main.TRUE
+ if Ping_Result==main.FALSE:
+ main.log.report("Intents have not been installed correctly, pings failed.")
+ if Ping_Result==main.TRUE:
+ main.log.report("Intents have been installed correctly and verified by pings")
+ utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
+ onpass="Intents have been installed correctly and pings work",
+ onfail ="Intents have not been installed correctly, pings failed." )
+
+ def CASE5(self,main) :
+ '''
+ Reading state of ONOS
+ '''
+ import time
+ import json
+ from subprocess import Popen, PIPE
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+
+ main.log.report("Setting up and gathering data for current state")
+ main.case("Setting up and gathering data for current state")
+ #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
+ #We can then compare them with eachother and also with past states
+
+ main.step("Get the Mastership of each switch from each controller")
+ global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ roles_not_null = main.ONOScli1.roles_not_null()
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+ ONOS1_mastership = main.ONOScli1.roles()
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ #TODO: Make this a meaningful check
+ if "Error" in ONOS1_mastership or not ONOS1_mastership:
+ main.log.report("Error in getting ONOS roles")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ consistent_mastership = main.FALSE
+ else:
+ mastership_state = ONOS1_mastership
+ consistent_mastership = main.TRUE
+
+
+ main.step("Get the intents from each controller")
+ global intent_state
+ intent_state = []
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ else:
+ intent_check = main.TRUE
+
+
+ main.step("Get the flows from each controller")
+ global flow_state
+ flow_state = []
+ ONOS1_flows = main.ONOScli1.flows( json_format=True )
+ flow_check = main.FALSE
+ if "Error" in ONOS1_flows or not ONOS1_flows:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
+ else:
+ #TODO: Do a better check, maybe compare flows on switches?
+ flow_state = ONOS1_flows
+ flow_check = main.TRUE
+
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,29):
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
+
+ #TODO: Compare switch flow tables with ONOS flow tables
+
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ temp = ()
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ for controller in range(num_controllers):
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+
+ topo_result = devices_results and ports_results and links_results
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+
+ final_assert = main.TRUE
+ final_assert = final_assert and topo_result and flow_check \
+ and intent_check and consistent_mastership and roles_not_null
+ utilities.assert_equals(expect=main.TRUE, actual=final_assert,
+ onpass="State check successful",
+ onfail="State check NOT successful")
+
+
+ def CASE6(self,main) :
+ '''
+ The Failure case.
+ '''
+ import time
+
+ main.log.report("Restart ONOS node")
+ main.log.case("Restart ONOS node")
+ main.ONOSbench.onos_kill(ONOS1_ip)
+ start = time.time()
+
+ main.step("Checking if ONOS is up yet")
+ count = 0
+ while count < 10:
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if onos1_isup == main.TRUE:
+ elapsed = time.time() - start
+ break
+ else:
+ count = count + 1
+
+ cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
+
+ case_results = main.TRUE and onos1_isup and cli_result
+ utilities.assert_equals(expect=main.TRUE, actual=case_results,
+ onpass="ONOS restart successful",
+ onfail="ONOS restart NOT successful")
+ main.log.info("ESTIMATE: ONOS took %s seconds to restart" % str(elapsed) )
+ time.sleep(5)
+
+ def CASE7(self,main) :
+ '''
+ Check state after ONOS failure
+ '''
+ import os
+ import json
+ main.case("Running ONOS Constant State Tests")
+
+ #Assert that each device has a master
+ roles_not_null = main.ONOScli1.roles_not_null()
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
+ main.step("Check if switch roles are consistent across all nodes")
+ ONOS1_mastership = main.ONOScli1.roles()
+ #FIXME: Refactor this whole case for single instance
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ if "Error" in ONOS1_mastership or not ONOS1_mastership:
+ main.log.report("Error in getting ONOS mastership")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ consistent_mastership = main.FALSE
+ else:
+ consistent_mastership = main.TRUE
+ main.log.report("Switch roles are consistent across all ONOS nodes")
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+ onpass="Switch roles are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of switch roles")
+
+
+ description2 = "Compare switch roles from before failure"
+ main.step(description2)
+
+ current_json = json.loads(ONOS1_mastership)
+ old_json = json.loads(mastership_state)
+ mastership_check = main.TRUE
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+
+ current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
+ old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
+ if current == old:
+ mastership_check = mastership_check and main.TRUE
+ else:
+ main.log.warn("Mastership of switch %s changed" % switchDPID)
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Mastership of Switches was not changed")
+ utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
+ onpass="Mastership of Switches was not changed",
+ onfail="Mastership of some switches changed")
+ mastership_check = mastership_check and consistent_mastership
+
+
+
+ main.step("Get the intents and compare across all nodes")
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ else:
+ intent_check = main.TRUE
+ main.log.report("Intents are consistent across all ONOS nodes")
+ utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+ onpass="Intents are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of intents")
+
+ #NOTE: Hazelcast has no durability, so intents are lost
+ '''
+ main.step("Compare current intents with intents before the failure")
+ if intent_state == ONOS1_intents:
+ same_intents = main.TRUE
+ main.log.report("Intents are consistent with before failure")
+ #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
+ else:
+ same_intents = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=same_intents,
+ onpass="Intents are consistent with before failure",
+ onfail="The Intents changed during failure")
+ intent_check = intent_check and same_intents
+ '''
+
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ Flow_Tables = main.TRUE
+ flows2=[]
+ for i in range(28):
+ main.log.info("Checking flow table on s" + str(i+1))
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
+ flows2.append(tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
+ if Flow_Tables == main.FALSE:
+ main.log.info("Differences in flow table for switch: "+str(i+1))
+ if Flow_Tables == main.TRUE:
+ main.log.report("No changes were found in the flow tables")
+ utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
+ onpass="No changes were found in the flow tables",
+ onfail="Changes were found in the flow tables")
+
+
+ #Test of LeadershipElection
+
+ leader = ONOS1_ip
+ leader_result = main.TRUE
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ #NOTE even though we restarted ONOS, it is the only one so onos 1 must be leader
+ if leaderN == leader:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ result = mastership_check and intent_check and Flow_Tables and roles_not_null\
+ and leader_result
+ result = int(result)
+ if result == main.TRUE:
+ main.log.report("Constant State Tests Passed")
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Constant State Tests Passed",
+ onfail="Constant state tests failed")
+
+ def CASE8 (self,main):
+ '''
+ Compare topo
+ '''
+ import sys
+ sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+ import json
+ import time
+
+ description ="Compare ONOS Topology view to Mininet topology"
+ main.case(description)
+ main.log.report(description)
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ temp = ()
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ topo_result = main.FALSE
+ elapsed = 0
+ count = 0
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
+ count = count + 1
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ elapsed = time.time() - start_time
+ print "CLI time: " + str(time.time() - cli_start)
+
+ for controller in range(num_controllers):
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+ topo_result = devices_results and ports_results and links_results
+
+ topo_result = topo_result and int(count <= 2)
+ main.log.report("Very crass estimate for topology discovery/convergence(note it takes about 1 seconds to read the topology from each ONOS instance): " +\
+ str(elapsed) + " seconds, " + str(count) +" tries" )
+ if elapsed > 60:
+ main.log.report("Giving up on topology convergence")
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+ if topo_result == main.TRUE:
+ main.log.report("ONOS topology view matches Mininet topology")
+
+
+ def CASE9 (self,main):
+ '''
+ Link s3-s28 down
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
+
+ description = "Turn off a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+
+ main.step("Kill Link between s3 and s28")
+ Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
+ onpass="Link down succesful",
+ onfail="Failed to bring link down")
+ #TODO do some sort of check here
+
+ def CASE10 (self,main):
+ '''
+ Link s3-s28 up
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
+
+ description = "Restore a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Bring link between s3 and s28 back up")
+ Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
+ onpass="Link up succesful",
+ onfail="Failed to bring link up")
+ #TODO do some sort of check here
+
+
+ def CASE11 (self, main) :
+ '''
+ Switch Down
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
+
+ description = "Killing a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ #TODO: Make this switch parameterizable
+ main.step("Kill s28 ")
+ main.log.report("Deleting s28")
+ main.Mininet1.del_switch("s28")
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+ time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
+
+ def CASE12 (self, main) :
+ '''
+ Switch Up
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
+ description = "Adding a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Add back s28")
+ main.log.report("Adding back s28")
+ main.Mininet1.add_switch("s28", dpid = '0000000000002800')
+ #TODO: New dpid or same? Ask Thomas?
+ main.Mininet1.add_link('s28', 's3')
+ main.Mininet1.add_link('s28', 's6')
+ main.Mininet1.add_link('s28', 'h28')
+ main.Mininet1.assign_sw_controller(sw="28",
+ ip1=ONOS1_ip,port1=ONOS1_port)
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
+ time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
+
+ def CASE13 (self, main) :
+ '''
+ Clean up
+ '''
+ import os
+ import time
+ #printing colors to terminal
+ colors = {}
+ colors['cyan'] = '\033[96m'
+ colors['purple'] = '\033[95m'
+ colors['blue'] = '\033[94m'
+ colors['green'] = '\033[92m'
+ colors['yellow'] = '\033[93m'
+ colors['red'] = '\033[91m'
+ colors['end'] = '\033[0m'
+ description = "Test Cleanup"
+ main.log.report(description)
+ main.case(description)
+ main.step("Killing tcpdumps")
+ main.Mininet2.stop_tcpdump()
+
+ main.step("Checking ONOS Logs for errors")
+ print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS1_ip)
+ main.step("Copying MN pcap and ONOS log files to test station")
+ testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
+ #NOTE: MN Pcap file is being saved to ~/packet_captures
+ # scp this file as MN and TestON aren't necessarily the same vm
+ #FIXME: scp
+ #####mn files
+ #TODO: Load these from params
+ #NOTE: must end in /
+ log_folder = "/opt/onos/log/"
+ log_files = ["karaf.log", "karaf.log.1"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.handle.expect("\$")
+ print main.ONOSbench.handle.before
+
+ #std*.log's
+ #NOTE: must end in /
+ log_folder = "/opt/onos/var/"
+ log_files = ["stderr.log", "stdout.log"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+
+
+ #sleep so scp can finish
+ time.sleep(10)
+ main.step("Packing and rotating pcap archives")
+ os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
+
+
+ #TODO: actually check something here
+ utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+ onpass="Test cleanup successful",
+ onfail="Test cleanup NOT successful")
+
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ leader_result = main.TRUE
+ #install app on onos 1
+ main.log.info("Install leadership election app")
+ main.ONOScli1.feature_install("onos-app-election")
+ #wait for election
+ #check for leader
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ #all is well
+ pass
+ elif leader == None:
+ #No leader elected
+ main.log.report("No leader was elected")
+ leader_result = main.FALSE
+ elif leader == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ else:
+ #error in response
+ main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+ leader_result = main.FALSE
+
+
+
+
+ #install on other nodes and check for leader.
+ #Should be onos1 and each app should show the same leader
+ for controller in range(2,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ node.feature_install("onos-app-election")
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == ONOS1_ip:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+ def CASE15 ( self, main ) :
+ '''
+ Check that Leadership Election is still functional
+ '''
+ leader_result = main.TRUE
+ description = "Check that Leadership Election is still functional"
+ main.log.report(description)
+ main.case(description)
+ main.step("Find current leader and withdraw")
+ leader = main.ONOScli1.election_test_leader()
+ #TODO: do some sanity checking on leader before using it
+ withdraw_result = main.FALSE
+ if leader == ONOS1_ip:
+ old_leader = getattr( main, "ONOScli1" )
+ elif leader == None or leader == main.FALSE:
+ main.log.report("Leader for the election app should be an ONOS node,"\
+ +"instead got '"+str(leader)+"'")
+ leader_result = main.FALSE
+ withdraw_result = old_leader.election_test_withdraw()
+
+
+ main.step("Make sure new leader is elected")
+ leader_list = []
+ leaderN = main.ONOScli1.election_test_leader()
+ if leaderN == leader:
+ main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+ " as leader after they withdrew")
+ leader_result = main.FALSE
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leaderN == None:
+ main.log.info("There is no leader after the app withdrew from election")
+ if leader_result:
+ main.log.report("Leadership election tests passed(There is no leader after the old leader resigned)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ main.step("Run for election on old leader(just so everyone is in the hat)")
+ run_result = old_leader.election_test_run()
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ leader_result = main.TRUE
+ else:
+ leader_result = main.FALSE
+ #TODO: assert on run and withdraw results?
+
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="ONOS1's election app was not leader after it re-ran for election")
+