Updated drivers and tests from HA. Changed include:
- onos-check-logs function
- remove some print statements
- create a csv file for jenkin's plugin
- Add timestamp to long pings
- Remove cookies from flow table comparison
- when printing flow tables, check that a correct protocol format is
given
- add cli commands for leadership election sample app
- add command to check that all devices have a masster assigned
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
index e6f2fde..a82bbd5 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
@@ -112,6 +112,8 @@
"clean install")
main.ONOSbench.get_version(report=True)
+ cell_result = main.ONOSbench.set_cell("SingleHA")
+ verify_result = main.ONOSbench.verify_cell()
main.step("Creating ONOS package")
package_result = main.ONOSbench.onos_package()
@@ -123,15 +125,17 @@
main.step("Checking if ONOS is up yet")
#TODO: Refactor
# check bundle:list?
- #this should be enough for ONOS to start
- time.sleep(60)
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if onos1_isup:
+ break
if not onos1_isup:
main.log.report("ONOS1 didn't start!")
+
# TODO: if it becomes an issue, we can retry this step a few times
- cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+ cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
main.step("Start Packet Capture MN")
main.Mininet2.start_tcpdump(
@@ -142,7 +146,7 @@
case1_result = (clean_install_result and package_result and
cell_result and verify_result and onos1_install_result and
- onos1_isup and cli1_results)
+ onos1_isup and cli_result)
utilities.assert_equals(expect=main.TRUE, actual=case1_result,
onpass="Test startup successful",
@@ -186,8 +190,6 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
def CASE3(self,main) :
@@ -195,6 +197,7 @@
Assign intents
"""
+ #FIXME: Only call this once when using a persistent datastore!!!!
import time
import json
import re
@@ -204,6 +207,10 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -219,7 +226,7 @@
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -229,7 +236,10 @@
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -275,6 +285,15 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ roles_not_null = main.ONOScli1.roles_not_null()
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
#print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
#TODO: Make this a meaningful check
@@ -285,14 +304,11 @@
else:
mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
- main.log.report("Switch roles are consistent across all ONOS nodes")
- utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
- onpass="Switch roles are consistent across all ONOS nodes",
- onfail="ONOS nodes have different views of switch roles")
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
intent_check = main.FALSE
if "Error" in ONOS1_intents or not ONOS1_intents:
@@ -300,14 +316,11 @@
main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
else:
intent_check = main.TRUE
- main.log.report("Intents are consistent across all ONOS nodes")
- utilities.assert_equals(expect = main.TRUE,actual=intent_check,
- onpass="Intents are consistent across all ONOS nodes",
- onfail="ONOS nodes have different views of intents")
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
flow_check = main.FALSE
if "Error" in ONOS1_flows or not ONOS1_flows:
@@ -317,42 +330,16 @@
#TODO: Do a better check, maybe compare flows on switches?
flow_state = ONOS1_flows
flow_check = main.TRUE
- main.log.report("Flow count is consistent across all ONOS nodes")
- utilities.assert_equals(expect = main.TRUE,actual=flow_check,
- onpass="The flow count is consistent across all ONOS nodes",
- onfail="ONOS nodes have different flow counts")
main.step("Get the OF Table entries")
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
- main.step("Start continuous pings")
- main.Mininet2.pingLong(src=main.params['PING']['source1'],
- target=main.params['PING']['target1'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source2'],
- target=main.params['PING']['target2'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source3'],
- target=main.params['PING']['target3'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source4'],
- target=main.params['PING']['target4'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source5'],
- target=main.params['PING']['target5'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source6'],
- target=main.params['PING']['target6'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source7'],
- target=main.params['PING']['target7'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source8'],
- target=main.params['PING']['target8'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source9'],
- target=main.params['PING']['target9'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source10'],
- target=main.params['PING']['target10'],pingTime=500)
-
main.step("Create TestONTopology object")
ctrls = []
count = 1
@@ -417,7 +404,7 @@
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -436,7 +423,7 @@
main.step("Checking if ONOS is up yet")
count = 0
- while count < 10
+ while count < 10:
onos1_isup = main.ONOSbench.isup(ONOS1_ip)
if onos1_isup == main.TRUE:
elapsed = time.time() - start
@@ -446,11 +433,12 @@
cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
- case_results = main.TRUE and onosi1_isup and cli_result
+ case_results = main.TRUE and onos1_isup and cli_result
utilities.assert_equals(expect=main.TRUE, actual=case_results,
onpass="ONOS restart successful",
onfail="ONOS restart NOT successful")
- main.log.info("ONOS took %s seconds to restart" % str(elapsed) )
+ main.log.info("ESTIMATE: ONOS took %s seconds to restart" % str(elapsed) )
+ time.sleep(5)
def CASE7(self,main) :
'''
@@ -460,6 +448,14 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ roles_not_null = main.ONOScli1.roles_not_null()
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
#FIXME: Refactor this whole case for single instance
@@ -508,13 +504,14 @@
main.log.report("Error in getting ONOS intents")
main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
else:
- intent_state = ONOS1_intents
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost
+ '''
main.step("Compare current intents with intents before the failure")
if intent_state == ONOS1_intents:
same_intents = main.TRUE
@@ -526,6 +523,7 @@
onpass="Intents are consistent with before failure",
onfail="The Intents changed during failure")
intent_check = intent_check and same_intents
+ '''
@@ -534,42 +532,20 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables")
- main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
- #FIXME: This check is always failing. Investigate cause
- #NOTE: this may be something to do with file permsissions
- # or slight change in format
- main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
- Loss_In_Pings = main.FALSE
- #NOTE: checkForLoss returns main.FALSE with 0% packet loss
- for i in range(8,18):
- main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
- if Loss_In_Pings == main.TRUE:
- main.log.info("Loss in ping detected")
- elif Loss_In_Pings == main.ERROR:
- main.log.info("There are multiple mininet process running")
- elif Loss_In_Pings == main.FALSE:
- main.log.info("No Loss in the pings")
- main.log.report("No loss of dataplane connectivity")
- utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
- onpass="No Loss of connectivity",
- onfail="Loss of dataplane connectivity detected")
-
-
#TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = mastership_check and intent_check and Flow_Tables and roles_not_null
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -606,62 +582,62 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- for controller in range(1): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ elapsed = time.time() - start_time
+ print "CLI time: " + str(time.time() - cli_start)
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ for controller in range(1): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = devices_results and ports_results and links_results and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ topo_result = devices_results and ports_results and links_results
+
+ topo_result = topo_result and int(count <= 2)
+ main.log.report("Very crass estimate for topology discovery/convergence(note it takes about 1 seconds to read the topology from each ONOS instance): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
+ if elapsed > 60:
+ main.log.report("Giving up on topology convergence")
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
@@ -729,13 +705,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -743,7 +724,6 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -759,9 +739,15 @@
ip1=ONOS1_ip,port1=ONOS1_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -775,8 +761,13 @@
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print "Checking logs for errors on ONOS1:"
+ print main.ONOSbench.check_logs(ONOS1_ip)
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -788,8 +779,11 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.handle.expect("\$")
+ print main.ONOSbench.handle.before
#std*.log's
#NOTE: must end in /
@@ -798,7 +792,8 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )