Updated drivers and tests from HA. Changed include:
- onos-check-logs function
- remove some print statements
- create a csv file for jenkin's plugin
- Add timestamp to long pings
- Remove cookies from flow table comparison
- when printing flow tables, check that a correct protocol format is
given
- add cli commands for leadership election sample app
- add command to check that all devices have a masster assigned
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index e1e75f9..5259a47 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,[6],8,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
@@ -28,7 +28,7 @@
<port7>6633</port7>
</CTRL>
<TESTONUSER>admin</TESTONUSER>
- <TESTONIP>10.128.30.10</TESTONIP>
+ <TESTONIP>10.128.30.9</TESTONIP>
<PING>
<source1>h8</source1>
<source2>h9</source2>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 614d514..4e5b463 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -214,25 +214,6 @@
import json
import re
-
- '''
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS1_port = main.params['CTRL']['port1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS2_port = main.params['CTRL']['port2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS3_port = main.params['CTRL']['port3']
- ONOS4_ip = main.params['CTRL']['ip4']
- ONOS4_port = main.params['CTRL']['port4']
- ONOS5_ip = main.params['CTRL']['ip5']
- ONOS5_port = main.params['CTRL']['port5']
- ONOS6_ip = main.params['CTRL']['ip6']
- ONOS6_port = main.params['CTRL']['port6']
- ONOS7_ip = main.params['CTRL']['ip7']
- ONOS7_port = main.params['CTRL']['port7']
- '''
-
-
main.log.report("Assigning switches to controllers")
main.case("Assigning Controllers")
main.step("Assign switches to controllers")
@@ -270,8 +251,217 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
+ #Manually assign mastership to the controller we want
+ role_call = main.TRUE
+ role_check = main.TRUE
+
+ device_id = main.ONOScli1.get_device("1000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2800")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("5000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3004")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+ if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3008")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3009")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3010")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3011")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3012")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3013")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3014")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3015")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3016")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3017")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6007")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+ if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6018")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6019")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6020")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6021")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6022")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6023")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6024")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6025")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6026")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6027")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_call,
+ onpass="Re-assigned switch mastership to designated controller",
+ onfail="Something wrong with device_role calls")
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_check,
+ onpass="Switches were successfully reassigned to designated controller",
+ onfail="Switches were not successfully reassigned")
+ mastership_check = mastership_check and role_call and role_check
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
def CASE3(self,main) :
@@ -288,6 +478,16 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+ main.ONOScli2.feature_install("onos-app-fwd")
+ main.ONOScli3.feature_install("onos-app-fwd")
+ main.ONOScli4.feature_install("onos-app-fwd")
+ main.ONOScli5.feature_install("onos-app-fwd")
+ main.ONOScli6.feature_install("onos-app-fwd")
+ main.ONOScli7.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -309,7 +509,7 @@
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -319,7 +519,10 @@
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -365,6 +568,25 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
ONOS3_mastership = main.ONOScli3.roles()
@@ -421,6 +643,7 @@
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
ONOS2_intents = main.ONOScli2.intents( json_format=True )
ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -475,6 +698,7 @@
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
ONOS2_flows = main.ONOScli2.flows( json_format=True )
ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -532,7 +756,7 @@
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
@@ -583,7 +807,6 @@
devices.append( main.ONOScli5.devices() )
devices.append( main.ONOScli6.devices() )
devices.append( main.ONOScli7.devices() )
- '''
hosts = []
hosts.append( main.ONOScli1.hosts() )
hosts.append( main.ONOScli2.hosts() )
@@ -592,7 +815,6 @@
hosts.append( main.ONOScli5.hosts() )
hosts.append( main.ONOScli6.hosts() )
hosts.append( main.ONOScli7.hosts() )
- '''
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -609,6 +831,91 @@
links.append( main.ONOScli5.links() )
links.append( main.ONOScli6.links() )
links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
main.step("Comparing ONOS topology to MN")
@@ -644,14 +951,16 @@
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result\
+ and consistent_paths_result
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -676,6 +985,24 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
@@ -707,7 +1034,6 @@
and ONOS1_mastership == ONOS5_mastership\
and ONOS1_mastership == ONOS6_mastership\
and ONOS1_mastership == ONOS7_mastership:
- #mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
main.log.report("Switch roles are consistent across all ONOS nodes")
else:
@@ -789,25 +1115,35 @@
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
else:
- main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS2 intents: ")
+ print json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS3 intents: ")
+ print json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS4 intents: ")
+ print json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS5 intents: ")
+ print json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS6 intents: ")
+ print json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS7 intents: ")
+ print json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost
main.step("Compare current intents with intents before the failure")
+ #NOTE: this requires case 5 to pass for intent_state to be set.
+ # maybe we should stop the test if that fails?
if intent_state == ONOS1_intents:
same_intents = main.TRUE
main.log.report("Intents are consistent with before failure")
@@ -826,12 +1162,12 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -847,7 +1183,7 @@
#NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range(8,18):
main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
if Loss_In_Pings == main.TRUE:
main.log.info("Loss in ping detected")
elif Loss_In_Pings == main.ERROR:
@@ -861,7 +1197,7 @@
#TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -903,89 +1239,173 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
- print "cond 1:" + str(topo_result == main.FALSE)
- print "cond 2:" + str(elapsed < 120)
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
- for controller in range(7): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ elapsed = time.time() - start_time
+ cli_time = time.time() - cli_start
+ print "CLI time: " + str(cli_time)
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = topo_result and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
+
+
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+ topo_result = topo_result and int(count <= 2)
+ note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+ main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
@@ -1054,13 +1474,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -1068,7 +1493,6 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1090,9 +1514,15 @@
ip7=ONOS7_ip,port7=ONOS7_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -1106,8 +1536,26 @@
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print "Checking logs for errors on ONOS1:"
+ print main.ONOSbench.check_logs(ONOS1_ip)
+ print "Checking logs for errors on ONOS2:"
+ print main.ONOSbench.check_logs(ONOS2_ip)
+ print "Checking logs for errors on ONOS3:"
+ print main.ONOSbench.check_logs(ONOS3_ip)
+ print "Checking logs for errors on ONOS4:"
+ print main.ONOSbench.check_logs(ONOS4_ip)
+ print "Checking logs for errors on ONOS5:"
+ print main.ONOSbench.check_logs(ONOS5_ip)
+ print "Checking logs for errors on ONOS6:"
+ print main.ONOSbench.check_logs(ONOS6_ip)
+ print "Checking logs for errors on ONOS7:"
+ print main.ONOSbench.check_logs(ONOS7_ip)
+
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -1119,19 +1567,26 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
#std*.log's
@@ -1141,24 +1596,29 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
-
-
#sleep so scp can finish
time.sleep(10)
main.step("Packing and rotating pcap archives")
@@ -1169,3 +1629,22 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ #install app on onos 1
+ #wait for election
+ #check for leader
+ #install on other nodes
+ #check for leader. Should be onos1 and each app shows same leader
+ #
+
+
+
+
+
+ #Next Case
+ #add to reboot case?
+ #check for conistent leader
+ #new leader should have been elected