blob: 6a6fe230407860d4de4a4f77dbd853f27858421b [file] [log] [blame]
Jon Hall73cf9cc2014-11-20 22:28:38 -08001'''
2Description: This test is to determine if a single
3 instance ONOS 'cluster' can handle a restart
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign mastership to controllers
8CASE3: Assign intents
9CASE4: Ping across added host intents
10CASE5: Reading state of ONOS
11CASE6: The Failure case. Since this is the Sanity test, we do nothing.
12CASE7: Check state after control plane failure
13CASE8: Compare topo
14CASE9: Link s3-s28 down
15CASE10: Link s3-s28 up
16CASE11: Switch down
17CASE12: Switch up
18CASE13: Clean up
Jon Hall669173b2014-12-17 11:36:30 -080019CASE14: start election app on all onos nodes
20CASE15: Check that Leadership Election is still functional
Jon Hall73cf9cc2014-11-20 22:28:38 -080021'''
22class SingleInstanceHATestRestart:
23
24 def __init__(self) :
25 self.default = ''
26
27 def CASE1(self,main) :
28 '''
29 CASE1 is to compile ONOS and push it to the test machines
30
31 Startup sequence:
32 git pull
33 mvn clean install
34 onos-package
35 cell <name>
36 onos-verify-cell
37 NOTE: temporary - onos-remove-raft-logs
38 onos-install -f
39 onos-wait-for-start
40 '''
41 import time
42 main.log.report("ONOS Single node cluster restart HA test - initialization")
43 main.case("Setting up test environment")
44
45 # load some vairables from the params file
46 PULL_CODE = False
47 if main.params['Git'] == 'True':
48 PULL_CODE = True
49 cell_name = main.params['ENV']['cellName']
50
51 #set global variables
52 global ONOS1_ip
53 global ONOS1_port
54 global ONOS2_ip
55 global ONOS2_port
56 global ONOS3_ip
57 global ONOS3_port
58 global ONOS4_ip
59 global ONOS4_port
60 global ONOS5_ip
61 global ONOS5_port
62 global ONOS6_ip
63 global ONOS6_port
64 global ONOS7_ip
65 global ONOS7_port
Jon Hall669173b2014-12-17 11:36:30 -080066 global num_controllers
Jon Hall73cf9cc2014-11-20 22:28:38 -080067
68 ONOS1_ip = main.params['CTRL']['ip1']
69 ONOS1_port = main.params['CTRL']['port1']
70 ONOS2_ip = main.params['CTRL']['ip2']
71 ONOS2_port = main.params['CTRL']['port2']
72 ONOS3_ip = main.params['CTRL']['ip3']
73 ONOS3_port = main.params['CTRL']['port3']
74 ONOS4_ip = main.params['CTRL']['ip4']
75 ONOS4_port = main.params['CTRL']['port4']
76 ONOS5_ip = main.params['CTRL']['ip5']
77 ONOS5_port = main.params['CTRL']['port5']
78 ONOS6_ip = main.params['CTRL']['ip6']
79 ONOS6_port = main.params['CTRL']['port6']
80 ONOS7_ip = main.params['CTRL']['ip7']
81 ONOS7_port = main.params['CTRL']['port7']
Jon Hall669173b2014-12-17 11:36:30 -080082 num_controllers = int(main.params['num_controllers'])
Jon Hall73cf9cc2014-11-20 22:28:38 -080083
84
85 main.step("Applying cell variable to environment")
86 cell_result = main.ONOSbench.set_cell(cell_name)
87 verify_result = main.ONOSbench.verify_cell()
88
89 #FIXME:this is short term fix
90 main.log.report("Removing raft logs")
91 main.ONOSbench.onos_remove_raft_logs()
92 main.log.report("Uninstalling ONOS")
93 main.ONOSbench.onos_uninstall(ONOS1_ip)
94 main.ONOSbench.onos_uninstall(ONOS2_ip)
95 main.ONOSbench.onos_uninstall(ONOS3_ip)
96 main.ONOSbench.onos_uninstall(ONOS4_ip)
97 main.ONOSbench.onos_uninstall(ONOS5_ip)
98 main.ONOSbench.onos_uninstall(ONOS6_ip)
99 main.ONOSbench.onos_uninstall(ONOS7_ip)
100
101 clean_install_result = main.TRUE
102 git_pull_result = main.TRUE
103
104 main.step("Compiling the latest version of ONOS")
105 if PULL_CODE:
106 main.step("Git checkout and pull master")
107 main.ONOSbench.git_checkout("master")
108 git_pull_result = main.ONOSbench.git_pull()
109
110 main.step("Using mvn clean & install")
111 clean_install_result = main.TRUE
112 if git_pull_result == main.TRUE:
113 clean_install_result = main.ONOSbench.clean_install()
114 else:
115 main.log.warn("Did not pull new code so skipping mvn "+ \
116 "clean install")
117 main.ONOSbench.get_version(report=True)
118
Jon Hall94fd0472014-12-08 11:52:42 -0800119 cell_result = main.ONOSbench.set_cell("SingleHA")
120 verify_result = main.ONOSbench.verify_cell()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800121 main.step("Creating ONOS package")
122 package_result = main.ONOSbench.onos_package()
123
124 main.step("Installing ONOS package")
125 onos1_install_result = main.ONOSbench.onos_install(options="-f",
126 node=ONOS1_ip)
127
128
129 main.step("Checking if ONOS is up yet")
Jon Hall669173b2014-12-17 11:36:30 -0800130 #TODO check bundle:list?
Jon Hall94fd0472014-12-08 11:52:42 -0800131 for i in range(2):
132 onos1_isup = main.ONOSbench.isup(ONOS1_ip)
133 if onos1_isup:
134 break
Jon Hall73cf9cc2014-11-20 22:28:38 -0800135 if not onos1_isup:
136 main.log.report("ONOS1 didn't start!")
Jon Hall94fd0472014-12-08 11:52:42 -0800137
Jon Hall73cf9cc2014-11-20 22:28:38 -0800138 # TODO: if it becomes an issue, we can retry this step a few times
139
140
Jon Hall94fd0472014-12-08 11:52:42 -0800141 cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800142
143 main.step("Start Packet Capture MN")
144 main.Mininet2.start_tcpdump(
145 str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
146 intf = main.params['MNtcpdump']['intf'],
147 port = main.params['MNtcpdump']['port'])
148
149
150 case1_result = (clean_install_result and package_result and
151 cell_result and verify_result and onos1_install_result and
Jon Hall94fd0472014-12-08 11:52:42 -0800152 onos1_isup and cli_result)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800153
154 utilities.assert_equals(expect=main.TRUE, actual=case1_result,
155 onpass="Test startup successful",
156 onfail="Test startup NOT successful")
157
158
159 if case1_result==main.FALSE:
160 main.cleanup()
161 main.exit()
162
163 def CASE2(self,main) :
164 '''
165 Assign mastership to controllers
166 '''
167 import time
168 import json
169 import re
170
171 main.log.report("Assigning switches to controllers")
172 main.case("Assigning Controllers")
173 main.step("Assign switches to controllers")
174
175 for i in range (1,29):
176 main.Mininet1.assign_sw_controller(sw=str(i),
177 ip1=ONOS1_ip,port1=ONOS1_port)
178
179 mastership_check = main.TRUE
180 for i in range (1,29):
181 response = main.Mininet1.get_sw_controller("s"+str(i))
Jon Hallffb386d2014-11-21 13:43:38 -0800182 try:
183 main.log.info(str(response))
184 except:
185 main.log.info(repr(response))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800186 if re.search("tcp:"+ONOS1_ip,response):
187 mastership_check = mastership_check and main.TRUE
188 else:
189 mastership_check = main.FALSE
190 if mastership_check == main.TRUE:
191 main.log.report("Switch mastership assigned correctly")
192 utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
193 onpass="Switch mastership assigned correctly",
194 onfail="Switches not assigned correctly to controllers")
195
Jon Hall73cf9cc2014-11-20 22:28:38 -0800196
197
198 def CASE3(self,main) :
199 """
200 Assign intents
201
202 """
Jon Hall669173b2014-12-17 11:36:30 -0800203 #FIXME: we must reinstall intents until we have a persistant datastore!
Jon Hall73cf9cc2014-11-20 22:28:38 -0800204 import time
205 import json
206 import re
207 main.log.report("Adding host intents")
208 main.case("Adding host Intents")
209
210 main.step("Discovering Hosts( Via pingall for now)")
211 #FIXME: Once we have a host discovery mechanism, use that instead
212
Jon Hall94fd0472014-12-08 11:52:42 -0800213 #install onos-app-fwd
214 main.log.info("Install reactive forwarding app")
215 main.ONOScli1.feature_install("onos-app-fwd")
216
Jon Hall73cf9cc2014-11-20 22:28:38 -0800217 #REACTIVE FWD test
218 ping_result = main.FALSE
219 time1 = time.time()
220 ping_result = main.Mininet1.pingall()
221 time2 = time.time()
222 main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
223
224 #uninstall onos-app-fwd
225 main.log.info("Uninstall reactive forwarding app")
226 main.ONOScli1.feature_uninstall("onos-app-fwd")
Jon Hall669173b2014-12-17 11:36:30 -0800227 #timeout for fwd flows
228 time.sleep(10)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800229
230 main.step("Add host intents")
231 #TODO: move the host numbers to params
232 import json
233 intents_json= json.loads(main.ONOScli1.hosts())
Jon Hall94fd0472014-12-08 11:52:42 -0800234 intent_add_result = True
Jon Hall73cf9cc2014-11-20 22:28:38 -0800235 for i in range(8,18):
236 main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
237 host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
238 host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800239 host1_id = main.ONOScli1.get_host(host1)['id']
240 host2_id = main.ONOScli1.get_host(host2)['id']
Jon Hall669173b2014-12-17 11:36:30 -0800241 #NOTE: get host can return None
242 if host1_id and host2_id:
243 tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
244 else:
245 main.log.error("Error, get_host() failed")
246 tmp_result = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800247 intent_add_result = bool(intent_add_result and tmp_result)
248 utilities.assert_equals(expect=True, actual=intent_add_result,
249 onpass="Switch mastership correctly assigned",
250 onfail="Error in (re)assigning switch mastership")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800251 #TODO Check if intents all exist in datastore
252 #NOTE: Do we need to print this once the test is working?
253 #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
254 # sort_keys=True, indent=4, separators=(',', ': ') ) )
255
256 def CASE4(self,main) :
257 """
258 Ping across added host intents
259 """
260 description = " Ping across added host intents"
261 main.log.report(description)
262 main.case(description)
263 Ping_Result = main.TRUE
264 for i in range(8,18):
265 ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
266 Ping_Result = Ping_Result and ping
267 if ping==main.FALSE:
268 main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
269 elif ping==main.TRUE:
270 main.log.info("Ping test passed!")
271 Ping_Result = main.TRUE
272 if Ping_Result==main.FALSE:
273 main.log.report("Intents have not been installed correctly, pings failed.")
274 if Ping_Result==main.TRUE:
275 main.log.report("Intents have been installed correctly and verified by pings")
276 utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
277 onpass="Intents have been installed correctly and pings work",
278 onfail ="Intents have not been installed correctly, pings failed." )
279
280 def CASE5(self,main) :
281 '''
282 Reading state of ONOS
283 '''
284 import time
285 import json
286 from subprocess import Popen, PIPE
287 from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
288
289 main.log.report("Setting up and gathering data for current state")
290 main.case("Setting up and gathering data for current state")
291 #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
292 #We can then compare them with eachother and also with past states
293
294 main.step("Get the Mastership of each switch from each controller")
295 global mastership_state
Jon Hall94fd0472014-12-08 11:52:42 -0800296 mastership_state = []
297
298 #Assert that each device has a master
299 roles_not_null = main.ONOScli1.roles_not_null()
300 utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
301 onpass="Each device has a master",
302 onfail="Some devices don't have a master assigned")
303
304
Jon Hall73cf9cc2014-11-20 22:28:38 -0800305 ONOS1_mastership = main.ONOScli1.roles()
306 #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
307 #TODO: Make this a meaningful check
308 if "Error" in ONOS1_mastership or not ONOS1_mastership:
309 main.log.report("Error in getting ONOS roles")
310 main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
311 consistent_mastership = main.FALSE
312 else:
313 mastership_state = ONOS1_mastership
314 consistent_mastership = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800315
316
317 main.step("Get the intents from each controller")
318 global intent_state
Jon Hall94fd0472014-12-08 11:52:42 -0800319 intent_state = []
Jon Hall73cf9cc2014-11-20 22:28:38 -0800320 ONOS1_intents = main.ONOScli1.intents( json_format=True )
321 intent_check = main.FALSE
322 if "Error" in ONOS1_intents or not ONOS1_intents:
323 main.log.report("Error in getting ONOS intents")
324 main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
325 else:
326 intent_check = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800327
328
329 main.step("Get the flows from each controller")
330 global flow_state
Jon Hall94fd0472014-12-08 11:52:42 -0800331 flow_state = []
Jon Hall73cf9cc2014-11-20 22:28:38 -0800332 ONOS1_flows = main.ONOScli1.flows( json_format=True )
333 flow_check = main.FALSE
334 if "Error" in ONOS1_flows or not ONOS1_flows:
335 main.log.report("Error in getting ONOS intents")
336 main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
337 else:
338 #TODO: Do a better check, maybe compare flows on switches?
339 flow_state = ONOS1_flows
340 flow_check = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800341
342
343 main.step("Get the OF Table entries")
344 global flows
345 flows=[]
346 for i in range(1,29):
Jon Hall94fd0472014-12-08 11:52:42 -0800347 flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800348
349 #TODO: Compare switch flow tables with ONOS flow tables
350
Jon Hall73cf9cc2014-11-20 22:28:38 -0800351 main.step("Create TestONTopology object")
352 ctrls = []
353 count = 1
354 temp = ()
355 temp = temp + (getattr(main,('ONOS' + str(count))),)
356 temp = temp + ("ONOS"+str(count),)
357 temp = temp + (main.params['CTRL']['ip'+str(count)],)
358 temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
359 ctrls.append(temp)
360 MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
361
362 main.step("Collecting topology information from ONOS")
363 devices = []
364 devices.append( main.ONOScli1.devices() )
365 '''
366 hosts = []
367 hosts.append( main.ONOScli1.hosts() )
368 '''
369 ports = []
370 ports.append( main.ONOScli1.ports() )
371 links = []
372 links.append( main.ONOScli1.links() )
373
374
375 main.step("Comparing ONOS topology to MN")
376 devices_results = main.TRUE
377 ports_results = main.TRUE
378 links_results = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -0800379 for controller in range(num_controllers):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800380 if devices[controller] or not "Error" in devices[controller]:
381 current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
382 else:
383 current_devices_result = main.FALSE
384 utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
385 onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
386 onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
387
388 if ports[controller] or not "Error" in ports[controller]:
389 current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
390 else:
391 current_ports_result = main.FALSE
392 utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
393 onpass="ONOS"+str(int(controller+1))+" ports view is correct",
394 onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
395
396 if links[controller] or not "Error" in links[controller]:
397 current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
398 else:
399 current_links_result = main.FALSE
400 utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
401 onpass="ONOS"+str(int(controller+1))+" links view is correct",
402 onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
403
404 devices_results = devices_results and current_devices_result
405 ports_results = ports_results and current_ports_result
406 links_results = links_results and current_links_result
407
408 topo_result = devices_results and ports_results and links_results
409 utilities.assert_equals(expect=main.TRUE, actual=topo_result,
410 onpass="Topology Check Test successful",
411 onfail="Topology Check Test NOT successful")
412
413 final_assert = main.TRUE
414 final_assert = final_assert and topo_result and flow_check \
Jon Hall94fd0472014-12-08 11:52:42 -0800415 and intent_check and consistent_mastership and roles_not_null
Jon Hall73cf9cc2014-11-20 22:28:38 -0800416 utilities.assert_equals(expect=main.TRUE, actual=final_assert,
417 onpass="State check successful",
418 onfail="State check NOT successful")
419
420
421 def CASE6(self,main) :
422 '''
Jon Hallffb386d2014-11-21 13:43:38 -0800423 The Failure case.
Jon Hall73cf9cc2014-11-20 22:28:38 -0800424 '''
Jon Hallffb386d2014-11-21 13:43:38 -0800425 import time
Jon Hall73cf9cc2014-11-20 22:28:38 -0800426
427 main.log.report("Restart ONOS node")
428 main.log.case("Restart ONOS node")
429 main.ONOSbench.onos_kill(ONOS1_ip)
Jon Hallffb386d2014-11-21 13:43:38 -0800430 start = time.time()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800431
432 main.step("Checking if ONOS is up yet")
Jon Hallffb386d2014-11-21 13:43:38 -0800433 count = 0
Jon Hall94fd0472014-12-08 11:52:42 -0800434 while count < 10:
Jon Hallffb386d2014-11-21 13:43:38 -0800435 onos1_isup = main.ONOSbench.isup(ONOS1_ip)
436 if onos1_isup == main.TRUE:
437 elapsed = time.time() - start
438 break
439 else:
440 count = count + 1
Jon Hall73cf9cc2014-11-20 22:28:38 -0800441
Jon Hallffb386d2014-11-21 13:43:38 -0800442 cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800443
Jon Hall94fd0472014-12-08 11:52:42 -0800444 case_results = main.TRUE and onos1_isup and cli_result
Jon Hall73cf9cc2014-11-20 22:28:38 -0800445 utilities.assert_equals(expect=main.TRUE, actual=case_results,
446 onpass="ONOS restart successful",
447 onfail="ONOS restart NOT successful")
Jon Hall94fd0472014-12-08 11:52:42 -0800448 main.log.info("ESTIMATE: ONOS took %s seconds to restart" % str(elapsed) )
449 time.sleep(5)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800450
451 def CASE7(self,main) :
452 '''
453 Check state after ONOS failure
454 '''
455 import os
456 import json
457 main.case("Running ONOS Constant State Tests")
458
Jon Hall94fd0472014-12-08 11:52:42 -0800459 #Assert that each device has a master
460 roles_not_null = main.ONOScli1.roles_not_null()
461 utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
462 onpass="Each device has a master",
463 onfail="Some devices don't have a master assigned")
464
465
466
Jon Hall73cf9cc2014-11-20 22:28:38 -0800467 main.step("Check if switch roles are consistent across all nodes")
468 ONOS1_mastership = main.ONOScli1.roles()
Jon Hallffb386d2014-11-21 13:43:38 -0800469 #FIXME: Refactor this whole case for single instance
Jon Hall73cf9cc2014-11-20 22:28:38 -0800470 #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
471 if "Error" in ONOS1_mastership or not ONOS1_mastership:
Jon Hallffb386d2014-11-21 13:43:38 -0800472 main.log.report("Error in getting ONOS mastership")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800473 main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
474 consistent_mastership = main.FALSE
475 else:
476 consistent_mastership = main.TRUE
477 main.log.report("Switch roles are consistent across all ONOS nodes")
478 utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
479 onpass="Switch roles are consistent across all ONOS nodes",
480 onfail="ONOS nodes have different views of switch roles")
481
482
483 description2 = "Compare switch roles from before failure"
484 main.step(description2)
485
486 current_json = json.loads(ONOS1_mastership)
487 old_json = json.loads(mastership_state)
488 mastership_check = main.TRUE
489 for i in range(1,29):
490 switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
491
492 current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
493 old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
494 if current == old:
495 mastership_check = mastership_check and main.TRUE
496 else:
497 main.log.warn("Mastership of switch %s changed" % switchDPID)
498 mastership_check = main.FALSE
499 if mastership_check == main.TRUE:
500 main.log.report("Mastership of Switches was not changed")
501 utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
502 onpass="Mastership of Switches was not changed",
503 onfail="Mastership of some switches changed")
504 mastership_check = mastership_check and consistent_mastership
505
506
507
508 main.step("Get the intents and compare across all nodes")
509 ONOS1_intents = main.ONOScli1.intents( json_format=True )
510 intent_check = main.FALSE
511 if "Error" in ONOS1_intents or not ONOS1_intents:
512 main.log.report("Error in getting ONOS intents")
513 main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
514 else:
Jon Hall73cf9cc2014-11-20 22:28:38 -0800515 intent_check = main.TRUE
516 main.log.report("Intents are consistent across all ONOS nodes")
517 utilities.assert_equals(expect = main.TRUE,actual=intent_check,
518 onpass="Intents are consistent across all ONOS nodes",
519 onfail="ONOS nodes have different views of intents")
520
Jon Hall94fd0472014-12-08 11:52:42 -0800521 #NOTE: Hazelcast has no durability, so intents are lost
522 '''
Jon Hall73cf9cc2014-11-20 22:28:38 -0800523 main.step("Compare current intents with intents before the failure")
524 if intent_state == ONOS1_intents:
525 same_intents = main.TRUE
526 main.log.report("Intents are consistent with before failure")
527 #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
528 else:
529 same_intents = main.FALSE
530 utilities.assert_equals(expect = main.TRUE,actual=same_intents,
531 onpass="Intents are consistent with before failure",
532 onfail="The Intents changed during failure")
533 intent_check = intent_check and same_intents
Jon Hall94fd0472014-12-08 11:52:42 -0800534 '''
Jon Hall73cf9cc2014-11-20 22:28:38 -0800535
536
537
538 main.step("Get the OF Table entries and compare to before component failure")
539 Flow_Tables = main.TRUE
540 flows2=[]
541 for i in range(28):
542 main.log.info("Checking flow table on s" + str(i+1))
Jon Hall94fd0472014-12-08 11:52:42 -0800543 tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800544 flows2.append(tmp_flows)
Jon Hall94fd0472014-12-08 11:52:42 -0800545 temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
546 Flow_Tables = Flow_Tables and temp_result
Jon Hall73cf9cc2014-11-20 22:28:38 -0800547 if Flow_Tables == main.FALSE:
548 main.log.info("Differences in flow table for switch: "+str(i+1))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800549 if Flow_Tables == main.TRUE:
550 main.log.report("No changes were found in the flow tables")
551 utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
552 onpass="No changes were found in the flow tables",
553 onfail="Changes were found in the flow tables")
554
Jon Hall669173b2014-12-17 11:36:30 -0800555
556 #Test of LeadershipElection
557
558 leader = ONOS1_ip
559 leader_result = main.TRUE
560 for controller in range(1,num_controllers+1):
561 node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
562 leaderN = node.election_test_leader()
563 #verify leader is ONOS1
564 #NOTE even though we restarted ONOS, it is the only one so onos 1 must be leader
565 if leaderN == leader:
566 #all is well
567 pass
568 elif leaderN == main.FALSE:
569 #error in response
570 main.log.report("Something is wrong with election_test_leader function, check the error logs")
571 leader_result = main.FALSE
572 elif leader != leaderN:
573 leader_result = main.FALSE
574 main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
575 " as the leader of the election app. Leader should be "+str(leader) )
576 if leader_result:
577 main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
578 utilities.assert_equals(expect=main.TRUE, actual=leader_result,
579 onpass="Leadership election passed",
580 onfail="Something went wrong with Leadership election")
581
582
583 result = mastership_check and intent_check and Flow_Tables and roles_not_null\
584 and leader_result
Jon Hall73cf9cc2014-11-20 22:28:38 -0800585 result = int(result)
586 if result == main.TRUE:
587 main.log.report("Constant State Tests Passed")
588 utilities.assert_equals(expect=main.TRUE,actual=result,
589 onpass="Constant State Tests Passed",
590 onfail="Constant state tests failed")
591
592 def CASE8 (self,main):
593 '''
594 Compare topo
595 '''
596 import sys
597 sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
598 from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
599 import json
600 import time
601
602 description ="Compare ONOS Topology view to Mininet topology"
603 main.case(description)
604 main.log.report(description)
605 main.step("Create TestONTopology object")
606 ctrls = []
607 count = 1
608 temp = ()
609 temp = temp + (getattr(main,('ONOS' + str(count))),)
610 temp = temp + ("ONOS"+str(count),)
611 temp = temp + (main.params['CTRL']['ip'+str(count)],)
612 temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
613 ctrls.append(temp)
614 MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
615
Jon Hall73cf9cc2014-11-20 22:28:38 -0800616 main.step("Comparing ONOS topology to MN")
617 devices_results = main.TRUE
618 ports_results = main.TRUE
619 links_results = main.TRUE
620 topo_result = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800621 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -0800622 count = 0
Jon Hall94fd0472014-12-08 11:52:42 -0800623 main.step("Collecting topology information from ONOS")
624 start_time = time.time()
625 while topo_result == main.FALSE and elapsed < 60:
Jon Hallffb386d2014-11-21 13:43:38 -0800626 count = count + 1
Jon Hall94fd0472014-12-08 11:52:42 -0800627 if count > 1:
628 MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
629 cli_start = time.time()
630 devices = []
631 devices.append( main.ONOScli1.devices() )
632 '''
633 hosts = []
634 hosts.append( main.ONOScli1.hosts() )
635 '''
636 ports = []
637 ports.append( main.ONOScli1.ports() )
638 links = []
639 links.append( main.ONOScli1.links() )
640 elapsed = time.time() - start_time
641 print "CLI time: " + str(time.time() - cli_start)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800642
Jon Hall669173b2014-12-17 11:36:30 -0800643 for controller in range(num_controllers):
Jon Hall94fd0472014-12-08 11:52:42 -0800644 if devices[controller] or not "Error" in devices[controller]:
645 current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
646 else:
647 current_devices_result = main.FALSE
648 utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
649 onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
650 onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800651
Jon Hall94fd0472014-12-08 11:52:42 -0800652 if ports[controller] or not "Error" in ports[controller]:
653 current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
654 else:
655 current_ports_result = main.FALSE
656 utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
657 onpass="ONOS"+str(int(controller+1))+" ports view is correct",
658 onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800659
Jon Hall94fd0472014-12-08 11:52:42 -0800660 if links[controller] or not "Error" in links[controller]:
661 current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
662 else:
663 current_links_result = main.FALSE
664 utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
665 onpass="ONOS"+str(int(controller+1))+" links view is correct",
666 onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800667 devices_results = devices_results and current_devices_result
668 ports_results = ports_results and current_ports_result
669 links_results = links_results and current_links_result
Jon Hall94fd0472014-12-08 11:52:42 -0800670 topo_result = devices_results and ports_results and links_results
671
672 topo_result = topo_result and int(count <= 2)
673 main.log.report("Very crass estimate for topology discovery/convergence(note it takes about 1 seconds to read the topology from each ONOS instance): " +\
Jon Hallffb386d2014-11-21 13:43:38 -0800674 str(elapsed) + " seconds, " + str(count) +" tries" )
Jon Hall94fd0472014-12-08 11:52:42 -0800675 if elapsed > 60:
676 main.log.report("Giving up on topology convergence")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800677 utilities.assert_equals(expect=main.TRUE, actual=topo_result,
678 onpass="Topology Check Test successful",
679 onfail="Topology Check Test NOT successful")
680 if topo_result == main.TRUE:
681 main.log.report("ONOS topology view matches Mininet topology")
682
683
684 def CASE9 (self,main):
685 '''
686 Link s3-s28 down
687 '''
688 #NOTE: You should probably run a topology check after this
689
Jon Hall669173b2014-12-17 11:36:30 -0800690 link_sleep = float(main.params['timers']['LinkDiscovery'])
Jon Hall73cf9cc2014-11-20 22:28:38 -0800691
692 description = "Turn off a link to ensure that Link Discovery is working properly"
693 main.log.report(description)
694 main.case(description)
695
696
697 main.step("Kill Link between s3 and s28")
698 Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
699 main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
700 time.sleep(link_sleep)
701 utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
702 onpass="Link down succesful",
703 onfail="Failed to bring link down")
704 #TODO do some sort of check here
705
706 def CASE10 (self,main):
707 '''
708 Link s3-s28 up
709 '''
710 #NOTE: You should probably run a topology check after this
711
Jon Hall669173b2014-12-17 11:36:30 -0800712 link_sleep = float(main.params['timers']['LinkDiscovery'])
Jon Hall73cf9cc2014-11-20 22:28:38 -0800713
714 description = "Restore a link to ensure that Link Discovery is working properly"
715 main.log.report(description)
716 main.case(description)
717
718 main.step("Bring link between s3 and s28 back up")
719 Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
720 main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
721 time.sleep(link_sleep)
722 utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
723 onpass="Link up succesful",
724 onfail="Failed to bring link up")
725 #TODO do some sort of check here
726
727
728 def CASE11 (self, main) :
729 '''
730 Switch Down
731 '''
732 #NOTE: You should probably run a topology check after this
733 import time
734
Jon Hall669173b2014-12-17 11:36:30 -0800735 switch_sleep = float(main.params['timers']['SwitchDiscovery'])
Jon Hall73cf9cc2014-11-20 22:28:38 -0800736
737 description = "Killing a switch to ensure it is discovered correctly"
738 main.log.report(description)
739 main.case(description)
740
741 #TODO: Make this switch parameterizable
742 main.step("Kill s28 ")
743 main.log.report("Deleting s28")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800744 main.Mininet1.del_switch("s28")
745 main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
746 time.sleep(switch_sleep)
Jon Hall94fd0472014-12-08 11:52:42 -0800747 device = main.ONOScli1.get_device(dpid="0028")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800748 #Peek at the deleted switch
Jon Hall94fd0472014-12-08 11:52:42 -0800749 main.log.warn( str(device) )
750 result = main.FALSE
751 if device and device['available'] == False:
752 result = main.TRUE
753 utilities.assert_equals(expect=main.TRUE,actual=result,
754 onpass="Kill switch succesful",
755 onfail="Failed to kill switch?")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800756
757 def CASE12 (self, main) :
758 '''
759 Switch Up
760 '''
761 #NOTE: You should probably run a topology check after this
762 import time
Jon Hall669173b2014-12-17 11:36:30 -0800763
764 switch_sleep = float(main.params['timers']['SwitchDiscovery'])
Jon Hall73cf9cc2014-11-20 22:28:38 -0800765 description = "Adding a switch to ensure it is discovered correctly"
766 main.log.report(description)
767 main.case(description)
768
769 main.step("Add back s28")
770 main.log.report("Adding back s28")
771 main.Mininet1.add_switch("s28", dpid = '0000000000002800')
772 #TODO: New dpid or same? Ask Thomas?
773 main.Mininet1.add_link('s28', 's3')
774 main.Mininet1.add_link('s28', 's6')
775 main.Mininet1.add_link('s28', 'h28')
776 main.Mininet1.assign_sw_controller(sw="28",
777 ip1=ONOS1_ip,port1=ONOS1_port)
778 main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
779 time.sleep(switch_sleep)
Jon Hall94fd0472014-12-08 11:52:42 -0800780 device = main.ONOScli1.get_device(dpid="0028")
781 #Peek at the deleted switch
782 main.log.warn( str(device) )
783 result = main.FALSE
784 if device and device['available'] == True:
785 result = main.TRUE
786 utilities.assert_equals(expect=main.TRUE,actual=result,
787 onpass="add switch succesful",
788 onfail="Failed to add switch?")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800789
790 def CASE13 (self, main) :
791 '''
792 Clean up
793 '''
794 import os
795 import time
Jon Hall669173b2014-12-17 11:36:30 -0800796 #printing colors to terminal
797 colors = {}
798 colors['cyan'] = '\033[96m'
799 colors['purple'] = '\033[95m'
800 colors['blue'] = '\033[94m'
801 colors['green'] = '\033[92m'
802 colors['yellow'] = '\033[93m'
803 colors['red'] = '\033[91m'
804 colors['end'] = '\033[0m'
Jon Hall73cf9cc2014-11-20 22:28:38 -0800805 description = "Test Cleanup"
806 main.log.report(description)
807 main.case(description)
808 main.step("Killing tcpdumps")
809 main.Mininet2.stop_tcpdump()
810
Jon Hall94fd0472014-12-08 11:52:42 -0800811 main.step("Checking ONOS Logs for errors")
Jon Hall669173b2014-12-17 11:36:30 -0800812 print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
Jon Hall94fd0472014-12-08 11:52:42 -0800813 print main.ONOSbench.check_logs(ONOS1_ip)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800814 main.step("Copying MN pcap and ONOS log files to test station")
815 testname = main.TEST
Jon Hall94fd0472014-12-08 11:52:42 -0800816 teststation_user = main.params['TESTONUSER']
817 teststation_IP = main.params['TESTONIP']
Jon Hall73cf9cc2014-11-20 22:28:38 -0800818 #NOTE: MN Pcap file is being saved to ~/packet_captures
819 # scp this file as MN and TestON aren't necessarily the same vm
820 #FIXME: scp
821 #####mn files
822 #TODO: Load these from params
823 #NOTE: must end in /
824 log_folder = "/opt/onos/log/"
825 log_files = ["karaf.log", "karaf.log.1"]
826 #NOTE: must end in /
827 dst_dir = "~/packet_captures/"
828 for f in log_files:
Jon Hall94fd0472014-12-08 11:52:42 -0800829 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
830 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800831 dst_dir + str(testname) + "-ONOS1-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800832 main.ONOSbench.handle.expect("\$")
833 print main.ONOSbench.handle.before
Jon Hall73cf9cc2014-11-20 22:28:38 -0800834
835 #std*.log's
836 #NOTE: must end in /
837 log_folder = "/opt/onos/var/"
838 log_files = ["stderr.log", "stdout.log"]
839 #NOTE: must end in /
840 dst_dir = "~/packet_captures/"
841 for f in log_files:
Jon Hall94fd0472014-12-08 11:52:42 -0800842 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
843 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800844 dst_dir + str(testname) + "-ONOS1-"+f )
845
846
847 #sleep so scp can finish
848 time.sleep(10)
849 main.step("Packing and rotating pcap archives")
850 os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
851
852
853 #TODO: actually check something here
854 utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
855 onpass="Test cleanup successful",
856 onfail="Test cleanup NOT successful")
Jon Hall669173b2014-12-17 11:36:30 -0800857
858 def CASE14 ( self, main ) :
859 '''
860 start election app on all onos nodes
861 '''
862 leader_result = main.TRUE
863 #install app on onos 1
864 main.log.info("Install leadership election app")
865 main.ONOScli1.feature_install("onos-app-election")
866 #wait for election
867 #check for leader
868 leader = main.ONOScli1.election_test_leader()
869 #verify leader is ONOS1
870 if leader == ONOS1_ip:
871 #all is well
872 pass
873 elif leader == None:
874 #No leader elected
875 main.log.report("No leader was elected")
876 leader_result = main.FALSE
877 elif leader == main.FALSE:
878 #error in response
879 #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
880 main.log.report("Something is wrong with election_test_leader function, check the error logs")
881 leader_result = main.FALSE
882 else:
883 #error in response
884 main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
885 leader_result = main.FALSE
886
887
888
889
890 #install on other nodes and check for leader.
891 #Should be onos1 and each app should show the same leader
892 for controller in range(2,num_controllers+1):
893 node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
894 node.feature_install("onos-app-election")
895 leaderN = node.election_test_leader()
896 #verify leader is ONOS1
897 if leaderN == ONOS1_ip:
898 #all is well
899 pass
900 elif leaderN == main.FALSE:
901 #error in response
902 #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
903 main.log.report("Something is wrong with election_test_leader function, check the error logs")
904 leader_result = main.FALSE
905 elif leader != leaderN:
906 leader_result = main.FALSE
907 main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
908 " as the leader of the election app. Leader should be "+str(leader) )
909 if leader_result:
910 main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
911 utilities.assert_equals(expect=main.TRUE, actual=leader_result,
912 onpass="Leadership election passed",
913 onfail="Something went wrong with Leadership election")
914
915 def CASE15 ( self, main ) :
916 '''
917 Check that Leadership Election is still functional
918 '''
919 leader_result = main.TRUE
920 description = "Check that Leadership Election is still functional"
921 main.log.report(description)
922 main.case(description)
923 main.step("Find current leader and withdraw")
924 leader = main.ONOScli1.election_test_leader()
925 #TODO: do some sanity checking on leader before using it
926 withdraw_result = main.FALSE
927 if leader == ONOS1_ip:
928 old_leader = getattr( main, "ONOScli1" )
929 elif leader == None or leader == main.FALSE:
930 main.log.report("Leader for the election app should be an ONOS node,"\
931 +"instead got '"+str(leader)+"'")
932 leader_result = main.FALSE
933 withdraw_result = old_leader.election_test_withdraw()
934
935
936 main.step("Make sure new leader is elected")
937 leader_list = []
938 leaderN = main.ONOScli1.election_test_leader()
939 if leaderN == leader:
940 main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
941 " as leader after they withdrew")
942 leader_result = main.FALSE
943 elif leaderN == main.FALSE:
944 #error in response
945 #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
946 main.log.report("Something is wrong with election_test_leader function, check the error logs")
947 leader_result = main.FALSE
948 elif leaderN == None:
949 main.log.info("There is no leader after the app withdrew from election")
950 if leader_result:
951 main.log.report("Leadership election tests passed(There is no leader after the old leader resigned)")
952 utilities.assert_equals(expect=main.TRUE, actual=leader_result,
953 onpass="Leadership election passed",
954 onfail="Something went wrong with Leadership election")
955
956
957 main.step("Run for election on old leader(just so everyone is in the hat)")
958 run_result = old_leader.election_test_run()
959 leader = main.ONOScli1.election_test_leader()
960 #verify leader is ONOS1
961 if leader == ONOS1_ip:
962 leader_result = main.TRUE
963 else:
964 leader_result = main.FALSE
965 #TODO: assert on run and withdraw results?
966
967 utilities.assert_equals(expect=main.TRUE, actual=leader_result,
968 onpass="Leadership election passed",
969 onfail="ONOS1's election app was not leader after it re-ran for election")
970