Gitiles
Code Review Sign In
gerrit.onosproject.org / OnosSystemTest / 94fd0475be24f2931453f77e27ee761e789bd4ce / . / TestON / tests / HATestMinorityRestart / HATestMinorityRestart.py
blob: 3fbc29df76eb40cacda15949a79e0b3b4cab61ac [file] [log] [blame]
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1'''
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign mastership to controllers
8CASE3: Assign intents
9CASE4: Ping across added host intents
10CASE5: Reading state of ONOS
11CASE6: The Failure case.
12CASE7: Check state after control plane failure
13CASE8: Compare topo
14CASE9: Link s3-s28 down
15CASE10: Link s3-s28 up
16CASE11: Switch down
17CASE12: Switch up
18CASE13: Clean up
19'''
20class HATestMinorityRestart:
21
22 def __init__(self) :
23 self.default = ''
24
25 def CASE1(self,main) :
26 '''
27 CASE1 is to compile ONOS and push it to the test machines
28
29 Startup sequence:
30 git pull
31 mvn clean install
32 onos-package
33 cell <name>
34 onos-verify-cell
35 NOTE: temporary - onos-remove-raft-logs
36 onos-install -f
37 onos-wait-for-start
38 '''
39 import time
40 main.log.report("ONOS HA test: Restart minority of ONOS nodes - initialization")
41 main.case("Setting up test environment")
42
43 # load some vairables from the params file
44 PULL_CODE = False
45 if main.params['Git'] == 'True':
46 PULL_CODE = True
47 cell_name = main.params['ENV']['cellName']
48
49 #set global variables
50 global ONOS1_ip
51 global ONOS1_port
52 global ONOS2_ip
53 global ONOS2_port
54 global ONOS3_ip
55 global ONOS3_port
56 global ONOS4_ip
57 global ONOS4_port
58 global ONOS5_ip
59 global ONOS5_port
60 global ONOS6_ip
61 global ONOS6_port
62 global ONOS7_ip
63 global ONOS7_port
64
65 ONOS1_ip = main.params['CTRL']['ip1']
66 ONOS1_port = main.params['CTRL']['port1']
67 ONOS2_ip = main.params['CTRL']['ip2']
68 ONOS2_port = main.params['CTRL']['port2']
69 ONOS3_ip = main.params['CTRL']['ip3']
70 ONOS3_port = main.params['CTRL']['port3']
71 ONOS4_ip = main.params['CTRL']['ip4']
72 ONOS4_port = main.params['CTRL']['port4']
73 ONOS5_ip = main.params['CTRL']['ip5']
74 ONOS5_port = main.params['CTRL']['port5']
75 ONOS6_ip = main.params['CTRL']['ip6']
76 ONOS6_port = main.params['CTRL']['port6']
77 ONOS7_ip = main.params['CTRL']['ip7']
78 ONOS7_port = main.params['CTRL']['port7']
79
80
81 main.step("Applying cell variable to environment")
82 cell_result = main.ONOSbench.set_cell(cell_name)
83 verify_result = main.ONOSbench.verify_cell()
84
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]85 #FIXME:this is short term fix
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]86 main.log.report("Removing raft logs")
87 main.ONOSbench.onos_remove_raft_logs()
88 main.log.report("Uninstalling ONOS")
89 main.ONOSbench.onos_uninstall(ONOS1_ip)
90 main.ONOSbench.onos_uninstall(ONOS2_ip)
91 main.ONOSbench.onos_uninstall(ONOS3_ip)
92 main.ONOSbench.onos_uninstall(ONOS4_ip)
93 main.ONOSbench.onos_uninstall(ONOS5_ip)
94 main.ONOSbench.onos_uninstall(ONOS6_ip)
95 main.ONOSbench.onos_uninstall(ONOS7_ip)
96
97 clean_install_result = main.TRUE
98 git_pull_result = main.TRUE
99
100 main.step("Compiling the latest version of ONOS")
101 if PULL_CODE:
102 main.step("Git checkout and pull master")
103 main.ONOSbench.git_checkout("master")
104 git_pull_result = main.ONOSbench.git_pull()
105
106 main.step("Using mvn clean & install")
107 clean_install_result = main.TRUE
108 if git_pull_result == main.TRUE:
109 clean_install_result = main.ONOSbench.clean_install()
110 else:
111 main.log.warn("Did not pull new code so skipping mvn "+ \
112 "clean install")
113 main.ONOSbench.get_version(report=True)
114
115 main.step("Creating ONOS package")
116 package_result = main.ONOSbench.onos_package()
117
118 main.step("Installing ONOS package")
119 onos1_install_result = main.ONOSbench.onos_install(options="-f",
120 node=ONOS1_ip)
121 onos2_install_result = main.ONOSbench.onos_install(options="-f",
122 node=ONOS2_ip)
123 onos3_install_result = main.ONOSbench.onos_install(options="-f",
124 node=ONOS3_ip)
125 onos4_install_result = main.ONOSbench.onos_install(options="-f",
126 node=ONOS4_ip)
127 onos5_install_result = main.ONOSbench.onos_install(options="-f",
128 node=ONOS5_ip)
129 onos6_install_result = main.ONOSbench.onos_install(options="-f",
130 node=ONOS6_ip)
131 onos7_install_result = main.ONOSbench.onos_install(options="-f",
132 node=ONOS7_ip)
133 onos_install_result = onos1_install_result and onos2_install_result\
134 and onos3_install_result and onos4_install_result\
135 and onos5_install_result and onos6_install_result\
136 and onos7_install_result
137 '''
138 #FIXME: work around until onos is less fragile
139 main.ONOSbench.handle.sendline("onos-cluster-install")
140 print main.ONOSbench.handle.expect("\$")
141 onos_install_result = main.TRUE
142 '''
143
144
145 main.step("Checking if ONOS is up yet")
146 #TODO: Refactor
147 # check bundle:list?
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]148 for i in range(2):
149 onos1_isup = main.ONOSbench.isup(ONOS1_ip)
150 if not onos1_isup:
151 main.log.report("ONOS1 didn't start!")
152 onos2_isup = main.ONOSbench.isup(ONOS2_ip)
153 if not onos2_isup:
154 main.log.report("ONOS2 didn't start!")
155 onos3_isup = main.ONOSbench.isup(ONOS3_ip)
156 if not onos3_isup:
157 main.log.report("ONOS3 didn't start!")
158 onos4_isup = main.ONOSbench.isup(ONOS4_ip)
159 if not onos4_isup:
160 main.log.report("ONOS4 didn't start!")
161 onos5_isup = main.ONOSbench.isup(ONOS5_ip)
162 if not onos5_isup:
163 main.log.report("ONOS5 didn't start!")
164 onos6_isup = main.ONOSbench.isup(ONOS6_ip)
165 if not onos6_isup:
166 main.log.report("ONOS6 didn't start!")
167 onos7_isup = main.ONOSbench.isup(ONOS7_ip)
168 if not onos7_isup:
169 main.log.report("ONOS7 didn't start!")
170 onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
171 and onos4_isup and onos5_isup and onos6_isup and onos7_isup
172 if onos_isup_result == main.TRUE:
173 break
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]174 # TODO: if it becomes an issue, we can retry this step a few times
175
176
177 cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
178 cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
179 cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
180 cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
181 cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
182 cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
183 cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
184 cli_results = cli_result1 and cli_result2 and cli_result3 and\
185 cli_result4 and cli_result5 and cli_result6 and cli_result7
186
187 main.step("Start Packet Capture MN")
188 main.Mininet2.start_tcpdump(
189 str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
190 intf = main.params['MNtcpdump']['intf'],
191 port = main.params['MNtcpdump']['port'])
192
193
194 case1_result = (clean_install_result and package_result and
195 cell_result and verify_result and onos_install_result and
196 onos_isup_result and cli_results)
197
198 utilities.assert_equals(expect=main.TRUE, actual=case1_result,
199 onpass="Test startup successful",
200 onfail="Test startup NOT successful")
201
202
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]203 if case1_result==main.FALSE:
204 main.cleanup()
205 main.exit()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]206
207 def CASE2(self,main) :
208 '''
209 Assign mastership to controllers
210 '''
211 import time
212 import json
213 import re
214
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]215 main.log.report("Assigning switches to controllers")
216 main.case("Assigning Controllers")
217 main.step("Assign switches to controllers")
218
219 for i in range (1,29):
220 main.Mininet1.assign_sw_controller(sw=str(i),count=7,
221 ip1=ONOS1_ip,port1=ONOS1_port,
222 ip2=ONOS2_ip,port2=ONOS2_port,
223 ip3=ONOS3_ip,port3=ONOS3_port,
224 ip4=ONOS4_ip,port4=ONOS4_port,
225 ip5=ONOS5_ip,port5=ONOS5_port,
226 ip6=ONOS6_ip,port6=ONOS6_port,
227 ip7=ONOS7_ip,port7=ONOS7_port)
228
229 mastership_check = main.TRUE
230 for i in range (1,29):
231 response = main.Mininet1.get_sw_controller("s"+str(i))
Jon Hallffb386d2014-11-21 13:43:38 -0800[diff] [blame]232 try:
233 main.log.info(str(response))
234 except:
235 main.log.info(repr(response))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]236 if re.search("tcp:"+ONOS1_ip,response)\
237 and re.search("tcp:"+ONOS2_ip,response)\
238 and re.search("tcp:"+ONOS3_ip,response)\
239 and re.search("tcp:"+ONOS4_ip,response)\
240 and re.search("tcp:"+ONOS5_ip,response)\
241 and re.search("tcp:"+ONOS6_ip,response)\
242 and re.search("tcp:"+ONOS7_ip,response):
243 mastership_check = mastership_check and main.TRUE
244 else:
245 mastership_check = main.FALSE
246 if mastership_check == main.TRUE:
247 main.log.report("Switch mastership assigned correctly")
248 utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
249 onpass="Switch mastership assigned correctly",
250 onfail="Switches not assigned correctly to controllers")
251
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]252 #Manually assign mastership to the controller we want
253 role_call = main.TRUE
254 role_check = main.TRUE
255
256 device_id = main.ONOScli1.get_device("1000")['id']
257 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
258 if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
259 role_check = role_check and main.TRUE
260 else:
261 role_check = role_check and main.FALSE
262
263 device_id = main.ONOScli1.get_device("2800")['id']
264 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
265 if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
266 role_check = role_check and main.TRUE
267 else:
268 role_check = role_check and main.FALSE
269
270 device_id = main.ONOScli1.get_device("2000")['id']
271 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
272 if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
273 role_check = role_check and main.TRUE
274 else:
275 role_check = role_check and main.FALSE
276
277 device_id = main.ONOScli1.get_device("3000")['id']
278 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
279 if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
280 role_check = role_check and main.TRUE
281 else:
282 role_check = role_check and main.FALSE
283
284 device_id = main.ONOScli1.get_device("5000")['id']
285 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
286 if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
287 role_check = role_check and main.TRUE
288 else:
289 role_check = role_check and main.FALSE
290
291 device_id = main.ONOScli1.get_device("6000")['id']
292 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
293 if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
294 role_check = role_check and main.TRUE
295 else:
296 role_check = role_check and main.FALSE
297
298 device_id = main.ONOScli1.get_device("3004")['id']
299 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
300 if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
301 role_check = role_check and main.TRUE
302 else:
303 role_check = role_check and main.FALSE
304
305 device_id = main.ONOScli1.get_device("3008")['id']
306 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
307 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
308 role_check = role_check and main.TRUE
309 else:
310 role_check = role_check and main.FALSE
311
312 device_id = main.ONOScli1.get_device("3009")['id']
313 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
314 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
315 role_check = role_check and main.TRUE
316 else:
317 role_check = role_check and main.FALSE
318
319 device_id = main.ONOScli1.get_device("3010")['id']
320 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
321 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
322 role_check = role_check and main.TRUE
323 else:
324 role_check = role_check and main.FALSE
325
326 device_id = main.ONOScli1.get_device("3011")['id']
327 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
328 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
329 role_check = role_check and main.TRUE
330 else:
331 role_check = role_check and main.FALSE
332
333 device_id = main.ONOScli1.get_device("3012")['id']
334 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
335 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
336 role_check = role_check and main.TRUE
337 else:
338 role_check = role_check and main.FALSE
339
340 device_id = main.ONOScli1.get_device("3013")['id']
341 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
342 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
343 role_check = role_check and main.TRUE
344 else:
345 role_check = role_check and main.FALSE
346
347 device_id = main.ONOScli1.get_device("3014")['id']
348 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
349 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
350 role_check = role_check and main.TRUE
351 else:
352 role_check = role_check and main.FALSE
353
354 device_id = main.ONOScli1.get_device("3015")['id']
355 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
356 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
357 role_check = role_check and main.TRUE
358 else:
359 role_check = role_check and main.FALSE
360
361 device_id = main.ONOScli1.get_device("3016")['id']
362 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
363 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
364 role_check = role_check and main.TRUE
365 else:
366 role_check = role_check and main.FALSE
367
368 device_id = main.ONOScli1.get_device("3017")['id']
369 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
370 if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
371 role_check = role_check and main.TRUE
372 else:
373 role_check = role_check and main.FALSE
374
375 device_id = main.ONOScli1.get_device("6007")['id']
376 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
377 if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
378 role_check = role_check and main.TRUE
379 else:
380 role_check = role_check and main.FALSE
381
382 device_id = main.ONOScli1.get_device("6018")['id']
383 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
384 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
385 role_check = role_check and main.TRUE
386 else:
387 role_check = role_check and main.FALSE
388
389 device_id = main.ONOScli1.get_device("6019")['id']
390 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
391 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
392 role_check = role_check and main.TRUE
393 else:
394 role_check = role_check and main.FALSE
395
396 device_id = main.ONOScli1.get_device("6020")['id']
397 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
398 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
399 role_check = role_check and main.TRUE
400 else:
401 role_check = role_check and main.FALSE
402
403 device_id = main.ONOScli1.get_device("6021")['id']
404 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
405 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
406 role_check = role_check and main.TRUE
407 else:
408 role_check = role_check and main.FALSE
409
410 device_id = main.ONOScli1.get_device("6022")['id']
411 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
412 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
413 role_check = role_check and main.TRUE
414 else:
415 role_check = role_check and main.FALSE
416
417 device_id = main.ONOScli1.get_device("6023")['id']
418 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
419 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
420 role_check = role_check and main.TRUE
421 else:
422 role_check = role_check and main.FALSE
423
424 device_id = main.ONOScli1.get_device("6024")['id']
425 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
426 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
427 role_check = role_check and main.TRUE
428 else:
429 role_check = role_check and main.FALSE
430
431 device_id = main.ONOScli1.get_device("6025")['id']
432 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
433 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
434 role_check = role_check and main.TRUE
435 else:
436 role_check = role_check and main.FALSE
437
438 device_id = main.ONOScli1.get_device("6026")['id']
439 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
440 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
441 role_check = role_check and main.TRUE
442 else:
443 role_check = role_check and main.FALSE
444
445 device_id = main.ONOScli1.get_device("6027")['id']
446 role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
447 if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
448 role_check = role_check and main.TRUE
449 else:
450 role_check = role_check and main.FALSE
451
452 utilities.assert_equals(expect = main.TRUE,actual=role_call,
453 onpass="Re-assigned switch mastership to designated controller",
454 onfail="Something wrong with device_role calls")
455
456 utilities.assert_equals(expect = main.TRUE,actual=role_check,
457 onpass="Switches were successfully reassigned to designated controller",
458 onfail="Switches were not successfully reassigned")
459 mastership_check = mastership_check and role_call and role_check
460 utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
461 onpass="Switch mastership correctly assigned",
462 onfail="Error in (re)assigning switch mastership")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]463
464
465 def CASE3(self,main) :
466 """
467 Assign intents
468
469 """
470 import time
471 import json
472 import re
473 main.log.report("Adding host intents")
474 main.case("Adding host Intents")
475
476 main.step("Discovering Hosts( Via pingall for now)")
477 #FIXME: Once we have a host discovery mechanism, use that instead
478
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]479 #install onos-app-fwd
480 main.log.info("Install reactive forwarding app")
481 main.ONOScli1.feature_install("onos-app-fwd")
482 main.ONOScli2.feature_install("onos-app-fwd")
483 main.ONOScli3.feature_install("onos-app-fwd")
484 main.ONOScli4.feature_install("onos-app-fwd")
485 main.ONOScli5.feature_install("onos-app-fwd")
486 main.ONOScli6.feature_install("onos-app-fwd")
487 main.ONOScli7.feature_install("onos-app-fwd")
488
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]489 #REACTIVE FWD test
490 ping_result = main.FALSE
491 time1 = time.time()
492 ping_result = main.Mininet1.pingall()
493 time2 = time.time()
494 main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
495
496 #uninstall onos-app-fwd
497 main.log.info("Uninstall reactive forwarding app")
498 main.ONOScli1.feature_uninstall("onos-app-fwd")
499 main.ONOScli2.feature_uninstall("onos-app-fwd")
500 main.ONOScli3.feature_uninstall("onos-app-fwd")
501 main.ONOScli4.feature_uninstall("onos-app-fwd")
502 main.ONOScli5.feature_uninstall("onos-app-fwd")
503 main.ONOScli6.feature_uninstall("onos-app-fwd")
504 main.ONOScli7.feature_uninstall("onos-app-fwd")
505
506 main.step("Add host intents")
507 #TODO: move the host numbers to params
508 import json
509 intents_json= json.loads(main.ONOScli1.hosts())
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]510 intent_add_result = True
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]511 for i in range(8,18):
512 main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
513 host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
514 host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
515 #NOTE: get host can return None
516 #TODO: handle this
517 host1_id = main.ONOScli1.get_host(host1)['id']
518 host2_id = main.ONOScli1.get_host(host2)['id']
519 tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]520 intent_add_result = bool(intent_add_result and tmp_result)
521 utilities.assert_equals(expect=True, actual=intent_add_result,
522 onpass="Switch mastership correctly assigned",
523 onfail="Error in (re)assigning switch mastership")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]524 #TODO Check if intents all exist in datastore
525 #NOTE: Do we need to print this once the test is working?
526 #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
527 # sort_keys=True, indent=4, separators=(',', ': ') ) )
528
529 def CASE4(self,main) :
530 """
531 Ping across added host intents
532 """
533 description = " Ping across added host intents"
534 main.log.report(description)
535 main.case(description)
536 Ping_Result = main.TRUE
537 for i in range(8,18):
538 ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
539 Ping_Result = Ping_Result and ping
540 if ping==main.FALSE:
541 main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
542 elif ping==main.TRUE:
543 main.log.info("Ping test passed!")
544 Ping_Result = main.TRUE
545 if Ping_Result==main.FALSE:
546 main.log.report("Intents have not been installed correctly, pings failed.")
547 if Ping_Result==main.TRUE:
548 main.log.report("Intents have been installed correctly and verified by pings")
549 utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
550 onpass="Intents have been installed correctly and pings work",
551 onfail ="Intents have not been installed correctly, pings failed." )
552
553 def CASE5(self,main) :
554 '''
555 Reading state of ONOS
556 '''
557 import time
558 import json
559 from subprocess import Popen, PIPE
560 from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
561
562 main.log.report("Setting up and gathering data for current state")
563 main.case("Setting up and gathering data for current state")
564 #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
565 #We can then compare them with eachother and also with past states
566
567 main.step("Get the Mastership of each switch from each controller")
568 global mastership_state
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]569 mastership_state = []
570
571 #Assert that each device has a master
572 ONOS1_master_not_null = main.ONOScli1.roles_not_null()
573 ONOS2_master_not_null = main.ONOScli2.roles_not_null()
574 ONOS3_master_not_null = main.ONOScli3.roles_not_null()
575 ONOS4_master_not_null = main.ONOScli4.roles_not_null()
576 ONOS5_master_not_null = main.ONOScli5.roles_not_null()
577 ONOS6_master_not_null = main.ONOScli6.roles_not_null()
578 ONOS7_master_not_null = main.ONOScli7.roles_not_null()
579 roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
580 ONOS3_master_not_null and ONOS4_master_not_null and\
581 ONOS5_master_not_null and ONOS6_master_not_null and\
582 ONOS7_master_not_null
583 utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
584 onpass="Each device has a master",
585 onfail="Some devices don't have a master assigned")
586
587
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]588 ONOS1_mastership = main.ONOScli1.roles()
589 ONOS2_mastership = main.ONOScli2.roles()
590 ONOS3_mastership = main.ONOScli3.roles()
591 ONOS4_mastership = main.ONOScli4.roles()
592 ONOS5_mastership = main.ONOScli5.roles()
593 ONOS6_mastership = main.ONOScli6.roles()
594 ONOS7_mastership = main.ONOScli7.roles()
595 #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
596 if "Error" in ONOS1_mastership or not ONOS1_mastership\
597 or "Error" in ONOS2_mastership or not ONOS2_mastership\
598 or "Error" in ONOS3_mastership or not ONOS3_mastership\
599 or "Error" in ONOS4_mastership or not ONOS4_mastership\
600 or "Error" in ONOS5_mastership or not ONOS5_mastership\
601 or "Error" in ONOS6_mastership or not ONOS6_mastership\
602 or "Error" in ONOS7_mastership or not ONOS7_mastership:
603 main.log.report("Error in getting ONOS roles")
604 main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
605 main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
606 main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
607 main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
608 main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
609 main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
610 main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
611 consistent_mastership = main.FALSE
612 elif ONOS1_mastership == ONOS2_mastership\
613 and ONOS1_mastership == ONOS3_mastership\
614 and ONOS1_mastership == ONOS4_mastership\
615 and ONOS1_mastership == ONOS5_mastership\
616 and ONOS1_mastership == ONOS6_mastership\
617 and ONOS1_mastership == ONOS7_mastership:
618 mastership_state = ONOS1_mastership
619 consistent_mastership = main.TRUE
620 main.log.report("Switch roles are consistent across all ONOS nodes")
621 else:
622 main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
623 sort_keys=True, indent=4, separators=(',', ': ')))
624 main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
625 sort_keys=True, indent=4, separators=(',', ': ')))
626 main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
627 sort_keys=True, indent=4, separators=(',', ': ')))
628 main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
629 sort_keys=True, indent=4, separators=(',', ': ')))
630 main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
631 sort_keys=True, indent=4, separators=(',', ': ')))
632 main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
633 sort_keys=True, indent=4, separators=(',', ': ')))
634 main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
635 sort_keys=True, indent=4, separators=(',', ': ')))
636 consistent_mastership = main.FALSE
637 utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
638 onpass="Switch roles are consistent across all ONOS nodes",
639 onfail="ONOS nodes have different views of switch roles")
640
641
642 main.step("Get the intents from each controller")
643 global intent_state
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]644 intent_state = []
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]645 ONOS1_intents = main.ONOScli1.intents( json_format=True )
646 ONOS2_intents = main.ONOScli2.intents( json_format=True )
647 ONOS3_intents = main.ONOScli3.intents( json_format=True )
648 ONOS4_intents = main.ONOScli4.intents( json_format=True )
649 ONOS5_intents = main.ONOScli5.intents( json_format=True )
650 ONOS6_intents = main.ONOScli6.intents( json_format=True )
651 ONOS7_intents = main.ONOScli7.intents( json_format=True )
652 intent_check = main.FALSE
653 if "Error" in ONOS1_intents or not ONOS1_intents\
654 or "Error" in ONOS2_intents or not ONOS2_intents\
655 or "Error" in ONOS3_intents or not ONOS3_intents\
656 or "Error" in ONOS4_intents or not ONOS4_intents\
657 or "Error" in ONOS5_intents or not ONOS5_intents\
658 or "Error" in ONOS6_intents or not ONOS6_intents\
659 or "Error" in ONOS7_intents or not ONOS7_intents:
660 main.log.report("Error in getting ONOS intents")
661 main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
662 main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
663 main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
664 main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
665 main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
666 main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
667 main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
668 elif ONOS1_intents == ONOS2_intents\
669 and ONOS1_intents == ONOS3_intents\
670 and ONOS1_intents == ONOS4_intents\
671 and ONOS1_intents == ONOS5_intents\
672 and ONOS1_intents == ONOS6_intents\
673 and ONOS1_intents == ONOS7_intents:
674 intent_state = ONOS1_intents
675 intent_check = main.TRUE
676 main.log.report("Intents are consistent across all ONOS nodes")
677 else:
678 main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
679 sort_keys=True, indent=4, separators=(',', ': ')))
680 main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
681 sort_keys=True, indent=4, separators=(',', ': ')))
682 main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
683 sort_keys=True, indent=4, separators=(',', ': ')))
684 main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
685 sort_keys=True, indent=4, separators=(',', ': ')))
686 main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
687 sort_keys=True, indent=4, separators=(',', ': ')))
688 main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
689 sort_keys=True, indent=4, separators=(',', ': ')))
690 main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
691 sort_keys=True, indent=4, separators=(',', ': ')))
692 utilities.assert_equals(expect = main.TRUE,actual=intent_check,
693 onpass="Intents are consistent across all ONOS nodes",
694 onfail="ONOS nodes have different views of intents")
695
696
697 main.step("Get the flows from each controller")
698 global flow_state
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]699 flow_state = []
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]700 ONOS1_flows = main.ONOScli1.flows( json_format=True )
701 ONOS2_flows = main.ONOScli2.flows( json_format=True )
702 ONOS3_flows = main.ONOScli3.flows( json_format=True )
703 ONOS4_flows = main.ONOScli4.flows( json_format=True )
704 ONOS5_flows = main.ONOScli5.flows( json_format=True )
705 ONOS6_flows = main.ONOScli6.flows( json_format=True )
706 ONOS7_flows = main.ONOScli7.flows( json_format=True )
707 flow_check = main.FALSE
708 if "Error" in ONOS1_flows or not ONOS1_flows\
709 or "Error" in ONOS2_flows or not ONOS2_flows\
710 or "Error" in ONOS3_flows or not ONOS3_flows\
711 or "Error" in ONOS4_flows or not ONOS4_flows\
712 or "Error" in ONOS5_flows or not ONOS5_flows\
713 or "Error" in ONOS6_flows or not ONOS6_flows\
714 or "Error" in ONOS7_flows or not ONOS7_flows:
715 main.log.report("Error in getting ONOS intents")
716 main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
717 main.log.warn("ONOS2 flows repsponse: "+ ONOS2_flows)
718 main.log.warn("ONOS3 flows repsponse: "+ ONOS3_flows)
719 main.log.warn("ONOS4 flows repsponse: "+ ONOS4_flows)
720 main.log.warn("ONOS5 flows repsponse: "+ ONOS5_flows)
721 main.log.warn("ONOS6 flows repsponse: "+ ONOS6_flows)
722 main.log.warn("ONOS7 flows repsponse: "+ ONOS7_flows)
723 elif len(json.loads(ONOS1_flows)) == len(json.loads(ONOS2_flows))\
724 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS3_flows))\
725 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS4_flows))\
726 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS5_flows))\
727 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS6_flows))\
728 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS7_flows)):
729 #TODO: Do a better check, maybe compare flows on switches?
730 flow_state = ONOS1_flows
731 flow_check = main.TRUE
732 main.log.report("Flow count is consistent across all ONOS nodes")
733 else:
734 main.log.warn("ONOS1 flows: "+ json.dumps(json.loads(ONOS1_flows),
735 sort_keys=True, indent=4, separators=(',', ': ')))
736 main.log.warn("ONOS2 flows: "+ json.dumps(json.loads(ONOS2_flows),
737 sort_keys=True, indent=4, separators=(',', ': ')))
738 main.log.warn("ONOS3 flows: "+ json.dumps(json.loads(ONOS3_flows),
739 sort_keys=True, indent=4, separators=(',', ': ')))
740 main.log.warn("ONOS4 flows: "+ json.dumps(json.loads(ONOS4_flows),
741 sort_keys=True, indent=4, separators=(',', ': ')))
742 main.log.warn("ONOS5 flows: "+ json.dumps(json.loads(ONOS5_flows),
743 sort_keys=True, indent=4, separators=(',', ': ')))
744 main.log.warn("ONOS6 flows: "+ json.dumps(json.loads(ONOS6_flows),
745 sort_keys=True, indent=4, separators=(',', ': ')))
746 main.log.warn("ONOS7 flows: "+ json.dumps(json.loads(ONOS7_flows),
747 sort_keys=True, indent=4, separators=(',', ': ')))
748 utilities.assert_equals(expect = main.TRUE,actual=flow_check,
749 onpass="The flow count is consistent across all ONOS nodes",
750 onfail="ONOS nodes have different flow counts")
751
752
753 main.step("Get the OF Table entries")
754 global flows
755 flows=[]
756 for i in range(1,29):
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]757 flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]758
759 #TODO: Compare switch flow tables with ONOS flow tables
760
761 main.step("Start continuous pings")
762 main.Mininet2.pingLong(src=main.params['PING']['source1'],
763 target=main.params['PING']['target1'],pingTime=500)
764 main.Mininet2.pingLong(src=main.params['PING']['source2'],
765 target=main.params['PING']['target2'],pingTime=500)
766 main.Mininet2.pingLong(src=main.params['PING']['source3'],
767 target=main.params['PING']['target3'],pingTime=500)
768 main.Mininet2.pingLong(src=main.params['PING']['source4'],
769 target=main.params['PING']['target4'],pingTime=500)
770 main.Mininet2.pingLong(src=main.params['PING']['source5'],
771 target=main.params['PING']['target5'],pingTime=500)
772 main.Mininet2.pingLong(src=main.params['PING']['source6'],
773 target=main.params['PING']['target6'],pingTime=500)
774 main.Mininet2.pingLong(src=main.params['PING']['source7'],
775 target=main.params['PING']['target7'],pingTime=500)
776 main.Mininet2.pingLong(src=main.params['PING']['source8'],
777 target=main.params['PING']['target8'],pingTime=500)
778 main.Mininet2.pingLong(src=main.params['PING']['source9'],
779 target=main.params['PING']['target9'],pingTime=500)
780 main.Mininet2.pingLong(src=main.params['PING']['source10'],
781 target=main.params['PING']['target10'],pingTime=500)
782
783 main.step("Create TestONTopology object")
784 ctrls = []
785 count = 1
786 while True:
787 temp = ()
788 if ('ip' + str(count)) in main.params['CTRL']:
789 temp = temp + (getattr(main,('ONOS' + str(count))),)
790 temp = temp + ("ONOS"+str(count),)
791 temp = temp + (main.params['CTRL']['ip'+str(count)],)
792 temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
793 ctrls.append(temp)
794 count = count + 1
795 else:
796 break
797 MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
798
799 main.step("Collecting topology information from ONOS")
800 devices = []
801 devices.append( main.ONOScli1.devices() )
802 devices.append( main.ONOScli2.devices() )
803 devices.append( main.ONOScli3.devices() )
804 devices.append( main.ONOScli4.devices() )
805 devices.append( main.ONOScli5.devices() )
806 devices.append( main.ONOScli6.devices() )
807 devices.append( main.ONOScli7.devices() )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]808 hosts = []
809 hosts.append( main.ONOScli1.hosts() )
810 hosts.append( main.ONOScli2.hosts() )
811 hosts.append( main.ONOScli3.hosts() )
812 hosts.append( main.ONOScli4.hosts() )
813 hosts.append( main.ONOScli5.hosts() )
814 hosts.append( main.ONOScli6.hosts() )
815 hosts.append( main.ONOScli7.hosts() )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]816 ports = []
817 ports.append( main.ONOScli1.ports() )
818 ports.append( main.ONOScli2.ports() )
819 ports.append( main.ONOScli3.ports() )
820 ports.append( main.ONOScli4.ports() )
821 ports.append( main.ONOScli5.ports() )
822 ports.append( main.ONOScli6.ports() )
823 ports.append( main.ONOScli7.ports() )
824 links = []
825 links.append( main.ONOScli1.links() )
826 links.append( main.ONOScli2.links() )
827 links.append( main.ONOScli3.links() )
828 links.append( main.ONOScli4.links() )
829 links.append( main.ONOScli5.links() )
830 links.append( main.ONOScli6.links() )
831 links.append( main.ONOScli7.links() )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]832 clusters = []
833 clusters.append( main.ONOScli1.clusters() )
834 clusters.append( main.ONOScli2.clusters() )
835 clusters.append( main.ONOScli3.clusters() )
836 clusters.append( main.ONOScli4.clusters() )
837 clusters.append( main.ONOScli5.clusters() )
838 clusters.append( main.ONOScli6.clusters() )
839 clusters.append( main.ONOScli7.clusters() )
840 paths = []
841 temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
842 paths.append( temp_topo.get('paths', False) )
843 temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
844 paths.append( temp_topo.get('paths', False) )
845 temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
846 paths.append( temp_topo.get('paths', False) )
847 temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
848 paths.append( temp_topo.get('paths', False) )
849 temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
850 paths.append( temp_topo.get('paths', False) )
851 temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
852 paths.append( temp_topo.get('paths', False) )
853 temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
854 paths.append( temp_topo.get('paths', False) )
855
856 #Compare json objects for hosts, dataplane clusters and paths
857
858 #hosts
859 consistent_hosts_result = main.TRUE
860 for controller in range( len( hosts ) ):
861 if not "Error" in hosts[controller]:
862 if hosts[controller] == hosts[0]:
863 continue
864 else:#hosts not consistent
865 main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
866 main.log.warn( repr( hosts[controller] ) )
867 consistent_hosts_result = main.FALSE
868
869 else:
870 main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
871 consistent_hosts_result = main.FALSE
872 main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
873 utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
874 onpass="Hosts view is consistent across all ONOS nodes",
875 onfail="ONOS nodes have different views of hosts")
876
877 #Strongly connected clusters of devices
878 consistent_clusters_result = main.TRUE
879 for controller in range( len( clusters ) ):
880 if not "Error" in clusters[controller]:
881 if clusters[controller] == clusters[0]:
882 continue
883 else:#clusters not consistent
884 main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
885 consistent_clusters_result = main.FALSE
886
887 else:
888 main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
889 consistent_clusters_result = main.FALSE
890 main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
891 utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
892 onpass="Clusters view is consistent across all ONOS nodes",
893 onfail="ONOS nodes have different views of clusters")
894 num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
895 utilities.assert_equals(expect = 1, actual = num_clusters,
896 onpass="ONOS shows 1 SCC",
897 onfail="ONOS shows "+str(num_clusters) +" SCCs")
898
899
900 #paths
901 consistent_paths_result = main.TRUE
902 for controller in range( len( paths ) ):
903 if not "Error" in paths[controller]:
904 if paths[controller] == paths[0]:
905 continue
906 else:#paths not consistent
907 main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
908 consistent_paths_result = main.FALSE
909
910 else:
911 main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
912 consistent_paths_result = main.FALSE
913 main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
914 utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
915 onpass="Paths count is consistent across all ONOS nodes",
916 onfail="ONOS nodes have different counts of paths")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]917
918
919 main.step("Comparing ONOS topology to MN")
920 devices_results = main.TRUE
921 ports_results = main.TRUE
922 links_results = main.TRUE
923 for controller in range(7): #TODO parameterize the number of controllers
924 if devices[controller] or not "Error" in devices[controller]:
925 current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
926 else:
927 current_devices_result = main.FALSE
928 utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
929 onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
930 onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
931
932 if ports[controller] or not "Error" in ports[controller]:
933 current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
934 else:
935 current_ports_result = main.FALSE
936 utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
937 onpass="ONOS"+str(int(controller+1))+" ports view is correct",
938 onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
939
940 if links[controller] or not "Error" in links[controller]:
941 current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
942 else:
943 current_links_result = main.FALSE
944 utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
945 onpass="ONOS"+str(int(controller+1))+" links view is correct",
946 onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
947
948 devices_results = devices_results and current_devices_result
949 ports_results = ports_results and current_ports_result
950 links_results = links_results and current_links_result
951
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]952 topo_result = devices_results and ports_results and links_results\
953 and consistent_hosts_result and consistent_clusters_result\
954 and consistent_paths_result
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]955 utilities.assert_equals(expect=main.TRUE, actual=topo_result,
956 onpass="Topology Check Test successful",
957 onfail="Topology Check Test NOT successful")
958
959 final_assert = main.TRUE
960 final_assert = final_assert and topo_result and flow_check \
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]961 and intent_check and consistent_mastership and roles_not_null
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]962 utilities.assert_equals(expect=main.TRUE, actual=final_assert,
963 onpass="State check successful",
964 onfail="State check NOT successful")
965
966
967 def CASE6(self,main) :
968 '''
969 The Failure case.
970 '''
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]971 import time
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]972 main.log.report("Killing 3 ONOS nodes")
973 main.log.case("Restart minority of ONOS nodes")
974 #TODO: Randomize these nodes
975 main.ONOSbench.onos_kill(ONOS1_ip)
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]976 time.sleep(10)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]977 main.ONOSbench.onos_kill(ONOS2_ip)
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]978 time.sleep(10)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]979 main.ONOSbench.onos_kill(ONOS3_ip)
980
981 main.step("Checking if ONOS is up yet")
Jon Hallffb386d2014-11-21 13:43:38 -0800[diff] [blame]982 count = 0
983 onos_isup_result = main.FALSE
984 while onos_isup_result == main.FALSE and count < 10:
985 onos1_isup = main.ONOSbench.isup(ONOS1_ip)
986 onos2_isup = main.ONOSbench.isup(ONOS2_ip)
987 onos3_isup = main.ONOSbench.isup(ONOS3_ip)
988 onos_isup_result = onos1_isup and onos2_isup and onos3_isup
989 count = count + 1
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]990 # TODO: if it becomes an issue, we can retry this step a few times
991
992
993 cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
994 cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
995 cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
996 cli_results = cli_result1 and cli_result2 and cli_result3
997
998 case_results = main.TRUE and onos_isup_result and cli_results
999 utilities.assert_equals(expect=main.TRUE, actual=case_results,
1000 onpass="ONOS restart successful",
1001 onfail="ONOS restart NOT successful")
1002
1003
1004 def CASE7(self,main) :
1005 '''
1006 Check state after ONOS failure
1007 '''
1008 import os
1009 import json
1010 main.case("Running ONOS Constant State Tests")
1011
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1012 #Assert that each device has a master
1013 ONOS1_master_not_null = main.ONOScli1.roles_not_null()
1014 ONOS2_master_not_null = main.ONOScli2.roles_not_null()
1015 ONOS3_master_not_null = main.ONOScli3.roles_not_null()
1016 ONOS4_master_not_null = main.ONOScli4.roles_not_null()
1017 ONOS5_master_not_null = main.ONOScli5.roles_not_null()
1018 ONOS6_master_not_null = main.ONOScli6.roles_not_null()
1019 ONOS7_master_not_null = main.ONOScli7.roles_not_null()
1020 roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
1021 ONOS3_master_not_null and ONOS4_master_not_null and\
1022 ONOS5_master_not_null and ONOS6_master_not_null and\
1023 ONOS7_master_not_null
1024 utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
1025 onpass="Each device has a master",
1026 onfail="Some devices don't have a master assigned")
1027
1028
1029
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1030 main.step("Check if switch roles are consistent across all nodes")
1031 ONOS1_mastership = main.ONOScli1.roles()
1032 ONOS2_mastership = main.ONOScli2.roles()
1033 ONOS3_mastership = main.ONOScli3.roles()
1034 ONOS4_mastership = main.ONOScli4.roles()
1035 ONOS5_mastership = main.ONOScli5.roles()
1036 ONOS6_mastership = main.ONOScli6.roles()
1037 ONOS7_mastership = main.ONOScli7.roles()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1038 #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
1039 if "Error" in ONOS1_mastership or not ONOS1_mastership\
1040 or "Error" in ONOS2_mastership or not ONOS2_mastership\
1041 or "Error" in ONOS3_mastership or not ONOS3_mastership\
1042 or "Error" in ONOS4_mastership or not ONOS4_mastership\
1043 or "Error" in ONOS5_mastership or not ONOS5_mastership\
1044 or "Error" in ONOS6_mastership or not ONOS6_mastership\
1045 or "Error" in ONOS7_mastership or not ONOS7_mastership:
1046 main.log.error("Error in getting ONOS mastership")
1047 main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
1048 main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
1049 main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
1050 main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
1051 main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
1052 main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
1053 main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
1054 consistent_mastership = main.FALSE
1055 elif ONOS1_mastership == ONOS2_mastership\
1056 and ONOS1_mastership == ONOS3_mastership\
1057 and ONOS1_mastership == ONOS4_mastership\
1058 and ONOS1_mastership == ONOS5_mastership\
1059 and ONOS1_mastership == ONOS6_mastership\
1060 and ONOS1_mastership == ONOS7_mastership:
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1061 consistent_mastership = main.TRUE
1062 main.log.report("Switch roles are consistent across all ONOS nodes")
1063 else:
1064 main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
1065 sort_keys=True, indent=4, separators=(',', ': ')))
1066 main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
1067 sort_keys=True, indent=4, separators=(',', ': ')))
1068 main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
1069 sort_keys=True, indent=4, separators=(',', ': ')))
1070 main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
1071 sort_keys=True, indent=4, separators=(',', ': ')))
1072 main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
1073 sort_keys=True, indent=4, separators=(',', ': ')))
1074 main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
1075 sort_keys=True, indent=4, separators=(',', ': ')))
1076 main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
1077 sort_keys=True, indent=4, separators=(',', ': ')))
1078 consistent_mastership = main.FALSE
1079 utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
1080 onpass="Switch roles are consistent across all ONOS nodes",
1081 onfail="ONOS nodes have different views of switch roles")
1082
1083
1084 description2 = "Compare switch roles from before failure"
1085 main.step(description2)
1086
1087 current_json = json.loads(ONOS1_mastership)
1088 old_json = json.loads(mastership_state)
1089 mastership_check = main.TRUE
1090 for i in range(1,29):
1091 switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
1092
1093 current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
1094 old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
1095 if current == old:
1096 mastership_check = mastership_check and main.TRUE
1097 else:
1098 main.log.warn("Mastership of switch %s changed" % switchDPID)
1099 mastership_check = main.FALSE
1100 if mastership_check == main.TRUE:
1101 main.log.report("Mastership of Switches was not changed")
1102 utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
1103 onpass="Mastership of Switches was not changed",
1104 onfail="Mastership of some switches changed")
1105 #NOTE: we expect mastership to change on controller failure
1106 mastership_check = mastership_check #and consistent_mastership
1107
1108
1109
1110 main.step("Get the intents and compare across all nodes")
1111 ONOS1_intents = main.ONOScli1.intents( json_format=True )
1112 ONOS2_intents = main.ONOScli2.intents( json_format=True )
1113 ONOS3_intents = main.ONOScli3.intents( json_format=True )
1114 ONOS4_intents = main.ONOScli4.intents( json_format=True )
1115 ONOS5_intents = main.ONOScli5.intents( json_format=True )
1116 ONOS6_intents = main.ONOScli6.intents( json_format=True )
1117 ONOS7_intents = main.ONOScli7.intents( json_format=True )
1118 intent_check = main.FALSE
1119 if "Error" in ONOS1_intents or not ONOS1_intents\
1120 or "Error" in ONOS2_intents or not ONOS2_intents\
1121 or "Error" in ONOS3_intents or not ONOS3_intents\
1122 or "Error" in ONOS4_intents or not ONOS4_intents\
1123 or "Error" in ONOS5_intents or not ONOS5_intents\
1124 or "Error" in ONOS6_intents or not ONOS6_intents\
1125 or "Error" in ONOS7_intents or not ONOS7_intents:
1126 main.log.report("Error in getting ONOS intents")
1127 main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
1128 main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
1129 main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
1130 main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
1131 main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
1132 main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
1133 main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
1134 elif ONOS1_intents == ONOS2_intents\
1135 and ONOS1_intents == ONOS3_intents\
1136 and ONOS1_intents == ONOS4_intents\
1137 and ONOS1_intents == ONOS5_intents\
1138 and ONOS1_intents == ONOS6_intents\
1139 and ONOS1_intents == ONOS7_intents:
1140 intent_check = main.TRUE
1141 main.log.report("Intents are consistent across all ONOS nodes")
1142 else:
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1143 main.log.warn("ONOS1 intents: ")
1144 print json.dumps(json.loads(ONOS1_intents),
1145 sort_keys=True, indent=4, separators=(',', ': '))
1146 main.log.warn("ONOS2 intents: ")
1147 print json.dumps(json.loads(ONOS2_intents),
1148 sort_keys=True, indent=4, separators=(',', ': '))
1149 main.log.warn("ONOS3 intents: ")
1150 print json.dumps(json.loads(ONOS3_intents),
1151 sort_keys=True, indent=4, separators=(',', ': '))
1152 main.log.warn("ONOS4 intents: ")
1153 print json.dumps(json.loads(ONOS4_intents),
1154 sort_keys=True, indent=4, separators=(',', ': '))
1155 main.log.warn("ONOS5 intents: ")
1156 print json.dumps(json.loads(ONOS5_intents),
1157 sort_keys=True, indent=4, separators=(',', ': '))
1158 main.log.warn("ONOS6 intents: ")
1159 print json.dumps(json.loads(ONOS6_intents),
1160 sort_keys=True, indent=4, separators=(',', ': '))
1161 main.log.warn("ONOS7 intents: ")
1162 print json.dumps(json.loads(ONOS7_intents),
1163 sort_keys=True, indent=4, separators=(',', ': '))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1164 utilities.assert_equals(expect = main.TRUE,actual=intent_check,
1165 onpass="Intents are consistent across all ONOS nodes",
1166 onfail="ONOS nodes have different views of intents")
1167
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1168 #NOTE: Hazelcast has no durability, so intents are lost
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1169 main.step("Compare current intents with intents before the failure")
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1170 #NOTE: this requires case 5 to pass for intent_state to be set.
1171 # maybe we should stop the test if that fails?
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1172 if intent_state == ONOS1_intents:
1173 same_intents = main.TRUE
1174 main.log.report("Intents are consistent with before failure")
1175 #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
1176 else:
1177 same_intents = main.FALSE
1178 utilities.assert_equals(expect = main.TRUE,actual=same_intents,
1179 onpass="Intents are consistent with before failure",
1180 onfail="The Intents changed during failure")
1181 intent_check = intent_check and same_intents
1182
1183
1184
1185 main.step("Get the OF Table entries and compare to before component failure")
1186 Flow_Tables = main.TRUE
1187 flows2=[]
1188 for i in range(28):
1189 main.log.info("Checking flow table on s" + str(i+1))
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1190 tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1191 flows2.append(tmp_flows)
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1192 temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
1193 Flow_Tables = Flow_Tables and temp_result
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1194 if Flow_Tables == main.FALSE:
1195 main.log.info("Differences in flow table for switch: "+str(i+1))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1196 if Flow_Tables == main.TRUE:
1197 main.log.report("No changes were found in the flow tables")
1198 utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
1199 onpass="No changes were found in the flow tables",
1200 onfail="Changes were found in the flow tables")
1201
1202 main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
1203 #FIXME: This check is always failing. Investigate cause
1204 #NOTE: this may be something to do with file permsissions
1205 # or slight change in format
1206 main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
1207 Loss_In_Pings = main.FALSE
1208 #NOTE: checkForLoss returns main.FALSE with 0% packet loss
1209 for i in range(8,18):
1210 main.log.info("Checking for a loss in pings along flow from s" + str(i))
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1211 Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1212 if Loss_In_Pings == main.TRUE:
1213 main.log.info("Loss in ping detected")
1214 elif Loss_In_Pings == main.ERROR:
1215 main.log.info("There are multiple mininet process running")
1216 elif Loss_In_Pings == main.FALSE:
1217 main.log.info("No Loss in the pings")
1218 main.log.report("No loss of dataplane connectivity")
1219 utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
1220 onpass="No Loss of connectivity",
1221 onfail="Loss of dataplane connectivity detected")
1222
1223
1224 #TODO:add topology to this or leave as a seperate case?
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1225 result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1226 result = int(result)
1227 if result == main.TRUE:
1228 main.log.report("Constant State Tests Passed")
1229 utilities.assert_equals(expect=main.TRUE,actual=result,
1230 onpass="Constant State Tests Passed",
1231 onfail="Constant state tests failed")
1232
1233 def CASE8 (self,main):
1234 '''
1235 Compare topo
1236 '''
1237 import sys
1238 sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
1239 from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
1240 import json
1241 import time
1242
1243 description ="Compare ONOS Topology view to Mininet topology"
1244 main.case(description)
1245 main.log.report(description)
1246 main.step("Create TestONTopology object")
1247 ctrls = []
1248 count = 1
1249 while True:
1250 temp = ()
1251 if ('ip' + str(count)) in main.params['CTRL']:
1252 temp = temp + (getattr(main,('ONOS' + str(count))),)
1253 temp = temp + ("ONOS"+str(count),)
1254 temp = temp + (main.params['CTRL']['ip'+str(count)],)
1255 temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
1256 ctrls.append(temp)
1257 count = count + 1
1258 else:
1259 break
1260 MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
1261
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1262 main.step("Comparing ONOS topology to MN")
1263 devices_results = main.TRUE
1264 ports_results = main.TRUE
1265 links_results = main.TRUE
1266 topo_result = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1267 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -0800[diff] [blame]1268 count = 0
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1269 main.step("Collecting topology information from ONOS")
1270 start_time = time.time()
1271 while topo_result == main.FALSE and elapsed < 60:
Jon Hallffb386d2014-11-21 13:43:38 -0800[diff] [blame]1272 count = count + 1
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1273 if count > 1:
1274 MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
1275 cli_start = time.time()
1276 devices = []
1277 devices.append( main.ONOScli1.devices() )
1278 devices.append( main.ONOScli2.devices() )
1279 devices.append( main.ONOScli3.devices() )
1280 devices.append( main.ONOScli4.devices() )
1281 devices.append( main.ONOScli5.devices() )
1282 devices.append( main.ONOScli6.devices() )
1283 devices.append( main.ONOScli7.devices() )
1284 hosts = []
1285 hosts.append( main.ONOScli1.hosts() )
1286 hosts.append( main.ONOScli2.hosts() )
1287 hosts.append( main.ONOScli3.hosts() )
1288 hosts.append( main.ONOScli4.hosts() )
1289 hosts.append( main.ONOScli5.hosts() )
1290 hosts.append( main.ONOScli6.hosts() )
1291 hosts.append( main.ONOScli7.hosts() )
1292 ports = []
1293 ports.append( main.ONOScli1.ports() )
1294 ports.append( main.ONOScli2.ports() )
1295 ports.append( main.ONOScli3.ports() )
1296 ports.append( main.ONOScli4.ports() )
1297 ports.append( main.ONOScli5.ports() )
1298 ports.append( main.ONOScli6.ports() )
1299 ports.append( main.ONOScli7.ports() )
1300 links = []
1301 links.append( main.ONOScli1.links() )
1302 links.append( main.ONOScli2.links() )
1303 links.append( main.ONOScli3.links() )
1304 links.append( main.ONOScli4.links() )
1305 links.append( main.ONOScli5.links() )
1306 links.append( main.ONOScli6.links() )
1307 links.append( main.ONOScli7.links() )
1308 clusters = []
1309 clusters.append( main.ONOScli1.clusters() )
1310 clusters.append( main.ONOScli2.clusters() )
1311 clusters.append( main.ONOScli3.clusters() )
1312 clusters.append( main.ONOScli4.clusters() )
1313 clusters.append( main.ONOScli5.clusters() )
1314 clusters.append( main.ONOScli6.clusters() )
1315 clusters.append( main.ONOScli7.clusters() )
1316 paths = []
1317 temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
1318 paths.append( temp_topo.get('paths', False) )
1319 temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
1320 paths.append( temp_topo.get('paths', False) )
1321 temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
1322 paths.append( temp_topo.get('paths', False) )
1323 temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
1324 paths.append( temp_topo.get('paths', False) )
1325 temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
1326 paths.append( temp_topo.get('paths', False) )
1327 temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
1328 paths.append( temp_topo.get('paths', False) )
1329 temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
1330 paths.append( temp_topo.get('paths', False) )
Jon Hallffb386d2014-11-21 13:43:38 -0800[diff] [blame]1331
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1332
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1333 elapsed = time.time() - start_time
1334 cli_time = time.time() - cli_start
1335 print "CLI time: " + str(cli_time)
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1336
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1337 for controller in range(7): #TODO parameterize the number of controllers
1338 if devices[controller] or not "Error" in devices[controller]:
1339 current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
1340 else:
1341 current_devices_result = main.FALSE
1342 utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
1343 onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
1344 onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1345
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1346 if ports[controller] or not "Error" in ports[controller]:
1347 current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
1348 else:
1349 current_ports_result = main.FALSE
1350 utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
1351 onpass="ONOS"+str(int(controller+1))+" ports view is correct",
1352 onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
1353
1354 if links[controller] or not "Error" in links[controller]:
1355 current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
1356 else:
1357 current_links_result = main.FALSE
1358 utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
1359 onpass="ONOS"+str(int(controller+1))+" links view is correct",
1360 onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1361 devices_results = devices_results and current_devices_result
1362 ports_results = ports_results and current_ports_result
1363 links_results = links_results and current_links_result
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1364
1365 #Compare json objects for hosts, dataplane clusters and paths
1366
1367 #hosts
1368 consistent_hosts_result = main.TRUE
1369 for controller in range( len( hosts ) ):
1370 if not "Error" in hosts[controller]:
1371 if hosts[controller] == hosts[0]:
1372 continue
1373 else:#hosts not consistent
1374 main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
1375 main.log.warn( repr( hosts[controller] ) )
1376 consistent_hosts_result = main.FALSE
1377
1378 else:
1379 main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
1380 consistent_hosts_result = main.FALSE
1381 if consistent_hosts_result == main.FALSE:
1382 for controller in range( len( hosts ) ):
1383 main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
1384 utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
1385 onpass="Hosts view is consistent across all ONOS nodes",
1386 onfail="ONOS nodes have different views of hosts")
1387
1388 #Strongly connected clusters of devices
1389 consistent_clusters_result = main.TRUE
1390 for controller in range( len( clusters ) ):
1391 if not "Error" in clusters[controller]:
1392 if clusters[controller] == clusters[0]:
1393 continue
1394 else:#clusters not consistent
1395 main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
1396 consistent_clusters_result = main.FALSE
1397
1398 else:
1399 main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
1400 consistent_clusters_result = main.FALSE
1401 main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
1402 utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
1403 onpass="Clusters view is consistent across all ONOS nodes",
1404 onfail="ONOS nodes have different views of clusters")
1405 num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
1406 utilities.assert_equals(expect = 1, actual = num_clusters,
1407 onpass="ONOS shows 1 SCC",
1408 onfail="ONOS shows "+str(num_clusters) +" SCCs")
1409
1410
1411 #paths
1412 consistent_paths_result = main.TRUE
1413 for controller in range( len( paths ) ):
1414 if not "Error" in paths[controller]:
1415 if paths[controller] == paths[0]:
1416 continue
1417 else:#paths not consistent
1418 main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
1419 consistent_paths_result = main.FALSE
1420
1421 else:
1422 main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
1423 consistent_paths_result = main.FALSE
1424 main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
1425 utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
1426 onpass="Paths count is consistent across all ONOS nodes",
1427 onfail="ONOS nodes have different counts of paths")
1428
1429
1430 topo_result = devices_results and ports_results and links_results\
1431 and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
1432
1433 topo_result = topo_result and int(count <= 2)
1434 note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
1435 main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
Jon Hallffb386d2014-11-21 13:43:38 -0800[diff] [blame]1436 str(elapsed) + " seconds, " + str(count) +" tries" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1437 utilities.assert_equals(expect=main.TRUE, actual=topo_result,
1438 onpass="Topology Check Test successful",
1439 onfail="Topology Check Test NOT successful")
1440 if topo_result == main.TRUE:
1441 main.log.report("ONOS topology view matches Mininet topology")
1442
1443
1444 def CASE9 (self,main):
1445 '''
1446 Link s3-s28 down
1447 '''
1448 #NOTE: You should probably run a topology check after this
1449
1450 link_sleep = int(main.params['timers']['LinkDiscovery'])
1451
1452 description = "Turn off a link to ensure that Link Discovery is working properly"
1453 main.log.report(description)
1454 main.case(description)
1455
1456
1457 main.step("Kill Link between s3 and s28")
1458 Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
1459 main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
1460 time.sleep(link_sleep)
1461 utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
1462 onpass="Link down succesful",
1463 onfail="Failed to bring link down")
1464 #TODO do some sort of check here
1465
1466 def CASE10 (self,main):
1467 '''
1468 Link s3-s28 up
1469 '''
1470 #NOTE: You should probably run a topology check after this
1471
1472 link_sleep = int(main.params['timers']['LinkDiscovery'])
1473
1474 description = "Restore a link to ensure that Link Discovery is working properly"
1475 main.log.report(description)
1476 main.case(description)
1477
1478 main.step("Bring link between s3 and s28 back up")
1479 Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
1480 main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
1481 time.sleep(link_sleep)
1482 utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
1483 onpass="Link up succesful",
1484 onfail="Failed to bring link up")
1485 #TODO do some sort of check here
1486
1487
1488 def CASE11 (self, main) :
1489 '''
1490 Switch Down
1491 '''
1492 #NOTE: You should probably run a topology check after this
1493 import time
1494
1495 switch_sleep = int(main.params['timers']['SwitchDiscovery'])
1496
1497 description = "Killing a switch to ensure it is discovered correctly"
1498 main.log.report(description)
1499 main.case(description)
1500
1501 #TODO: Make this switch parameterizable
1502 main.step("Kill s28 ")
1503 main.log.report("Deleting s28")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1504 main.Mininet1.del_switch("s28")
1505 main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
1506 time.sleep(switch_sleep)
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1507 device = main.ONOScli1.get_device(dpid="0028")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1508 #Peek at the deleted switch
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1509 main.log.warn( str(device) )
1510 result = main.FALSE
1511 if device and device['available'] == False:
1512 result = main.TRUE
1513 utilities.assert_equals(expect=main.TRUE,actual=result,
1514 onpass="Kill switch succesful",
1515 onfail="Failed to kill switch?")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1516
1517 def CASE12 (self, main) :
1518 '''
1519 Switch Up
1520 '''
1521 #NOTE: You should probably run a topology check after this
1522 import time
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1523 description = "Adding a switch to ensure it is discovered correctly"
1524 main.log.report(description)
1525 main.case(description)
1526
1527 main.step("Add back s28")
1528 main.log.report("Adding back s28")
1529 main.Mininet1.add_switch("s28", dpid = '0000000000002800')
1530 #TODO: New dpid or same? Ask Thomas?
1531 main.Mininet1.add_link('s28', 's3')
1532 main.Mininet1.add_link('s28', 's6')
1533 main.Mininet1.add_link('s28', 'h28')
1534 main.Mininet1.assign_sw_controller(sw="28",count=7,
1535 ip1=ONOS1_ip,port1=ONOS1_port,
1536 ip2=ONOS2_ip,port2=ONOS2_port,
1537 ip3=ONOS3_ip,port3=ONOS3_port,
1538 ip4=ONOS4_ip,port4=ONOS4_port,
1539 ip5=ONOS5_ip,port5=ONOS5_port,
1540 ip6=ONOS6_ip,port6=ONOS6_port,
1541 ip7=ONOS7_ip,port7=ONOS7_port)
1542 main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
1543 time.sleep(switch_sleep)
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1544 device = main.ONOScli1.get_device(dpid="0028")
1545 #Peek at the deleted switch
1546 main.log.warn( str(device) )
1547 result = main.FALSE
1548 if device and device['available'] == True:
1549 result = main.TRUE
1550 utilities.assert_equals(expect=main.TRUE,actual=result,
1551 onpass="add switch succesful",
1552 onfail="Failed to add switch?")
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1553
1554 def CASE13 (self, main) :
1555 '''
1556 Clean up
1557 '''
1558 import os
1559 import time
1560 description = "Test Cleanup"
1561 main.log.report(description)
1562 main.case(description)
1563 main.step("Killing tcpdumps")
1564 main.Mininet2.stop_tcpdump()
1565
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1566 main.step("Checking ONOS Logs for errors")
1567 print "Checking logs for errors on ONOS1:"
1568 print main.ONOSbench.check_logs(ONOS1_ip)
1569 print "Checking logs for errors on ONOS2:"
1570 print main.ONOSbench.check_logs(ONOS2_ip)
1571 print "Checking logs for errors on ONOS3:"
1572 print main.ONOSbench.check_logs(ONOS3_ip)
1573 print "Checking logs for errors on ONOS4:"
1574 print main.ONOSbench.check_logs(ONOS4_ip)
1575 print "Checking logs for errors on ONOS5:"
1576 print main.ONOSbench.check_logs(ONOS5_ip)
1577 print "Checking logs for errors on ONOS6:"
1578 print main.ONOSbench.check_logs(ONOS6_ip)
1579 print "Checking logs for errors on ONOS7:"
1580 print main.ONOSbench.check_logs(ONOS7_ip)
1581
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1582 main.step("Copying MN pcap and ONOS log files to test station")
1583 testname = main.TEST
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1584 teststation_user = main.params['TESTONUSER']
1585 teststation_IP = main.params['TESTONIP']
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1586 #NOTE: MN Pcap file is being saved to ~/packet_captures
1587 # scp this file as MN and TestON aren't necessarily the same vm
1588 #FIXME: scp
1589 #####mn files
1590 #TODO: Load these from params
1591 #NOTE: must end in /
1592 log_folder = "/opt/onos/log/"
1593 log_files = ["karaf.log", "karaf.log.1"]
1594 #NOTE: must end in /
1595 dst_dir = "~/packet_captures/"
1596 for f in log_files:
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1597 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
1598 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1599 dst_dir + str(testname) + "-ONOS1-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1600 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
1601 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1602 dst_dir + str(testname) + "-ONOS2-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1603 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
1604 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1605 dst_dir + str(testname) + "-ONOS3-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1606 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
1607 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1608 dst_dir + str(testname) + "-ONOS4-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1609 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
1610 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1611 dst_dir + str(testname) + "-ONOS5-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1612 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
1613 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1614 dst_dir + str(testname) + "-ONOS6-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1615 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
1616 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1617 dst_dir + str(testname) + "-ONOS7-"+f )
1618
1619 #std*.log's
1620 #NOTE: must end in /
1621 log_folder = "/opt/onos/var/"
1622 log_files = ["stderr.log", "stdout.log"]
1623 #NOTE: must end in /
1624 dst_dir = "~/packet_captures/"
1625 for f in log_files:
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1626 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
1627 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1628 dst_dir + str(testname) + "-ONOS1-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1629 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
1630 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1631 dst_dir + str(testname) + "-ONOS2-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1632 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
1633 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1634 dst_dir + str(testname) + "-ONOS3-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1635 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
1636 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1637 dst_dir + str(testname) + "-ONOS4-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1638 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
1639 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1640 dst_dir + str(testname) + "-ONOS5-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1641 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
1642 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1643 dst_dir + str(testname) + "-ONOS6-"+f )
Jon Hall94fd0472014-12-08 11:52:42 -0800[diff] [blame^]1644 main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
1645 teststation_user +"@"+teststation_IP+":"+\
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1646 dst_dir + str(testname) + "-ONOS7-"+f )
1647
1648
Jon Hall73cf9cc2014-11-20 22:28:38 -0800[diff] [blame]1649 #sleep so scp can finish
1650 time.sleep(10)
1651 main.step("Packing and rotating pcap archives")
1652 os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
1653
1654
1655 #TODO: actually check something here
1656 utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
1657 onpass="Test cleanup successful",
1658 onfail="Test cleanup NOT successful")