blob: 2af775d30f82162f25d4d936a4e1439ec30922c6 [file] [log] [blame]
Jon Hall73cf9cc2014-11-20 22:28:38 -08001'''
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign mastership to controllers
8CASE3: Assign intents
9CASE4: Ping across added host intents
10CASE5: Reading state of ONOS
11CASE6: The Failure case.
12CASE7: Check state after control plane failure
13CASE8: Compare topo
14CASE9: Link s3-s28 down
15CASE10: Link s3-s28 up
16CASE11: Switch down
17CASE12: Switch up
18CASE13: Clean up
19'''
20class HATestMinorityRestart:
21
22 def __init__(self) :
23 self.default = ''
24
25 def CASE1(self,main) :
26 '''
27 CASE1 is to compile ONOS and push it to the test machines
28
29 Startup sequence:
30 git pull
31 mvn clean install
32 onos-package
33 cell <name>
34 onos-verify-cell
35 NOTE: temporary - onos-remove-raft-logs
36 onos-install -f
37 onos-wait-for-start
38 '''
39 import time
40 main.log.report("ONOS HA test: Restart minority of ONOS nodes - initialization")
41 main.case("Setting up test environment")
42
43 # load some vairables from the params file
44 PULL_CODE = False
45 if main.params['Git'] == 'True':
46 PULL_CODE = True
47 cell_name = main.params['ENV']['cellName']
48
49 #set global variables
50 global ONOS1_ip
51 global ONOS1_port
52 global ONOS2_ip
53 global ONOS2_port
54 global ONOS3_ip
55 global ONOS3_port
56 global ONOS4_ip
57 global ONOS4_port
58 global ONOS5_ip
59 global ONOS5_port
60 global ONOS6_ip
61 global ONOS6_port
62 global ONOS7_ip
63 global ONOS7_port
64
65 ONOS1_ip = main.params['CTRL']['ip1']
66 ONOS1_port = main.params['CTRL']['port1']
67 ONOS2_ip = main.params['CTRL']['ip2']
68 ONOS2_port = main.params['CTRL']['port2']
69 ONOS3_ip = main.params['CTRL']['ip3']
70 ONOS3_port = main.params['CTRL']['port3']
71 ONOS4_ip = main.params['CTRL']['ip4']
72 ONOS4_port = main.params['CTRL']['port4']
73 ONOS5_ip = main.params['CTRL']['ip5']
74 ONOS5_port = main.params['CTRL']['port5']
75 ONOS6_ip = main.params['CTRL']['ip6']
76 ONOS6_port = main.params['CTRL']['port6']
77 ONOS7_ip = main.params['CTRL']['ip7']
78 ONOS7_port = main.params['CTRL']['port7']
79
80
81 main.step("Applying cell variable to environment")
82 cell_result = main.ONOSbench.set_cell(cell_name)
83 verify_result = main.ONOSbench.verify_cell()
84
85 #FIXME:this is short term fix
86 main.log.report("Removing raft logs")
87 main.ONOSbench.onos_remove_raft_logs()
88 main.log.report("Uninstalling ONOS")
89 main.ONOSbench.onos_uninstall(ONOS1_ip)
90 main.ONOSbench.onos_uninstall(ONOS2_ip)
91 main.ONOSbench.onos_uninstall(ONOS3_ip)
92 main.ONOSbench.onos_uninstall(ONOS4_ip)
93 main.ONOSbench.onos_uninstall(ONOS5_ip)
94 main.ONOSbench.onos_uninstall(ONOS6_ip)
95 main.ONOSbench.onos_uninstall(ONOS7_ip)
96
97 clean_install_result = main.TRUE
98 git_pull_result = main.TRUE
99
100 main.step("Compiling the latest version of ONOS")
101 if PULL_CODE:
102 main.step("Git checkout and pull master")
103 main.ONOSbench.git_checkout("master")
104 git_pull_result = main.ONOSbench.git_pull()
105
106 main.step("Using mvn clean & install")
107 clean_install_result = main.TRUE
108 if git_pull_result == main.TRUE:
109 clean_install_result = main.ONOSbench.clean_install()
110 else:
111 main.log.warn("Did not pull new code so skipping mvn "+ \
112 "clean install")
113 main.ONOSbench.get_version(report=True)
114
115 main.step("Creating ONOS package")
116 package_result = main.ONOSbench.onos_package()
117
118 main.step("Installing ONOS package")
119 onos1_install_result = main.ONOSbench.onos_install(options="-f",
120 node=ONOS1_ip)
121 onos2_install_result = main.ONOSbench.onos_install(options="-f",
122 node=ONOS2_ip)
123 onos3_install_result = main.ONOSbench.onos_install(options="-f",
124 node=ONOS3_ip)
125 onos4_install_result = main.ONOSbench.onos_install(options="-f",
126 node=ONOS4_ip)
127 onos5_install_result = main.ONOSbench.onos_install(options="-f",
128 node=ONOS5_ip)
129 onos6_install_result = main.ONOSbench.onos_install(options="-f",
130 node=ONOS6_ip)
131 onos7_install_result = main.ONOSbench.onos_install(options="-f",
132 node=ONOS7_ip)
133 onos_install_result = onos1_install_result and onos2_install_result\
134 and onos3_install_result and onos4_install_result\
135 and onos5_install_result and onos6_install_result\
136 and onos7_install_result
137 '''
138 #FIXME: work around until onos is less fragile
139 main.ONOSbench.handle.sendline("onos-cluster-install")
140 print main.ONOSbench.handle.expect("\$")
141 onos_install_result = main.TRUE
142 '''
143
144
145 main.step("Checking if ONOS is up yet")
146 #TODO: Refactor
147 # check bundle:list?
148 onos1_isup = main.ONOSbench.isup(ONOS1_ip)
149 if not onos1_isup:
150 main.log.report("ONOS1 didn't start!")
151 onos2_isup = main.ONOSbench.isup(ONOS2_ip)
152 if not onos2_isup:
153 main.log.report("ONOS2 didn't start!")
154 onos3_isup = main.ONOSbench.isup(ONOS3_ip)
155 if not onos3_isup:
156 main.log.report("ONOS3 didn't start!")
157 onos4_isup = main.ONOSbench.isup(ONOS4_ip)
158 if not onos4_isup:
159 main.log.report("ONOS4 didn't start!")
160 onos5_isup = main.ONOSbench.isup(ONOS5_ip)
161 if not onos5_isup:
162 main.log.report("ONOS5 didn't start!")
163 onos6_isup = main.ONOSbench.isup(ONOS6_ip)
164 if not onos6_isup:
165 main.log.report("ONOS6 didn't start!")
166 onos7_isup = main.ONOSbench.isup(ONOS7_ip)
167 if not onos7_isup:
168 main.log.report("ONOS7 didn't start!")
169 onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
170 and onos4_isup and onos5_isup and onos6_isup and onos7_isup
171 # TODO: if it becomes an issue, we can retry this step a few times
172
173
174 cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
175 cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
176 cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
177 cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
178 cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
179 cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
180 cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
181 cli_results = cli_result1 and cli_result2 and cli_result3 and\
182 cli_result4 and cli_result5 and cli_result6 and cli_result7
183
184 main.step("Start Packet Capture MN")
185 main.Mininet2.start_tcpdump(
186 str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
187 intf = main.params['MNtcpdump']['intf'],
188 port = main.params['MNtcpdump']['port'])
189
190
191 case1_result = (clean_install_result and package_result and
192 cell_result and verify_result and onos_install_result and
193 onos_isup_result and cli_results)
194
195 utilities.assert_equals(expect=main.TRUE, actual=case1_result,
196 onpass="Test startup successful",
197 onfail="Test startup NOT successful")
198
199
200 #if case1_result==main.FALSE:
201 # main.cleanup()
202 # main.exit()
203
204 def CASE2(self,main) :
205 '''
206 Assign mastership to controllers
207 '''
208 import time
209 import json
210 import re
211
212
213 '''
214 ONOS1_ip = main.params['CTRL']['ip1']
215 ONOS1_port = main.params['CTRL']['port1']
216 ONOS2_ip = main.params['CTRL']['ip2']
217 ONOS2_port = main.params['CTRL']['port2']
218 ONOS3_ip = main.params['CTRL']['ip3']
219 ONOS3_port = main.params['CTRL']['port3']
220 ONOS4_ip = main.params['CTRL']['ip4']
221 ONOS4_port = main.params['CTRL']['port4']
222 ONOS5_ip = main.params['CTRL']['ip5']
223 ONOS5_port = main.params['CTRL']['port5']
224 ONOS6_ip = main.params['CTRL']['ip6']
225 ONOS6_port = main.params['CTRL']['port6']
226 ONOS7_ip = main.params['CTRL']['ip7']
227 ONOS7_port = main.params['CTRL']['port7']
228 '''
229
230
231 main.log.report("Assigning switches to controllers")
232 main.case("Assigning Controllers")
233 main.step("Assign switches to controllers")
234
235 for i in range (1,29):
236 main.Mininet1.assign_sw_controller(sw=str(i),count=7,
237 ip1=ONOS1_ip,port1=ONOS1_port,
238 ip2=ONOS2_ip,port2=ONOS2_port,
239 ip3=ONOS3_ip,port3=ONOS3_port,
240 ip4=ONOS4_ip,port4=ONOS4_port,
241 ip5=ONOS5_ip,port5=ONOS5_port,
242 ip6=ONOS6_ip,port6=ONOS6_port,
243 ip7=ONOS7_ip,port7=ONOS7_port)
244
245 mastership_check = main.TRUE
246 for i in range (1,29):
247 response = main.Mininet1.get_sw_controller("s"+str(i))
Jon Hallffb386d2014-11-21 13:43:38 -0800248 try:
249 main.log.info(str(response))
250 except:
251 main.log.info(repr(response))
Jon Hall73cf9cc2014-11-20 22:28:38 -0800252 if re.search("tcp:"+ONOS1_ip,response)\
253 and re.search("tcp:"+ONOS2_ip,response)\
254 and re.search("tcp:"+ONOS3_ip,response)\
255 and re.search("tcp:"+ONOS4_ip,response)\
256 and re.search("tcp:"+ONOS5_ip,response)\
257 and re.search("tcp:"+ONOS6_ip,response)\
258 and re.search("tcp:"+ONOS7_ip,response):
259 mastership_check = mastership_check and main.TRUE
260 else:
261 mastership_check = main.FALSE
262 if mastership_check == main.TRUE:
263 main.log.report("Switch mastership assigned correctly")
264 utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
265 onpass="Switch mastership assigned correctly",
266 onfail="Switches not assigned correctly to controllers")
267
268 #TODO: If assign roles is working reliably then manually
269 # assign mastership to the controller we want
270
271
272 def CASE3(self,main) :
273 """
274 Assign intents
275
276 """
277 import time
278 import json
279 import re
280 main.log.report("Adding host intents")
281 main.case("Adding host Intents")
282
283 main.step("Discovering Hosts( Via pingall for now)")
284 #FIXME: Once we have a host discovery mechanism, use that instead
285
286 #REACTIVE FWD test
287 ping_result = main.FALSE
288 time1 = time.time()
289 ping_result = main.Mininet1.pingall()
290 time2 = time.time()
291 main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
292
293 #uninstall onos-app-fwd
294 main.log.info("Uninstall reactive forwarding app")
295 main.ONOScli1.feature_uninstall("onos-app-fwd")
296 main.ONOScli2.feature_uninstall("onos-app-fwd")
297 main.ONOScli3.feature_uninstall("onos-app-fwd")
298 main.ONOScli4.feature_uninstall("onos-app-fwd")
299 main.ONOScli5.feature_uninstall("onos-app-fwd")
300 main.ONOScli6.feature_uninstall("onos-app-fwd")
301 main.ONOScli7.feature_uninstall("onos-app-fwd")
302
303 main.step("Add host intents")
304 #TODO: move the host numbers to params
305 import json
306 intents_json= json.loads(main.ONOScli1.hosts())
307 intent_add_result = main.FALSE
308 for i in range(8,18):
309 main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
310 host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
311 host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
312 #NOTE: get host can return None
313 #TODO: handle this
314 host1_id = main.ONOScli1.get_host(host1)['id']
315 host2_id = main.ONOScli1.get_host(host2)['id']
316 tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
317 intent_add_result = intent_add_result and tmp_result
318 #TODO Check if intents all exist in datastore
319 #NOTE: Do we need to print this once the test is working?
320 #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
321 # sort_keys=True, indent=4, separators=(',', ': ') ) )
322
323 def CASE4(self,main) :
324 """
325 Ping across added host intents
326 """
327 description = " Ping across added host intents"
328 main.log.report(description)
329 main.case(description)
330 Ping_Result = main.TRUE
331 for i in range(8,18):
332 ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
333 Ping_Result = Ping_Result and ping
334 if ping==main.FALSE:
335 main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
336 elif ping==main.TRUE:
337 main.log.info("Ping test passed!")
338 Ping_Result = main.TRUE
339 if Ping_Result==main.FALSE:
340 main.log.report("Intents have not been installed correctly, pings failed.")
341 if Ping_Result==main.TRUE:
342 main.log.report("Intents have been installed correctly and verified by pings")
343 utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
344 onpass="Intents have been installed correctly and pings work",
345 onfail ="Intents have not been installed correctly, pings failed." )
346
347 def CASE5(self,main) :
348 '''
349 Reading state of ONOS
350 '''
351 import time
352 import json
353 from subprocess import Popen, PIPE
354 from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
355
356 main.log.report("Setting up and gathering data for current state")
357 main.case("Setting up and gathering data for current state")
358 #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
359 #We can then compare them with eachother and also with past states
360
361 main.step("Get the Mastership of each switch from each controller")
362 global mastership_state
363 ONOS1_mastership = main.ONOScli1.roles()
364 ONOS2_mastership = main.ONOScli2.roles()
365 ONOS3_mastership = main.ONOScli3.roles()
366 ONOS4_mastership = main.ONOScli4.roles()
367 ONOS5_mastership = main.ONOScli5.roles()
368 ONOS6_mastership = main.ONOScli6.roles()
369 ONOS7_mastership = main.ONOScli7.roles()
370 #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
371 if "Error" in ONOS1_mastership or not ONOS1_mastership\
372 or "Error" in ONOS2_mastership or not ONOS2_mastership\
373 or "Error" in ONOS3_mastership or not ONOS3_mastership\
374 or "Error" in ONOS4_mastership or not ONOS4_mastership\
375 or "Error" in ONOS5_mastership or not ONOS5_mastership\
376 or "Error" in ONOS6_mastership or not ONOS6_mastership\
377 or "Error" in ONOS7_mastership or not ONOS7_mastership:
378 main.log.report("Error in getting ONOS roles")
379 main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
380 main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
381 main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
382 main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
383 main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
384 main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
385 main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
386 consistent_mastership = main.FALSE
387 elif ONOS1_mastership == ONOS2_mastership\
388 and ONOS1_mastership == ONOS3_mastership\
389 and ONOS1_mastership == ONOS4_mastership\
390 and ONOS1_mastership == ONOS5_mastership\
391 and ONOS1_mastership == ONOS6_mastership\
392 and ONOS1_mastership == ONOS7_mastership:
393 mastership_state = ONOS1_mastership
394 consistent_mastership = main.TRUE
395 main.log.report("Switch roles are consistent across all ONOS nodes")
396 else:
397 main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
398 sort_keys=True, indent=4, separators=(',', ': ')))
399 main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
400 sort_keys=True, indent=4, separators=(',', ': ')))
401 main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
402 sort_keys=True, indent=4, separators=(',', ': ')))
403 main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
404 sort_keys=True, indent=4, separators=(',', ': ')))
405 main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
406 sort_keys=True, indent=4, separators=(',', ': ')))
407 main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
408 sort_keys=True, indent=4, separators=(',', ': ')))
409 main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
410 sort_keys=True, indent=4, separators=(',', ': ')))
411 consistent_mastership = main.FALSE
412 utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
413 onpass="Switch roles are consistent across all ONOS nodes",
414 onfail="ONOS nodes have different views of switch roles")
415
416
417 main.step("Get the intents from each controller")
418 global intent_state
419 ONOS1_intents = main.ONOScli1.intents( json_format=True )
420 ONOS2_intents = main.ONOScli2.intents( json_format=True )
421 ONOS3_intents = main.ONOScli3.intents( json_format=True )
422 ONOS4_intents = main.ONOScli4.intents( json_format=True )
423 ONOS5_intents = main.ONOScli5.intents( json_format=True )
424 ONOS6_intents = main.ONOScli6.intents( json_format=True )
425 ONOS7_intents = main.ONOScli7.intents( json_format=True )
426 intent_check = main.FALSE
427 if "Error" in ONOS1_intents or not ONOS1_intents\
428 or "Error" in ONOS2_intents or not ONOS2_intents\
429 or "Error" in ONOS3_intents or not ONOS3_intents\
430 or "Error" in ONOS4_intents or not ONOS4_intents\
431 or "Error" in ONOS5_intents or not ONOS5_intents\
432 or "Error" in ONOS6_intents or not ONOS6_intents\
433 or "Error" in ONOS7_intents or not ONOS7_intents:
434 main.log.report("Error in getting ONOS intents")
435 main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
436 main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
437 main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
438 main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
439 main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
440 main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
441 main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
442 elif ONOS1_intents == ONOS2_intents\
443 and ONOS1_intents == ONOS3_intents\
444 and ONOS1_intents == ONOS4_intents\
445 and ONOS1_intents == ONOS5_intents\
446 and ONOS1_intents == ONOS6_intents\
447 and ONOS1_intents == ONOS7_intents:
448 intent_state = ONOS1_intents
449 intent_check = main.TRUE
450 main.log.report("Intents are consistent across all ONOS nodes")
451 else:
452 main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
453 sort_keys=True, indent=4, separators=(',', ': ')))
454 main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
455 sort_keys=True, indent=4, separators=(',', ': ')))
456 main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
457 sort_keys=True, indent=4, separators=(',', ': ')))
458 main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
459 sort_keys=True, indent=4, separators=(',', ': ')))
460 main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
461 sort_keys=True, indent=4, separators=(',', ': ')))
462 main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
463 sort_keys=True, indent=4, separators=(',', ': ')))
464 main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
465 sort_keys=True, indent=4, separators=(',', ': ')))
466 utilities.assert_equals(expect = main.TRUE,actual=intent_check,
467 onpass="Intents are consistent across all ONOS nodes",
468 onfail="ONOS nodes have different views of intents")
469
470
471 main.step("Get the flows from each controller")
472 global flow_state
473 ONOS1_flows = main.ONOScli1.flows( json_format=True )
474 ONOS2_flows = main.ONOScli2.flows( json_format=True )
475 ONOS3_flows = main.ONOScli3.flows( json_format=True )
476 ONOS4_flows = main.ONOScli4.flows( json_format=True )
477 ONOS5_flows = main.ONOScli5.flows( json_format=True )
478 ONOS6_flows = main.ONOScli6.flows( json_format=True )
479 ONOS7_flows = main.ONOScli7.flows( json_format=True )
480 flow_check = main.FALSE
481 if "Error" in ONOS1_flows or not ONOS1_flows\
482 or "Error" in ONOS2_flows or not ONOS2_flows\
483 or "Error" in ONOS3_flows or not ONOS3_flows\
484 or "Error" in ONOS4_flows or not ONOS4_flows\
485 or "Error" in ONOS5_flows or not ONOS5_flows\
486 or "Error" in ONOS6_flows or not ONOS6_flows\
487 or "Error" in ONOS7_flows or not ONOS7_flows:
488 main.log.report("Error in getting ONOS intents")
489 main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
490 main.log.warn("ONOS2 flows repsponse: "+ ONOS2_flows)
491 main.log.warn("ONOS3 flows repsponse: "+ ONOS3_flows)
492 main.log.warn("ONOS4 flows repsponse: "+ ONOS4_flows)
493 main.log.warn("ONOS5 flows repsponse: "+ ONOS5_flows)
494 main.log.warn("ONOS6 flows repsponse: "+ ONOS6_flows)
495 main.log.warn("ONOS7 flows repsponse: "+ ONOS7_flows)
496 elif len(json.loads(ONOS1_flows)) == len(json.loads(ONOS2_flows))\
497 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS3_flows))\
498 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS4_flows))\
499 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS5_flows))\
500 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS6_flows))\
501 and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS7_flows)):
502 #TODO: Do a better check, maybe compare flows on switches?
503 flow_state = ONOS1_flows
504 flow_check = main.TRUE
505 main.log.report("Flow count is consistent across all ONOS nodes")
506 else:
507 main.log.warn("ONOS1 flows: "+ json.dumps(json.loads(ONOS1_flows),
508 sort_keys=True, indent=4, separators=(',', ': ')))
509 main.log.warn("ONOS2 flows: "+ json.dumps(json.loads(ONOS2_flows),
510 sort_keys=True, indent=4, separators=(',', ': ')))
511 main.log.warn("ONOS3 flows: "+ json.dumps(json.loads(ONOS3_flows),
512 sort_keys=True, indent=4, separators=(',', ': ')))
513 main.log.warn("ONOS4 flows: "+ json.dumps(json.loads(ONOS4_flows),
514 sort_keys=True, indent=4, separators=(',', ': ')))
515 main.log.warn("ONOS5 flows: "+ json.dumps(json.loads(ONOS5_flows),
516 sort_keys=True, indent=4, separators=(',', ': ')))
517 main.log.warn("ONOS6 flows: "+ json.dumps(json.loads(ONOS6_flows),
518 sort_keys=True, indent=4, separators=(',', ': ')))
519 main.log.warn("ONOS7 flows: "+ json.dumps(json.loads(ONOS7_flows),
520 sort_keys=True, indent=4, separators=(',', ': ')))
521 utilities.assert_equals(expect = main.TRUE,actual=flow_check,
522 onpass="The flow count is consistent across all ONOS nodes",
523 onfail="ONOS nodes have different flow counts")
524
525
526 main.step("Get the OF Table entries")
527 global flows
528 flows=[]
529 for i in range(1,29):
530 flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
531
532 #TODO: Compare switch flow tables with ONOS flow tables
533
534 main.step("Start continuous pings")
535 main.Mininet2.pingLong(src=main.params['PING']['source1'],
536 target=main.params['PING']['target1'],pingTime=500)
537 main.Mininet2.pingLong(src=main.params['PING']['source2'],
538 target=main.params['PING']['target2'],pingTime=500)
539 main.Mininet2.pingLong(src=main.params['PING']['source3'],
540 target=main.params['PING']['target3'],pingTime=500)
541 main.Mininet2.pingLong(src=main.params['PING']['source4'],
542 target=main.params['PING']['target4'],pingTime=500)
543 main.Mininet2.pingLong(src=main.params['PING']['source5'],
544 target=main.params['PING']['target5'],pingTime=500)
545 main.Mininet2.pingLong(src=main.params['PING']['source6'],
546 target=main.params['PING']['target6'],pingTime=500)
547 main.Mininet2.pingLong(src=main.params['PING']['source7'],
548 target=main.params['PING']['target7'],pingTime=500)
549 main.Mininet2.pingLong(src=main.params['PING']['source8'],
550 target=main.params['PING']['target8'],pingTime=500)
551 main.Mininet2.pingLong(src=main.params['PING']['source9'],
552 target=main.params['PING']['target9'],pingTime=500)
553 main.Mininet2.pingLong(src=main.params['PING']['source10'],
554 target=main.params['PING']['target10'],pingTime=500)
555
556 main.step("Create TestONTopology object")
557 ctrls = []
558 count = 1
559 while True:
560 temp = ()
561 if ('ip' + str(count)) in main.params['CTRL']:
562 temp = temp + (getattr(main,('ONOS' + str(count))),)
563 temp = temp + ("ONOS"+str(count),)
564 temp = temp + (main.params['CTRL']['ip'+str(count)],)
565 temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
566 ctrls.append(temp)
567 count = count + 1
568 else:
569 break
570 MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
571
572 main.step("Collecting topology information from ONOS")
573 devices = []
574 devices.append( main.ONOScli1.devices() )
575 devices.append( main.ONOScli2.devices() )
576 devices.append( main.ONOScli3.devices() )
577 devices.append( main.ONOScli4.devices() )
578 devices.append( main.ONOScli5.devices() )
579 devices.append( main.ONOScli6.devices() )
580 devices.append( main.ONOScli7.devices() )
581 '''
582 hosts = []
583 hosts.append( main.ONOScli1.hosts() )
584 hosts.append( main.ONOScli2.hosts() )
585 hosts.append( main.ONOScli3.hosts() )
586 hosts.append( main.ONOScli4.hosts() )
587 hosts.append( main.ONOScli5.hosts() )
588 hosts.append( main.ONOScli6.hosts() )
589 hosts.append( main.ONOScli7.hosts() )
590 '''
591 ports = []
592 ports.append( main.ONOScli1.ports() )
593 ports.append( main.ONOScli2.ports() )
594 ports.append( main.ONOScli3.ports() )
595 ports.append( main.ONOScli4.ports() )
596 ports.append( main.ONOScli5.ports() )
597 ports.append( main.ONOScli6.ports() )
598 ports.append( main.ONOScli7.ports() )
599 links = []
600 links.append( main.ONOScli1.links() )
601 links.append( main.ONOScli2.links() )
602 links.append( main.ONOScli3.links() )
603 links.append( main.ONOScli4.links() )
604 links.append( main.ONOScli5.links() )
605 links.append( main.ONOScli6.links() )
606 links.append( main.ONOScli7.links() )
607
608
609 main.step("Comparing ONOS topology to MN")
610 devices_results = main.TRUE
611 ports_results = main.TRUE
612 links_results = main.TRUE
613 for controller in range(7): #TODO parameterize the number of controllers
614 if devices[controller] or not "Error" in devices[controller]:
615 current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
616 else:
617 current_devices_result = main.FALSE
618 utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
619 onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
620 onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
621
622 if ports[controller] or not "Error" in ports[controller]:
623 current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
624 else:
625 current_ports_result = main.FALSE
626 utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
627 onpass="ONOS"+str(int(controller+1))+" ports view is correct",
628 onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
629
630 if links[controller] or not "Error" in links[controller]:
631 current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
632 else:
633 current_links_result = main.FALSE
634 utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
635 onpass="ONOS"+str(int(controller+1))+" links view is correct",
636 onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
637
638 devices_results = devices_results and current_devices_result
639 ports_results = ports_results and current_ports_result
640 links_results = links_results and current_links_result
641
642 topo_result = devices_results and ports_results and links_results
643 utilities.assert_equals(expect=main.TRUE, actual=topo_result,
644 onpass="Topology Check Test successful",
645 onfail="Topology Check Test NOT successful")
646
647 final_assert = main.TRUE
648 final_assert = final_assert and topo_result and flow_check \
649 and intent_check and consistent_mastership
650 utilities.assert_equals(expect=main.TRUE, actual=final_assert,
651 onpass="State check successful",
652 onfail="State check NOT successful")
653
654
655 def CASE6(self,main) :
656 '''
657 The Failure case.
658 '''
659 main.log.report("Killing 3 ONOS nodes")
660 main.log.case("Restart minority of ONOS nodes")
661 #TODO: Randomize these nodes
662 main.ONOSbench.onos_kill(ONOS1_ip)
663 main.ONOSbench.onos_kill(ONOS2_ip)
664 main.ONOSbench.onos_kill(ONOS3_ip)
665
666 main.step("Checking if ONOS is up yet")
Jon Hallffb386d2014-11-21 13:43:38 -0800667 count = 0
668 onos_isup_result = main.FALSE
669 while onos_isup_result == main.FALSE and count < 10:
670 onos1_isup = main.ONOSbench.isup(ONOS1_ip)
671 onos2_isup = main.ONOSbench.isup(ONOS2_ip)
672 onos3_isup = main.ONOSbench.isup(ONOS3_ip)
673 onos_isup_result = onos1_isup and onos2_isup and onos3_isup
674 count = count + 1
Jon Hall73cf9cc2014-11-20 22:28:38 -0800675 # TODO: if it becomes an issue, we can retry this step a few times
676
677
678 cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
679 cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
680 cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
681 cli_results = cli_result1 and cli_result2 and cli_result3
682
683 case_results = main.TRUE and onos_isup_result and cli_results
684 utilities.assert_equals(expect=main.TRUE, actual=case_results,
685 onpass="ONOS restart successful",
686 onfail="ONOS restart NOT successful")
687
688
689 def CASE7(self,main) :
690 '''
691 Check state after ONOS failure
692 '''
693 import os
694 import json
695 main.case("Running ONOS Constant State Tests")
696
697 main.step("Check if switch roles are consistent across all nodes")
698 ONOS1_mastership = main.ONOScli1.roles()
699 ONOS2_mastership = main.ONOScli2.roles()
700 ONOS3_mastership = main.ONOScli3.roles()
701 ONOS4_mastership = main.ONOScli4.roles()
702 ONOS5_mastership = main.ONOScli5.roles()
703 ONOS6_mastership = main.ONOScli6.roles()
704 ONOS7_mastership = main.ONOScli7.roles()
705 print type(ONOS1_mastership)
706 print ONOS1_mastership
707 print type(ONOS2_mastership)
708 print ONOS2_mastership
709 print type(ONOS3_mastership)
710 print ONOS3_mastership
711 print type(ONOS4_mastership)
712 print ONOS4_mastership
713 print type(ONOS5_mastership)
714 print ONOS5_mastership
715 print type(ONOS6_mastership)
716 print ONOS6_mastership
717 print type(ONOS7_mastership)
718 print ONOS7_mastership
719 #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
720 if "Error" in ONOS1_mastership or not ONOS1_mastership\
721 or "Error" in ONOS2_mastership or not ONOS2_mastership\
722 or "Error" in ONOS3_mastership or not ONOS3_mastership\
723 or "Error" in ONOS4_mastership or not ONOS4_mastership\
724 or "Error" in ONOS5_mastership or not ONOS5_mastership\
725 or "Error" in ONOS6_mastership or not ONOS6_mastership\
726 or "Error" in ONOS7_mastership or not ONOS7_mastership:
727 main.log.error("Error in getting ONOS mastership")
728 main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
729 main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
730 main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
731 main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
732 main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
733 main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
734 main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
735 consistent_mastership = main.FALSE
736 elif ONOS1_mastership == ONOS2_mastership\
737 and ONOS1_mastership == ONOS3_mastership\
738 and ONOS1_mastership == ONOS4_mastership\
739 and ONOS1_mastership == ONOS5_mastership\
740 and ONOS1_mastership == ONOS6_mastership\
741 and ONOS1_mastership == ONOS7_mastership:
742 #mastership_state = ONOS1_mastership
743 consistent_mastership = main.TRUE
744 main.log.report("Switch roles are consistent across all ONOS nodes")
745 else:
746 main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
747 sort_keys=True, indent=4, separators=(',', ': ')))
748 main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
749 sort_keys=True, indent=4, separators=(',', ': ')))
750 main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
751 sort_keys=True, indent=4, separators=(',', ': ')))
752 main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
753 sort_keys=True, indent=4, separators=(',', ': ')))
754 main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
755 sort_keys=True, indent=4, separators=(',', ': ')))
756 main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
757 sort_keys=True, indent=4, separators=(',', ': ')))
758 main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
759 sort_keys=True, indent=4, separators=(',', ': ')))
760 consistent_mastership = main.FALSE
761 utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
762 onpass="Switch roles are consistent across all ONOS nodes",
763 onfail="ONOS nodes have different views of switch roles")
764
765
766 description2 = "Compare switch roles from before failure"
767 main.step(description2)
768
769 current_json = json.loads(ONOS1_mastership)
770 old_json = json.loads(mastership_state)
771 mastership_check = main.TRUE
772 for i in range(1,29):
773 switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
774
775 current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
776 old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
777 if current == old:
778 mastership_check = mastership_check and main.TRUE
779 else:
780 main.log.warn("Mastership of switch %s changed" % switchDPID)
781 mastership_check = main.FALSE
782 if mastership_check == main.TRUE:
783 main.log.report("Mastership of Switches was not changed")
784 utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
785 onpass="Mastership of Switches was not changed",
786 onfail="Mastership of some switches changed")
787 #NOTE: we expect mastership to change on controller failure
788 mastership_check = mastership_check #and consistent_mastership
789
790
791
792 main.step("Get the intents and compare across all nodes")
793 ONOS1_intents = main.ONOScli1.intents( json_format=True )
794 ONOS2_intents = main.ONOScli2.intents( json_format=True )
795 ONOS3_intents = main.ONOScli3.intents( json_format=True )
796 ONOS4_intents = main.ONOScli4.intents( json_format=True )
797 ONOS5_intents = main.ONOScli5.intents( json_format=True )
798 ONOS6_intents = main.ONOScli6.intents( json_format=True )
799 ONOS7_intents = main.ONOScli7.intents( json_format=True )
800 intent_check = main.FALSE
801 if "Error" in ONOS1_intents or not ONOS1_intents\
802 or "Error" in ONOS2_intents or not ONOS2_intents\
803 or "Error" in ONOS3_intents or not ONOS3_intents\
804 or "Error" in ONOS4_intents or not ONOS4_intents\
805 or "Error" in ONOS5_intents or not ONOS5_intents\
806 or "Error" in ONOS6_intents or not ONOS6_intents\
807 or "Error" in ONOS7_intents or not ONOS7_intents:
808 main.log.report("Error in getting ONOS intents")
809 main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
810 main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
811 main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
812 main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
813 main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
814 main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
815 main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
816 elif ONOS1_intents == ONOS2_intents\
817 and ONOS1_intents == ONOS3_intents\
818 and ONOS1_intents == ONOS4_intents\
819 and ONOS1_intents == ONOS5_intents\
820 and ONOS1_intents == ONOS6_intents\
821 and ONOS1_intents == ONOS7_intents:
822 intent_check = main.TRUE
823 main.log.report("Intents are consistent across all ONOS nodes")
824 else:
825 main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
826 sort_keys=True, indent=4, separators=(',', ': ')))
827 main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
828 sort_keys=True, indent=4, separators=(',', ': ')))
829 main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
830 sort_keys=True, indent=4, separators=(',', ': ')))
831 main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
832 sort_keys=True, indent=4, separators=(',', ': ')))
833 main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
834 sort_keys=True, indent=4, separators=(',', ': ')))
835 main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
836 sort_keys=True, indent=4, separators=(',', ': ')))
837 main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
838 sort_keys=True, indent=4, separators=(',', ': ')))
839 utilities.assert_equals(expect = main.TRUE,actual=intent_check,
840 onpass="Intents are consistent across all ONOS nodes",
841 onfail="ONOS nodes have different views of intents")
842
843 main.step("Compare current intents with intents before the failure")
844 if intent_state == ONOS1_intents:
845 same_intents = main.TRUE
846 main.log.report("Intents are consistent with before failure")
847 #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
848 else:
849 same_intents = main.FALSE
850 utilities.assert_equals(expect = main.TRUE,actual=same_intents,
851 onpass="Intents are consistent with before failure",
852 onfail="The Intents changed during failure")
853 intent_check = intent_check and same_intents
854
855
856
857 main.step("Get the OF Table entries and compare to before component failure")
858 Flow_Tables = main.TRUE
859 flows2=[]
860 for i in range(28):
861 main.log.info("Checking flow table on s" + str(i+1))
862 tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
863 flows2.append(tmp_flows)
864 Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
865 if Flow_Tables == main.FALSE:
866 main.log.info("Differences in flow table for switch: "+str(i+1))
867 break
868 if Flow_Tables == main.TRUE:
869 main.log.report("No changes were found in the flow tables")
870 utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
871 onpass="No changes were found in the flow tables",
872 onfail="Changes were found in the flow tables")
873
874 main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
875 #FIXME: This check is always failing. Investigate cause
876 #NOTE: this may be something to do with file permsissions
877 # or slight change in format
878 main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
879 Loss_In_Pings = main.FALSE
880 #NOTE: checkForLoss returns main.FALSE with 0% packet loss
881 for i in range(8,18):
882 main.log.info("Checking for a loss in pings along flow from s" + str(i))
883 Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
884 if Loss_In_Pings == main.TRUE:
885 main.log.info("Loss in ping detected")
886 elif Loss_In_Pings == main.ERROR:
887 main.log.info("There are multiple mininet process running")
888 elif Loss_In_Pings == main.FALSE:
889 main.log.info("No Loss in the pings")
890 main.log.report("No loss of dataplane connectivity")
891 utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
892 onpass="No Loss of connectivity",
893 onfail="Loss of dataplane connectivity detected")
894
895
896 #TODO:add topology to this or leave as a seperate case?
897 result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
898 result = int(result)
899 if result == main.TRUE:
900 main.log.report("Constant State Tests Passed")
901 utilities.assert_equals(expect=main.TRUE,actual=result,
902 onpass="Constant State Tests Passed",
903 onfail="Constant state tests failed")
904
905 def CASE8 (self,main):
906 '''
907 Compare topo
908 '''
909 import sys
910 sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
911 from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
912 import json
913 import time
914
915 description ="Compare ONOS Topology view to Mininet topology"
916 main.case(description)
917 main.log.report(description)
918 main.step("Create TestONTopology object")
919 ctrls = []
920 count = 1
921 while True:
922 temp = ()
923 if ('ip' + str(count)) in main.params['CTRL']:
924 temp = temp + (getattr(main,('ONOS' + str(count))),)
925 temp = temp + ("ONOS"+str(count),)
926 temp = temp + (main.params['CTRL']['ip'+str(count)],)
927 temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
928 ctrls.append(temp)
929 count = count + 1
930 else:
931 break
932 MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
933
Jon Hall73cf9cc2014-11-20 22:28:38 -0800934 main.step("Comparing ONOS topology to MN")
935 devices_results = main.TRUE
936 ports_results = main.TRUE
937 links_results = main.TRUE
938 topo_result = main.FALSE
939 start_time = time.time()
940 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -0800941 count = 0
Jon Hall73cf9cc2014-11-20 22:28:38 -0800942 while topo_result == main.FALSE and elapsed < 120:
Jon Hallffb386d2014-11-21 13:43:38 -0800943 count = count + 1
Jon Hall73cf9cc2014-11-20 22:28:38 -0800944 try:
Jon Hallffb386d2014-11-21 13:43:38 -0800945 main.step("Collecting topology information from ONOS")
946 devices = []
947 devices.append( main.ONOScli1.devices() )
948 devices.append( main.ONOScli2.devices() )
949 devices.append( main.ONOScli3.devices() )
950 devices.append( main.ONOScli4.devices() )
951 devices.append( main.ONOScli5.devices() )
952 devices.append( main.ONOScli6.devices() )
953 devices.append( main.ONOScli7.devices() )
954 '''
955 hosts = []
956 hosts.append( main.ONOScli1.hosts() )
957 hosts.append( main.ONOScli2.hosts() )
958 hosts.append( main.ONOScli3.hosts() )
959 hosts.append( main.ONOScli4.hosts() )
960 hosts.append( main.ONOScli5.hosts() )
961 hosts.append( main.ONOScli6.hosts() )
962 hosts.append( main.ONOScli7.hosts() )
963 '''
964 ports = []
965 ports.append( main.ONOScli1.ports() )
966 ports.append( main.ONOScli2.ports() )
967 ports.append( main.ONOScli3.ports() )
968 ports.append( main.ONOScli4.ports() )
969 ports.append( main.ONOScli5.ports() )
970 ports.append( main.ONOScli6.ports() )
971 ports.append( main.ONOScli7.ports() )
972 links = []
973 links.append( main.ONOScli1.links() )
974 links.append( main.ONOScli2.links() )
975 links.append( main.ONOScli3.links() )
976 links.append( main.ONOScli4.links() )
977 links.append( main.ONOScli5.links() )
978 links.append( main.ONOScli6.links() )
979 links.append( main.ONOScli7.links() )
980
Jon Hall73cf9cc2014-11-20 22:28:38 -0800981 for controller in range(7): #TODO parameterize the number of controllers
982 if devices[controller] or not "Error" in devices[controller]:
983 current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
984 else:
985 current_devices_result = main.FALSE
986 utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
987 onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
988 onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
989
990 if ports[controller] or not "Error" in ports[controller]:
991 current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
992 else:
993 current_ports_result = main.FALSE
994 utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
995 onpass="ONOS"+str(int(controller+1))+" ports view is correct",
996 onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
997
998 if links[controller] or not "Error" in links[controller]:
999 current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
1000 else:
1001 current_links_result = main.FALSE
1002 utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
1003 onpass="ONOS"+str(int(controller+1))+" links view is correct",
1004 onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
1005 except:
1006 main.log.error("something went wrong in topo comparison")
1007 main.log.warn( repr( devices ) )
1008 main.log.warn( repr( ports ) )
1009 main.log.warn( repr( links ) )
1010
1011 devices_results = devices_results and current_devices_result
1012 ports_results = ports_results and current_ports_result
1013 links_results = links_results and current_links_result
Jon Hallffb386d2014-11-21 13:43:38 -08001014 topo_result = devices_results and ports_results and links_results
1015 elapsed = time.time() - start_time
Jon Hall73cf9cc2014-11-20 22:28:38 -08001016 time_threshold = elapsed < 1
Jon Hallffb386d2014-11-21 13:43:38 -08001017 topo_result = topo_result and time_threshold
Jon Hall73cf9cc2014-11-20 22:28:38 -08001018 #TODO make sure this step is non-blocking. IE add a timeout
Jon Hallffb386d2014-11-21 13:43:38 -08001019 main.log.report("Very crass estimate for topology discovery/convergence: " +\
1020 str(elapsed) + " seconds, " + str(count) +" tries" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001021 utilities.assert_equals(expect=main.TRUE, actual=topo_result,
1022 onpass="Topology Check Test successful",
1023 onfail="Topology Check Test NOT successful")
1024 if topo_result == main.TRUE:
1025 main.log.report("ONOS topology view matches Mininet topology")
1026
1027
1028 def CASE9 (self,main):
1029 '''
1030 Link s3-s28 down
1031 '''
1032 #NOTE: You should probably run a topology check after this
1033
1034 link_sleep = int(main.params['timers']['LinkDiscovery'])
1035
1036 description = "Turn off a link to ensure that Link Discovery is working properly"
1037 main.log.report(description)
1038 main.case(description)
1039
1040
1041 main.step("Kill Link between s3 and s28")
1042 Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
1043 main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
1044 time.sleep(link_sleep)
1045 utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
1046 onpass="Link down succesful",
1047 onfail="Failed to bring link down")
1048 #TODO do some sort of check here
1049
1050 def CASE10 (self,main):
1051 '''
1052 Link s3-s28 up
1053 '''
1054 #NOTE: You should probably run a topology check after this
1055
1056 link_sleep = int(main.params['timers']['LinkDiscovery'])
1057
1058 description = "Restore a link to ensure that Link Discovery is working properly"
1059 main.log.report(description)
1060 main.case(description)
1061
1062 main.step("Bring link between s3 and s28 back up")
1063 Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
1064 main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
1065 time.sleep(link_sleep)
1066 utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
1067 onpass="Link up succesful",
1068 onfail="Failed to bring link up")
1069 #TODO do some sort of check here
1070
1071
1072 def CASE11 (self, main) :
1073 '''
1074 Switch Down
1075 '''
1076 #NOTE: You should probably run a topology check after this
1077 import time
1078
1079 switch_sleep = int(main.params['timers']['SwitchDiscovery'])
1080
1081 description = "Killing a switch to ensure it is discovered correctly"
1082 main.log.report(description)
1083 main.case(description)
1084
1085 #TODO: Make this switch parameterizable
1086 main.step("Kill s28 ")
1087 main.log.report("Deleting s28")
1088 #FIXME: use new dynamic topo functions
1089 main.Mininet1.del_switch("s28")
1090 main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
1091 time.sleep(switch_sleep)
1092 #Peek at the deleted switch
1093 main.log.warn(main.ONOScli1.get_device(dpid="0028"))
1094 #TODO do some sort of check here
1095
1096 def CASE12 (self, main) :
1097 '''
1098 Switch Up
1099 '''
1100 #NOTE: You should probably run a topology check after this
1101 import time
1102 #FIXME: use new dynamic topo functions
1103 description = "Adding a switch to ensure it is discovered correctly"
1104 main.log.report(description)
1105 main.case(description)
1106
1107 main.step("Add back s28")
1108 main.log.report("Adding back s28")
1109 main.Mininet1.add_switch("s28", dpid = '0000000000002800')
1110 #TODO: New dpid or same? Ask Thomas?
1111 main.Mininet1.add_link('s28', 's3')
1112 main.Mininet1.add_link('s28', 's6')
1113 main.Mininet1.add_link('s28', 'h28')
1114 main.Mininet1.assign_sw_controller(sw="28",count=7,
1115 ip1=ONOS1_ip,port1=ONOS1_port,
1116 ip2=ONOS2_ip,port2=ONOS2_port,
1117 ip3=ONOS3_ip,port3=ONOS3_port,
1118 ip4=ONOS4_ip,port4=ONOS4_port,
1119 ip5=ONOS5_ip,port5=ONOS5_port,
1120 ip6=ONOS6_ip,port6=ONOS6_port,
1121 ip7=ONOS7_ip,port7=ONOS7_port)
1122 main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
1123 time.sleep(switch_sleep)
1124 #Peek at the added switch
1125 main.log.warn(main.ONOScli1.get_device(dpid="0028"))
1126 #TODO do some sort of check here
1127
1128 def CASE13 (self, main) :
1129 '''
1130 Clean up
1131 '''
1132 import os
1133 import time
1134 description = "Test Cleanup"
1135 main.log.report(description)
1136 main.case(description)
1137 main.step("Killing tcpdumps")
1138 main.Mininet2.stop_tcpdump()
1139
1140 main.step("Copying MN pcap and ONOS log files to test station")
1141 testname = main.TEST
1142 #NOTE: MN Pcap file is being saved to ~/packet_captures
1143 # scp this file as MN and TestON aren't necessarily the same vm
1144 #FIXME: scp
1145 #####mn files
1146 #TODO: Load these from params
1147 #NOTE: must end in /
1148 log_folder = "/opt/onos/log/"
1149 log_files = ["karaf.log", "karaf.log.1"]
1150 #NOTE: must end in /
1151 dst_dir = "~/packet_captures/"
1152 for f in log_files:
1153 main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
1154 dst_dir + str(testname) + "-ONOS1-"+f )
1155 main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
1156 dst_dir + str(testname) + "-ONOS2-"+f )
1157 main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
1158 dst_dir + str(testname) + "-ONOS3-"+f )
1159 main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
1160 dst_dir + str(testname) + "-ONOS4-"+f )
1161 main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
1162 dst_dir + str(testname) + "-ONOS5-"+f )
1163 main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
1164 dst_dir + str(testname) + "-ONOS6-"+f )
1165 main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
1166 dst_dir + str(testname) + "-ONOS7-"+f )
1167
1168 #std*.log's
1169 #NOTE: must end in /
1170 log_folder = "/opt/onos/var/"
1171 log_files = ["stderr.log", "stdout.log"]
1172 #NOTE: must end in /
1173 dst_dir = "~/packet_captures/"
1174 for f in log_files:
1175 main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
1176 dst_dir + str(testname) + "-ONOS1-"+f )
1177 main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
1178 dst_dir + str(testname) + "-ONOS2-"+f )
1179 main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
1180 dst_dir + str(testname) + "-ONOS3-"+f )
1181 main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
1182 dst_dir + str(testname) + "-ONOS4-"+f )
1183 main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
1184 dst_dir + str(testname) + "-ONOS5-"+f )
1185 main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
1186 dst_dir + str(testname) + "-ONOS6-"+f )
1187 main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
1188 dst_dir + str(testname) + "-ONOS7-"+f )
1189
1190
1191
1192
1193 #sleep so scp can finish
1194 time.sleep(10)
1195 main.step("Packing and rotating pcap archives")
1196 os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
1197
1198
1199 #TODO: actually check something here
1200 utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
1201 onpass="Test cleanup successful",
1202 onfail="Test cleanup NOT successful")