1 """
2 Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5 List of test cases:
6 CASE1: Compile ONOS and push it to the test machines
7 CASE2: Assign devices to controllers
8 CASE21: Assign mastership to controllers
9 CASE3: Assign intents
10 CASE4: Ping across added host intents
11 CASE5: Reading state of ONOS
12 CASE6: The Failure case.
13 CASE7: Check state after control plane failure
14 CASE8: Compare topo
15 CASE9: Link s3-s28 down
16 CASE10: Link s3-s28 up
17 CASE11: Switch down
18 CASE12: Switch up
19 CASE13: Clean up
20 CASE14: start election app on all onos nodes
21 CASE15: Check that Leadership Election is still functional
22 CASE16: Install Distributed Primitives app
23 CASE17: Check for basic functionality with distributed primitives
24 """
25
26
28
31
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
50 import imp
51 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
52 "initialization" )
53 main.case( "Setting up test environment" )
54 main.caseExplanation = "Setup the test environment including " +\
55 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57
58
59
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
66 main.numCtrls = int( main.params[ 'num_controllers' ] )
67 if main.ONOSbench.maxNodes:
68 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
70
71 global ONOS1Port
72 global ONOS2Port
73 global ONOS3Port
74 global ONOS4Port
75 global ONOS5Port
76 global ONOS6Port
77 global ONOS7Port
78
79 global labels
80 global data
81 labels = []
82 data = []
83
84
85
86 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
87 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
88 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
89 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
90 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
91 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
92 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
93
94 try:
95 fileName = "Counters"
96 path = main.params[ 'imports' ][ 'path' ]
97 main.Counters = imp.load_source( fileName,
98 path + fileName + ".py" )
99 except Exception as e:
100 main.log.exception( e )
101 main.cleanup()
102 main.exit()
103
104 main.CLIs = []
105 main.nodes = []
106 ipList = []
107 for i in range( 1, main.numCtrls + 1 ):
108 try:
109 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
110 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
111 ipList.append( main.nodes[ -1 ].ip_address )
112 except AttributeError:
113 break
114
115 main.step( "Create cell file" )
116 cellAppString = main.params[ 'ENV' ][ 'appString' ]
117 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
118 main.Mininet1.ip_address,
119 cellAppString, ipList )
120 main.step( "Applying cell variable to environment" )
121 cellResult = main.ONOSbench.setCell( cellName )
122 verifyResult = main.ONOSbench.verifyCell()
123
124
125 main.log.info( "Removing raft logs" )
126 main.ONOSbench.onosRemoveRaftLogs()
127
128 main.log.info( "Uninstalling ONOS" )
129 for node in main.nodes:
130 main.ONOSbench.onosUninstall( node.ip_address )
131
132
133 main.log.info( "Killing any ONOS processes" )
134 killResults = main.TRUE
135 for node in main.nodes:
136 killed = main.ONOSbench.onosKill( node.ip_address )
137 killResults = killResults and killed
138
139 cleanInstallResult = main.TRUE
140 gitPullResult = main.TRUE
141
142 main.step( "Starting Mininet" )
143
144
145 topoName = "obelisk.py"
146 filePath = main.ONOSbench.home + "/tools/test/topos/"
147 main.ONOSbench.copyMininetFile( topoName, filePath,
148 main.Mininet1.user_name,
149 main.Mininet1.ip_address )
150 mnResult = main.Mininet1.startNet( )
151 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
152 onpass="Mininet Started",
153 onfail="Error starting Mininet" )
154
155 main.step( "Git checkout and pull " + gitBranch )
156 if PULLCODE:
157 main.ONOSbench.gitCheckout( gitBranch )
158 gitPullResult = main.ONOSbench.gitPull()
159
160 utilities.assert_lesser( expect=0, actual=gitPullResult,
161 onpass="Git pull successful",
162 onfail="Git pull failed" )
163 main.ONOSbench.getVersion( report=True )
164
165 main.step( "Using mvn clean install" )
166 cleanInstallResult = main.TRUE
167 if PULLCODE and gitPullResult == main.TRUE:
168 cleanInstallResult = main.ONOSbench.cleanInstall()
169 else:
170 main.log.warn( "Did not pull new code so skipping mvn " +
171 "clean install" )
172 utilities.assert_equals( expect=main.TRUE,
173 actual=cleanInstallResult,
174 onpass="MCI successful",
175 onfail="MCI failed" )
176
177
178
179
180
181 job = "HAclusterRestart"
182 plotName = "Plot-HA"
183 graphs = '<ac:structured-macro ac:name="html">\n'
184 graphs += '<ac:plain-text-body><![CDATA[\n'
185 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
186 '/plot/' + plotName + '/getPlot?index=0' +\
187 '&width=500&height=300"' +\
188 'noborder="0" width="500" height="300" scrolling="yes" ' +\
189 'seamless="seamless"></iframe>\n'
190 graphs += ']]></ac:plain-text-body>\n'
191 graphs += '</ac:structured-macro>\n'
192 main.log.wiki(graphs)
193
194 main.step( "Creating ONOS package" )
195 packageResult = main.ONOSbench.onosPackage()
196 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
197 onpass="ONOS package successful",
198 onfail="ONOS package failed" )
199
200 main.step( "Installing ONOS package" )
201 onosInstallResult = main.TRUE
202 for node in main.nodes:
203 tmpResult = main.ONOSbench.onosInstall( options="-f",
204 node=node.ip_address )
205 onosInstallResult = onosInstallResult and tmpResult
206 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
207 onpass="ONOS install successful",
208 onfail="ONOS install failed" )
209
210 main.step( "Checking if ONOS is up yet" )
211 for i in range( 2 ):
212 onosIsupResult = main.TRUE
213 for node in main.nodes:
214 started = main.ONOSbench.isup( node.ip_address )
215 if not started:
216 main.log.error( node.name + " didn't start!" )
217 main.ONOSbench.onosStop( node.ip_address )
218 main.ONOSbench.onosStart( node.ip_address )
219 onosIsupResult = onosIsupResult and started
220 if onosIsupResult == main.TRUE:
221 break
222 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
223 onpass="ONOS startup successful",
224 onfail="ONOS startup failed" )
225
226 main.log.step( "Starting ONOS CLI sessions" )
227 cliResults = main.TRUE
228 threads = []
229 for i in range( main.numCtrls ):
230 t = main.Thread( target=main.CLIs[i].startOnosCli,
231 name="startOnosCli-" + str( i ),
232 args=[main.nodes[i].ip_address] )
233 threads.append( t )
234 t.start()
235
236 for t in threads:
237 t.join()
238 cliResults = cliResults and t.result
239 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
240 onpass="ONOS cli startup successful",
241 onfail="ONOS cli startup failed" )
242
243 if main.params[ 'tcpdump' ].lower() == "true":
244 main.step( "Start Packet Capture MN" )
245 main.Mininet2.startTcpdump(
246 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
247 + "-MN.pcap",
248 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
249 port=main.params[ 'MNtcpdump' ][ 'port' ] )
250
251 main.step( "App Ids check" )
252 appCheck = main.TRUE
253 threads = []
254 for i in range( main.numCtrls ):
255 t = main.Thread( target=main.CLIs[i].appToIDCheck,
256 name="appToIDCheck-" + str( i ),
257 args=[] )
258 threads.append( t )
259 t.start()
260
261 for t in threads:
262 t.join()
263 appCheck = appCheck and t.result
264 if appCheck != main.TRUE:
265 main.log.warn( main.CLIs[0].apps() )
266 main.log.warn( main.CLIs[0].appIDs() )
267 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
268 onpass="App Ids seem to be correct",
269 onfail="Something is wrong with app Ids" )
270
271 if cliResults == main.FALSE:
272 main.log.error( "Failed to start ONOS, stopping test" )
273 main.cleanup()
274 main.exit()
275
276 - def CASE2( self, main ):
277 """
278 Assign devices to controllers
279 """
280 import re
281 assert main.numCtrls, "main.numCtrls not defined"
282 assert main, "main not defined"
283 assert utilities.assert_equals, "utilities.assert_equals not defined"
284 assert main.CLIs, "main.CLIs not defined"
285 assert main.nodes, "main.nodes not defined"
286 assert ONOS1Port, "ONOS1Port not defined"
287 assert ONOS2Port, "ONOS2Port not defined"
288 assert ONOS3Port, "ONOS3Port not defined"
289 assert ONOS4Port, "ONOS4Port not defined"
290 assert ONOS5Port, "ONOS5Port not defined"
291 assert ONOS6Port, "ONOS6Port not defined"
292 assert ONOS7Port, "ONOS7Port not defined"
293
294 main.case( "Assigning devices to controllers" )
295 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
296 "and check that an ONOS node becomes the " +\
297 "master of the device."
298 main.step( "Assign switches to controllers" )
299
300 ipList = []
301 for i in range( main.numCtrls ):
302 ipList.append( main.nodes[ i ].ip_address )
303 swList = []
304 for i in range( 1, 29 ):
305 swList.append( "s" + str( i ) )
306 main.Mininet1.assignSwController( sw=swList, ip=ipList )
307
308 mastershipCheck = main.TRUE
309 for i in range( 1, 29 ):
310 response = main.Mininet1.getSwController( "s" + str( i ) )
311 try:
312 main.log.info( str( response ) )
313 except Exception:
314 main.log.info( repr( response ) )
315 for node in main.nodes:
316 if re.search( "tcp:" + node.ip_address, response ):
317 mastershipCheck = mastershipCheck and main.TRUE
318 else:
319 main.log.error( "Error, node " + node.ip_address + " is " +
320 "not in the list of controllers s" +
321 str( i ) + " is connecting to." )
322 mastershipCheck = main.FALSE
323 utilities.assert_equals(
324 expect=main.TRUE,
325 actual=mastershipCheck,
326 onpass="Switch mastership assigned correctly",
327 onfail="Switches not assigned correctly to controllers" )
328
330 """
331 Assign mastership to controllers
332 """
333 import time
334 assert main.numCtrls, "main.numCtrls not defined"
335 assert main, "main not defined"
336 assert utilities.assert_equals, "utilities.assert_equals not defined"
337 assert main.CLIs, "main.CLIs not defined"
338 assert main.nodes, "main.nodes not defined"
339 assert ONOS1Port, "ONOS1Port not defined"
340 assert ONOS2Port, "ONOS2Port not defined"
341 assert ONOS3Port, "ONOS3Port not defined"
342 assert ONOS4Port, "ONOS4Port not defined"
343 assert ONOS5Port, "ONOS5Port not defined"
344 assert ONOS6Port, "ONOS6Port not defined"
345 assert ONOS7Port, "ONOS7Port not defined"
346
347 main.case( "Assigning Controller roles for switches" )
348 main.caseExplanation = "Check that ONOS is connected to each " +\
349 "device. Then manually assign" +\
350 " mastership to specific ONOS nodes using" +\
351 " 'device-role'"
352 main.step( "Assign mastership of switches to specific controllers" )
353
354 roleCall = main.TRUE
355
356 ipList = [ ]
357 deviceList = []
358 try:
359
360
361
362 for i in range( 1, 29 ):
363
364 if i == 1:
365 c = 0
366 ip = main.nodes[ c ].ip_address
367 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
368 elif i == 2:
369 c = 1 % main.numCtrls
370 ip = main.nodes[ c ].ip_address
371 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
372 elif i == 3:
373 c = 1 % main.numCtrls
374 ip = main.nodes[ c ].ip_address
375 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
376 elif i == 4:
377 c = 3 % main.numCtrls
378 ip = main.nodes[ c ].ip_address
379 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
380 elif i == 5:
381 c = 2 % main.numCtrls
382 ip = main.nodes[ c ].ip_address
383 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
384 elif i == 6:
385 c = 2 % main.numCtrls
386 ip = main.nodes[ c ].ip_address
387 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
388 elif i == 7:
389 c = 5 % main.numCtrls
390 ip = main.nodes[ c ].ip_address
391 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
392 elif i >= 8 and i <= 17:
393 c = 4 % main.numCtrls
394 ip = main.nodes[ c ].ip_address
395 dpid = '3' + str( i ).zfill( 3 )
396 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
397 elif i >= 18 and i <= 27:
398 c = 6 % main.numCtrls
399 ip = main.nodes[ c ].ip_address
400 dpid = '6' + str( i ).zfill( 3 )
401 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
402 elif i == 28:
403 c = 0
404 ip = main.nodes[ c ].ip_address
405 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
406 else:
407 main.log.error( "You didn't write an else statement for " +
408 "switch s" + str( i ) )
409 roleCall = main.FALSE
410
411 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
412
413 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
414 ip )
415 ipList.append( ip )
416 deviceList.append( deviceId )
417 except ( AttributeError, AssertionError ):
418 main.log.exception( "Something is wrong with ONOS device view" )
419 main.log.info( main.ONOScli1.devices() )
420 utilities.assert_equals(
421 expect=main.TRUE,
422 actual=roleCall,
423 onpass="Re-assigned switch mastership to designated controller",
424 onfail="Something wrong with deviceRole calls" )
425
426 main.step( "Check mastership was correctly assigned" )
427 roleCheck = main.TRUE
428
429
430 time.sleep( 5 )
431 for i in range( len( ipList ) ):
432 ip = ipList[i]
433 deviceId = deviceList[i]
434
435 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
436 if ip in master:
437 roleCheck = roleCheck and main.TRUE
438 else:
439 roleCheck = roleCheck and main.FALSE
440 main.log.error( "Error, controller " + ip + " is not" +
441 " master " + "of device " +
442 str( deviceId ) + ". Master is " +
443 repr( master ) + "." )
444 utilities.assert_equals(
445 expect=main.TRUE,
446 actual=roleCheck,
447 onpass="Switches were successfully reassigned to designated " +
448 "controller",
449 onfail="Switches were not successfully reassigned" )
450
451 - def CASE3( self, main ):
452 """
453 Assign intents
454 """
455 import time
456 import json
457 assert main.numCtrls, "main.numCtrls not defined"
458 assert main, "main not defined"
459 assert utilities.assert_equals, "utilities.assert_equals not defined"
460 assert main.CLIs, "main.CLIs not defined"
461 assert main.nodes, "main.nodes not defined"
462 try:
463 labels
464 except NameError:
465 main.log.error( "labels not defined, setting to []" )
466 labels = []
467 try:
468 data
469 except NameError:
470 main.log.error( "data not defined, setting to []" )
471 data = []
472
473
474 main.case( "Adding host Intents" )
475 main.caseExplanation = "Discover hosts by using pingall then " +\
476 "assign predetermined host-to-host intents." +\
477 " After installation, check that the intent" +\
478 " is distributed to all nodes and the state" +\
479 " is INSTALLED"
480
481
482 main.step( "Install reactive forwarding app" )
483 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
484 utilities.assert_equals( expect=main.TRUE, actual=installResults,
485 onpass="Install fwd successful",
486 onfail="Install fwd failed" )
487
488 main.step( "Check app ids" )
489 appCheck = main.TRUE
490 threads = []
491 for i in range( main.numCtrls ):
492 t = main.Thread( target=main.CLIs[i].appToIDCheck,
493 name="appToIDCheck-" + str( i ),
494 args=[] )
495 threads.append( t )
496 t.start()
497
498 for t in threads:
499 t.join()
500 appCheck = appCheck and t.result
501 if appCheck != main.TRUE:
502 main.log.warn( main.CLIs[0].apps() )
503 main.log.warn( main.CLIs[0].appIDs() )
504 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
505 onpass="App Ids seem to be correct",
506 onfail="Something is wrong with app Ids" )
507
508 main.step( "Discovering Hosts( Via pingall for now )" )
509
510
511 pingResult = main.FALSE
512 for i in range(2):
513 time1 = time.time()
514 pingResult = main.Mininet1.pingall()
515 if i == 0:
516 utilities.assert_equals(
517 expect=main.TRUE,
518 actual=pingResult,
519 onpass="Reactive Pingall test passed",
520 onfail="Reactive Pingall failed, " +
521 "one or more ping pairs failed" )
522 time2 = time.time()
523 main.log.info( "Time for pingall: %2f seconds" %
524 ( time2 - time1 ) )
525
526 time.sleep( 11 )
527
528 main.step( "Uninstall reactive forwarding app" )
529 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
530 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
531 onpass="Uninstall fwd successful",
532 onfail="Uninstall fwd failed" )
533
534 main.step( "Check app ids" )
535 threads = []
536 appCheck2 = main.TRUE
537 for i in range( main.numCtrls ):
538 t = main.Thread( target=main.CLIs[i].appToIDCheck,
539 name="appToIDCheck-" + str( i ),
540 args=[] )
541 threads.append( t )
542 t.start()
543
544 for t in threads:
545 t.join()
546 appCheck2 = appCheck2 and t.result
547 if appCheck2 != main.TRUE:
548 main.log.warn( main.CLIs[0].apps() )
549 main.log.warn( main.CLIs[0].appIDs() )
550 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
551 onpass="App Ids seem to be correct",
552 onfail="Something is wrong with app Ids" )
553
554 main.step( "Add host intents via cli" )
555 intentIds = []
556
557
558 intentAddResult = True
559 hostResult = main.TRUE
560 for i in range( 8, 18 ):
561 main.log.info( "Adding host intent between h" + str( i ) +
562 " and h" + str( i + 10 ) )
563 host1 = "00:00:00:00:00:" + \
564 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
565 host2 = "00:00:00:00:00:" + \
566 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
567
568 host1Dict = main.ONOScli1.getHost( host1 )
569 host2Dict = main.ONOScli1.getHost( host2 )
570 host1Id = None
571 host2Id = None
572 if host1Dict and host2Dict:
573 host1Id = host1Dict.get( 'id', None )
574 host2Id = host2Dict.get( 'id', None )
575 if host1Id and host2Id:
576 nodeNum = ( i % main.numCtrls )
577 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
578 if tmpId:
579 main.log.info( "Added intent with id: " + tmpId )
580 intentIds.append( tmpId )
581 else:
582 main.log.error( "addHostIntent returned: " +
583 repr( tmpId ) )
584 else:
585 main.log.error( "Error, getHost() failed for h" + str( i ) +
586 " and/or h" + str( i + 10 ) )
587 hosts = main.CLIs[ 0 ].hosts()
588 main.log.warn( "Hosts output: " )
589 try:
590 main.log.warn( json.dumps( json.loads( hosts ),
591 sort_keys=True,
592 indent=4,
593 separators=( ',', ': ' ) ) )
594 except ( ValueError, TypeError ):
595 main.log.warn( repr( hosts ) )
596 hostResult = main.FALSE
597 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
598 onpass="Found a host id for each host",
599 onfail="Error looking up host ids" )
600
601 intentStart = time.time()
602 onosIds = main.ONOScli1.getAllIntentsId()
603 main.log.info( "Submitted intents: " + str( intentIds ) )
604 main.log.info( "Intents in ONOS: " + str( onosIds ) )
605 for intent in intentIds:
606 if intent in onosIds:
607 pass
608 else:
609 intentAddResult = False
610 if intentAddResult:
611 intentStop = time.time()
612 else:
613 intentStop = None
614
615 intents = main.ONOScli1.intents()
616 intentStates = []
617 installedCheck = True
618 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
619 count = 0
620 try:
621 for intent in json.loads( intents ):
622 state = intent.get( 'state', None )
623 if "INSTALLED" not in state:
624 installedCheck = False
625 intentId = intent.get( 'id', None )
626 intentStates.append( ( intentId, state ) )
627 except ( ValueError, TypeError ):
628 main.log.exception( "Error parsing intents" )
629
630 tmplist = [ i for i, s in intentStates ]
631 missingIntents = False
632 for i in intentIds:
633 if i not in tmplist:
634 intentStates.append( ( i, " - " ) )
635 missingIntents = True
636 intentStates.sort()
637 for i, s in intentStates:
638 count += 1
639 main.log.info( "%-6s%-15s%-15s" %
640 ( str( count ), str( i ), str( s ) ) )
641 leaders = main.ONOScli1.leaders()
642 try:
643 missing = False
644 if leaders:
645 parsedLeaders = json.loads( leaders )
646 main.log.warn( json.dumps( parsedLeaders,
647 sort_keys=True,
648 indent=4,
649 separators=( ',', ': ' ) ) )
650
651 topics = []
652 for i in range( 14 ):
653 topics.append( "intent-partition-" + str( i ) )
654 main.log.debug( topics )
655 ONOStopics = [ j['topic'] for j in parsedLeaders ]
656 for topic in topics:
657 if topic not in ONOStopics:
658 main.log.error( "Error: " + topic +
659 " not in leaders" )
660 missing = True
661 else:
662 main.log.error( "leaders() returned None" )
663 except ( ValueError, TypeError ):
664 main.log.exception( "Error parsing leaders" )
665 main.log.error( repr( leaders ) )
666
667 if missing:
668 for node in main.CLIs:
669 response = node.leaders( jsonFormat=False)
670 main.log.warn( str( node.name ) + " leaders output: \n" +
671 str( response ) )
672
673 partitions = main.ONOScli1.partitions()
674 try:
675 if partitions :
676 parsedPartitions = json.loads( partitions )
677 main.log.warn( json.dumps( parsedPartitions,
678 sort_keys=True,
679 indent=4,
680 separators=( ',', ': ' ) ) )
681
682
683 else:
684 main.log.error( "partitions() returned None" )
685 except ( ValueError, TypeError ):
686 main.log.exception( "Error parsing partitions" )
687 main.log.error( repr( partitions ) )
688 pendingMap = main.ONOScli1.pendingMap()
689 try:
690 if pendingMap :
691 parsedPending = json.loads( pendingMap )
692 main.log.warn( json.dumps( parsedPending,
693 sort_keys=True,
694 indent=4,
695 separators=( ',', ': ' ) ) )
696
697 else:
698 main.log.error( "pendingMap() returned None" )
699 except ( ValueError, TypeError ):
700 main.log.exception( "Error parsing pending map" )
701 main.log.error( repr( pendingMap ) )
702
703 intentAddResult = bool( intentAddResult and not missingIntents and
704 installedCheck )
705 if not intentAddResult:
706 main.log.error( "Error in pushing host intents to ONOS" )
707
708 main.step( "Intent Anti-Entropy dispersion" )
709 for i in range(100):
710 correct = True
711 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
712 for cli in main.CLIs:
713 onosIds = []
714 ids = cli.getAllIntentsId()
715 onosIds.append( ids )
716 main.log.debug( "Intents in " + cli.name + ": " +
717 str( sorted( onosIds ) ) )
718 if sorted( ids ) != sorted( intentIds ):
719 main.log.warn( "Set of intent IDs doesn't match" )
720 correct = False
721 break
722 else:
723 intents = json.loads( cli.intents() )
724 for intent in intents:
725 if intent[ 'state' ] != "INSTALLED":
726 main.log.warn( "Intent " + intent[ 'id' ] +
727 " is " + intent[ 'state' ] )
728 correct = False
729 break
730 if correct:
731 break
732 else:
733 time.sleep(1)
734 if not intentStop:
735 intentStop = time.time()
736 global gossipTime
737 gossipTime = intentStop - intentStart
738 main.log.info( "It took about " + str( gossipTime ) +
739 " seconds for all intents to appear in each node" )
740 append = False
741 title = "Gossip Intents"
742 count = 1
743 while append is False:
744 curTitle = title + str( count )
745 if curTitle not in labels:
746 labels.append( curTitle )
747 data.append( str( gossipTime ) )
748 append = True
749 else:
750 count += 1
751
752
753 utilities.assert_greater_equals(
754 expect=40, actual=gossipTime,
755 onpass="ECM anti-entropy for intents worked within " +
756 "expected time",
757 onfail="Intent ECM anti-entropy took too long" )
758 if gossipTime <= 40:
759 intentAddResult = True
760
761 if not intentAddResult or "key" in pendingMap:
762 import time
763 installedCheck = True
764 main.log.info( "Sleeping 60 seconds to see if intents are found" )
765 time.sleep( 60 )
766 onosIds = main.ONOScli1.getAllIntentsId()
767 main.log.info( "Submitted intents: " + str( intentIds ) )
768 main.log.info( "Intents in ONOS: " + str( onosIds ) )
769
770 intents = main.ONOScli1.intents()
771 intentStates = []
772 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
773 count = 0
774 try:
775 for intent in json.loads( intents ):
776
777 state = intent.get( 'state', None )
778 if "INSTALLED" not in state:
779 installedCheck = False
780 intentId = intent.get( 'id', None )
781 intentStates.append( ( intentId, state ) )
782 except ( ValueError, TypeError ):
783 main.log.exception( "Error parsing intents" )
784
785 tmplist = [ i for i, s in intentStates ]
786 for i in intentIds:
787 if i not in tmplist:
788 intentStates.append( ( i, " - " ) )
789 intentStates.sort()
790 for i, s in intentStates:
791 count += 1
792 main.log.info( "%-6s%-15s%-15s" %
793 ( str( count ), str( i ), str( s ) ) )
794 leaders = main.ONOScli1.leaders()
795 try:
796 missing = False
797 if leaders:
798 parsedLeaders = json.loads( leaders )
799 main.log.warn( json.dumps( parsedLeaders,
800 sort_keys=True,
801 indent=4,
802 separators=( ',', ': ' ) ) )
803
804
805 topics = []
806 for i in range( 14 ):
807 topics.append( "intent-partition-" + str( i ) )
808
809 topics.append( "org.onosproject.election" )
810 main.log.debug( topics )
811 ONOStopics = [ j['topic'] for j in parsedLeaders ]
812 for topic in topics:
813 if topic not in ONOStopics:
814 main.log.error( "Error: " + topic +
815 " not in leaders" )
816 missing = True
817 else:
818 main.log.error( "leaders() returned None" )
819 except ( ValueError, TypeError ):
820 main.log.exception( "Error parsing leaders" )
821 main.log.error( repr( leaders ) )
822
823 if missing:
824 for node in main.CLIs:
825 response = node.leaders( jsonFormat=False)
826 main.log.warn( str( node.name ) + " leaders output: \n" +
827 str( response ) )
828
829 partitions = main.ONOScli1.partitions()
830 try:
831 if partitions :
832 parsedPartitions = json.loads( partitions )
833 main.log.warn( json.dumps( parsedPartitions,
834 sort_keys=True,
835 indent=4,
836 separators=( ',', ': ' ) ) )
837
838
839 else:
840 main.log.error( "partitions() returned None" )
841 except ( ValueError, TypeError ):
842 main.log.exception( "Error parsing partitions" )
843 main.log.error( repr( partitions ) )
844 pendingMap = main.ONOScli1.pendingMap()
845 try:
846 if pendingMap :
847 parsedPending = json.loads( pendingMap )
848 main.log.warn( json.dumps( parsedPending,
849 sort_keys=True,
850 indent=4,
851 separators=( ',', ': ' ) ) )
852
853 else:
854 main.log.error( "pendingMap() returned None" )
855 except ( ValueError, TypeError ):
856 main.log.exception( "Error parsing pending map" )
857 main.log.error( repr( pendingMap ) )
858
859 - def CASE4( self, main ):
860 """
861 Ping across added host intents
862 """
863 import json
864 import time
865 assert main.numCtrls, "main.numCtrls not defined"
866 assert main, "main not defined"
867 assert utilities.assert_equals, "utilities.assert_equals not defined"
868 assert main.CLIs, "main.CLIs not defined"
869 assert main.nodes, "main.nodes not defined"
870 main.case( "Verify connectivity by sendind traffic across Intents" )
871 main.caseExplanation = "Ping across added host intents to check " +\
872 "functionality and check the state of " +\
873 "the intent"
874 main.step( "Ping across added host intents" )
875 PingResult = main.TRUE
876 for i in range( 8, 18 ):
877 ping = main.Mininet1.pingHost( src="h" + str( i ),
878 target="h" + str( i + 10 ) )
879 PingResult = PingResult and ping
880 if ping == main.FALSE:
881 main.log.warn( "Ping failed between h" + str( i ) +
882 " and h" + str( i + 10 ) )
883 elif ping == main.TRUE:
884 main.log.info( "Ping test passed!" )
885
886 if PingResult == main.FALSE:
887 main.log.error(
888 "Intents have not been installed correctly, pings failed." )
889
890 main.log.warn( "ONOS1 intents: " )
891 try:
892 tmpIntents = main.ONOScli1.intents()
893 main.log.warn( json.dumps( json.loads( tmpIntents ),
894 sort_keys=True,
895 indent=4,
896 separators=( ',', ': ' ) ) )
897 except ( ValueError, TypeError ):
898 main.log.warn( repr( tmpIntents ) )
899 utilities.assert_equals(
900 expect=main.TRUE,
901 actual=PingResult,
902 onpass="Intents have been installed correctly and pings work",
903 onfail="Intents have not been installed correctly, pings failed." )
904
905 main.step( "Check Intent state" )
906 installedCheck = False
907 loopCount = 0
908 while not installedCheck and loopCount < 40:
909 installedCheck = True
910
911 intents = main.ONOScli1.intents()
912 intentStates = []
913 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
914 count = 0
915
916 try:
917 for intent in json.loads( intents ):
918 state = intent.get( 'state', None )
919 if "INSTALLED" not in state:
920 installedCheck = False
921 intentId = intent.get( 'id', None )
922 intentStates.append( ( intentId, state ) )
923 except ( ValueError, TypeError ):
924 main.log.exception( "Error parsing intents." )
925
926 intentStates.sort()
927 for i, s in intentStates:
928 count += 1
929 main.log.info( "%-6s%-15s%-15s" %
930 ( str( count ), str( i ), str( s ) ) )
931 if not installedCheck:
932 time.sleep( 1 )
933 loopCount += 1
934 utilities.assert_equals( expect=True, actual=installedCheck,
935 onpass="Intents are all INSTALLED",
936 onfail="Intents are not all in " +
937 "INSTALLED state" )
938
939 main.step( "Check leadership of topics" )
940 leaders = main.ONOScli1.leaders()
941 topicCheck = main.TRUE
942 try:
943 if leaders:
944 parsedLeaders = json.loads( leaders )
945 main.log.warn( json.dumps( parsedLeaders,
946 sort_keys=True,
947 indent=4,
948 separators=( ',', ': ' ) ) )
949
950
951
952 topics = []
953 for i in range( 14 ):
954 topics.append( "intent-partition-" + str( i ) )
955
956
957
958 main.log.debug( topics )
959 ONOStopics = [ j['topic'] for j in parsedLeaders ]
960 for topic in topics:
961 if topic not in ONOStopics:
962 main.log.error( "Error: " + topic +
963 " not in leaders" )
964 topicCheck = main.FALSE
965 else:
966 main.log.error( "leaders() returned None" )
967 topicCheck = main.FALSE
968 except ( ValueError, TypeError ):
969 topicCheck = main.FALSE
970 main.log.exception( "Error parsing leaders" )
971 main.log.error( repr( leaders ) )
972
973
974 if topicCheck:
975 for node in main.CLIs:
976 response = node.leaders( jsonFormat=False)
977 main.log.warn( str( node.name ) + " leaders output: \n" +
978 str( response ) )
979
980 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
981 onpass="intent Partitions is in leaders",
982 onfail="Some topics were lost " )
983
984 partitions = main.ONOScli1.partitions()
985 try:
986 if partitions :
987 parsedPartitions = json.loads( partitions )
988 main.log.warn( json.dumps( parsedPartitions,
989 sort_keys=True,
990 indent=4,
991 separators=( ',', ': ' ) ) )
992
993
994 else:
995 main.log.error( "partitions() returned None" )
996 except ( ValueError, TypeError ):
997 main.log.exception( "Error parsing partitions" )
998 main.log.error( repr( partitions ) )
999
1000 pendingMap = main.ONOScli1.pendingMap()
1001 try:
1002 if pendingMap :
1003 parsedPending = json.loads( pendingMap )
1004 main.log.warn( json.dumps( parsedPending,
1005 sort_keys=True,
1006 indent=4,
1007 separators=( ',', ': ' ) ) )
1008
1009 else:
1010 main.log.error( "pendingMap() returned None" )
1011 except ( ValueError, TypeError ):
1012 main.log.exception( "Error parsing pending map" )
1013 main.log.error( repr( pendingMap ) )
1014
1015 if not installedCheck:
1016 main.log.info( "Waiting 60 seconds to see if the state of " +
1017 "intents change" )
1018 time.sleep( 60 )
1019
1020 intents = main.ONOScli1.intents()
1021 intentStates = []
1022 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1023 count = 0
1024
1025 try:
1026 for intent in json.loads( intents ):
1027 state = intent.get( 'state', None )
1028 if "INSTALLED" not in state:
1029 installedCheck = False
1030 intentId = intent.get( 'id', None )
1031 intentStates.append( ( intentId, state ) )
1032 except ( ValueError, TypeError ):
1033 main.log.exception( "Error parsing intents." )
1034 intentStates.sort()
1035 for i, s in intentStates:
1036 count += 1
1037 main.log.info( "%-6s%-15s%-15s" %
1038 ( str( count ), str( i ), str( s ) ) )
1039 leaders = main.ONOScli1.leaders()
1040 try:
1041 missing = False
1042 if leaders:
1043 parsedLeaders = json.loads( leaders )
1044 main.log.warn( json.dumps( parsedLeaders,
1045 sort_keys=True,
1046 indent=4,
1047 separators=( ',', ': ' ) ) )
1048
1049
1050 topics = []
1051 for i in range( 14 ):
1052 topics.append( "intent-partition-" + str( i ) )
1053
1054 topics.append( "org.onosproject.election" )
1055 main.log.debug( topics )
1056 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1057 for topic in topics:
1058 if topic not in ONOStopics:
1059 main.log.error( "Error: " + topic +
1060 " not in leaders" )
1061 missing = True
1062 else:
1063 main.log.error( "leaders() returned None" )
1064 except ( ValueError, TypeError ):
1065 main.log.exception( "Error parsing leaders" )
1066 main.log.error( repr( leaders ) )
1067 if missing:
1068 for node in main.CLIs:
1069 response = node.leaders( jsonFormat=False)
1070 main.log.warn( str( node.name ) + " leaders output: \n" +
1071 str( response ) )
1072
1073 partitions = main.ONOScli1.partitions()
1074 try:
1075 if partitions :
1076 parsedPartitions = json.loads( partitions )
1077 main.log.warn( json.dumps( parsedPartitions,
1078 sort_keys=True,
1079 indent=4,
1080 separators=( ',', ': ' ) ) )
1081
1082
1083 else:
1084 main.log.error( "partitions() returned None" )
1085 except ( ValueError, TypeError ):
1086 main.log.exception( "Error parsing partitions" )
1087 main.log.error( repr( partitions ) )
1088 pendingMap = main.ONOScli1.pendingMap()
1089 try:
1090 if pendingMap :
1091 parsedPending = json.loads( pendingMap )
1092 main.log.warn( json.dumps( parsedPending,
1093 sort_keys=True,
1094 indent=4,
1095 separators=( ',', ': ' ) ) )
1096
1097 else:
1098 main.log.error( "pendingMap() returned None" )
1099 except ( ValueError, TypeError ):
1100 main.log.exception( "Error parsing pending map" )
1101 main.log.error( repr( pendingMap ) )
1102
1103 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
1104 main.step( "Wait a minute then ping again" )
1105
1106 PingResult = main.TRUE
1107 for i in range( 8, 18 ):
1108 ping = main.Mininet1.pingHost( src="h" + str( i ),
1109 target="h" + str( i + 10 ) )
1110 PingResult = PingResult and ping
1111 if ping == main.FALSE:
1112 main.log.warn( "Ping failed between h" + str( i ) +
1113 " and h" + str( i + 10 ) )
1114 elif ping == main.TRUE:
1115 main.log.info( "Ping test passed!" )
1116
1117 if PingResult == main.FALSE:
1118 main.log.error(
1119 "Intents have not been installed correctly, pings failed." )
1120
1121 main.log.warn( "ONOS1 intents: " )
1122 try:
1123 tmpIntents = main.ONOScli1.intents()
1124 main.log.warn( json.dumps( json.loads( tmpIntents ),
1125 sort_keys=True,
1126 indent=4,
1127 separators=( ',', ': ' ) ) )
1128 except ( ValueError, TypeError ):
1129 main.log.warn( repr( tmpIntents ) )
1130 utilities.assert_equals(
1131 expect=main.TRUE,
1132 actual=PingResult,
1133 onpass="Intents have been installed correctly and pings work",
1134 onfail="Intents have not been installed correctly, pings failed." )
1135
1136 - def CASE5( self, main ):
1137 """
1138 Reading state of ONOS
1139 """
1140 import json
1141 import time
1142 assert main.numCtrls, "main.numCtrls not defined"
1143 assert main, "main not defined"
1144 assert utilities.assert_equals, "utilities.assert_equals not defined"
1145 assert main.CLIs, "main.CLIs not defined"
1146 assert main.nodes, "main.nodes not defined"
1147
1148 main.case( "Setting up and gathering data for current state" )
1149
1150
1151
1152
1153 main.step( "Check that each switch has a master" )
1154 global mastershipState
1155 mastershipState = '[]'
1156
1157
1158 rolesNotNull = main.TRUE
1159 threads = []
1160 for i in range( main.numCtrls ):
1161 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1162 name="rolesNotNull-" + str( i ),
1163 args=[] )
1164 threads.append( t )
1165 t.start()
1166
1167 for t in threads:
1168 t.join()
1169 rolesNotNull = rolesNotNull and t.result
1170 utilities.assert_equals(
1171 expect=main.TRUE,
1172 actual=rolesNotNull,
1173 onpass="Each device has a master",
1174 onfail="Some devices don't have a master assigned" )
1175
1176 main.step( "Get the Mastership of each switch from each controller" )
1177 ONOSMastership = []
1178 mastershipCheck = main.FALSE
1179 consistentMastership = True
1180 rolesResults = True
1181 threads = []
1182 for i in range( main.numCtrls ):
1183 t = main.Thread( target=main.CLIs[i].roles,
1184 name="roles-" + str( i ),
1185 args=[] )
1186 threads.append( t )
1187 t.start()
1188
1189 for t in threads:
1190 t.join()
1191 ONOSMastership.append( t.result )
1192
1193 for i in range( main.numCtrls ):
1194 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1195 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1196 " roles" )
1197 main.log.warn(
1198 "ONOS" + str( i + 1 ) + " mastership response: " +
1199 repr( ONOSMastership[i] ) )
1200 rolesResults = False
1201 utilities.assert_equals(
1202 expect=True,
1203 actual=rolesResults,
1204 onpass="No error in reading roles output",
1205 onfail="Error in reading roles from ONOS" )
1206
1207 main.step( "Check for consistency in roles from each controller" )
1208 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1209 main.log.info(
1210 "Switch roles are consistent across all ONOS nodes" )
1211 else:
1212 consistentMastership = False
1213 utilities.assert_equals(
1214 expect=True,
1215 actual=consistentMastership,
1216 onpass="Switch roles are consistent across all ONOS nodes",
1217 onfail="ONOS nodes have different views of switch roles" )
1218
1219 if rolesResults and not consistentMastership:
1220 for i in range( main.numCtrls ):
1221 try:
1222 main.log.warn(
1223 "ONOS" + str( i + 1 ) + " roles: ",
1224 json.dumps(
1225 json.loads( ONOSMastership[ i ] ),
1226 sort_keys=True,
1227 indent=4,
1228 separators=( ',', ': ' ) ) )
1229 except ( ValueError, TypeError ):
1230 main.log.warn( repr( ONOSMastership[ i ] ) )
1231 elif rolesResults and consistentMastership:
1232 mastershipCheck = main.TRUE
1233 mastershipState = ONOSMastership[ 0 ]
1234
1235 main.step( "Get the intents from each controller" )
1236 global intentState
1237 intentState = []
1238 ONOSIntents = []
1239 intentCheck = main.FALSE
1240 consistentIntents = True
1241 intentsResults = True
1242 threads = []
1243 for i in range( main.numCtrls ):
1244 t = main.Thread( target=main.CLIs[i].intents,
1245 name="intents-" + str( i ),
1246 args=[],
1247 kwargs={ 'jsonFormat': True } )
1248 threads.append( t )
1249 t.start()
1250
1251 for t in threads:
1252 t.join()
1253 ONOSIntents.append( t.result )
1254
1255 for i in range( main.numCtrls ):
1256 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1257 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1258 " intents" )
1259 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1260 repr( ONOSIntents[ i ] ) )
1261 intentsResults = False
1262 utilities.assert_equals(
1263 expect=True,
1264 actual=intentsResults,
1265 onpass="No error in reading intents output",
1266 onfail="Error in reading intents from ONOS" )
1267
1268 main.step( "Check for consistency in Intents from each controller" )
1269 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1270 main.log.info( "Intents are consistent across all ONOS " +
1271 "nodes" )
1272 else:
1273 consistentIntents = False
1274 main.log.error( "Intents not consistent" )
1275 utilities.assert_equals(
1276 expect=True,
1277 actual=consistentIntents,
1278 onpass="Intents are consistent across all ONOS nodes",
1279 onfail="ONOS nodes have different views of intents" )
1280
1281 if intentsResults:
1282
1283
1284
1285
1286
1287
1288 title = " Id"
1289 for n in range( main.numCtrls ):
1290 title += " " * 10 + "ONOS" + str( n + 1 )
1291 main.log.warn( title )
1292
1293 keys = []
1294 for nodeStr in ONOSIntents:
1295 node = json.loads( nodeStr )
1296 for intent in node:
1297 keys.append( intent.get( 'id' ) )
1298 keys = set( keys )
1299 for key in keys:
1300 row = "%-13s" % key
1301 for nodeStr in ONOSIntents:
1302 node = json.loads( nodeStr )
1303 for intent in node:
1304 if intent.get( 'id', "Error" ) == key:
1305 row += "%-15s" % intent.get( 'state' )
1306 main.log.warn( row )
1307
1308
1309 if intentsResults and not consistentIntents:
1310
1311 n = len(ONOSIntents)
1312 main.log.debug( "ONOS" + str( n ) + " intents: " )
1313 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1314 sort_keys=True,
1315 indent=4,
1316 separators=( ',', ': ' ) ) )
1317 for i in range( main.numCtrls ):
1318 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1319 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1320 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1321 sort_keys=True,
1322 indent=4,
1323 separators=( ',', ': ' ) ) )
1324 else:
1325 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
1326 str( n ) + " intents" )
1327 elif intentsResults and consistentIntents:
1328 intentCheck = main.TRUE
1329 intentState = ONOSIntents[ 0 ]
1330
1331 main.step( "Get the flows from each controller" )
1332 global flowState
1333 flowState = []
1334 ONOSFlows = []
1335 ONOSFlowsJson = []
1336 flowCheck = main.FALSE
1337 consistentFlows = True
1338 flowsResults = True
1339 threads = []
1340 for i in range( main.numCtrls ):
1341 t = main.Thread( target=main.CLIs[i].flows,
1342 name="flows-" + str( i ),
1343 args=[],
1344 kwargs={ 'jsonFormat': True } )
1345 threads.append( t )
1346 t.start()
1347
1348
1349 time.sleep(30)
1350 for t in threads:
1351 t.join()
1352 result = t.result
1353 ONOSFlows.append( result )
1354
1355 for i in range( main.numCtrls ):
1356 num = str( i + 1 )
1357 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1358 main.log.error( "Error in getting ONOS" + num + " flows" )
1359 main.log.warn( "ONOS" + num + " flows response: " +
1360 repr( ONOSFlows[ i ] ) )
1361 flowsResults = False
1362 ONOSFlowsJson.append( None )
1363 else:
1364 try:
1365 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1366 except ( ValueError, TypeError ):
1367
1368 main.log.exception( "Error in parsing ONOS" + num +
1369 " response as json." )
1370 main.log.error( repr( ONOSFlows[ i ] ) )
1371 ONOSFlowsJson.append( None )
1372 flowsResults = False
1373 utilities.assert_equals(
1374 expect=True,
1375 actual=flowsResults,
1376 onpass="No error in reading flows output",
1377 onfail="Error in reading flows from ONOS" )
1378
1379 main.step( "Check for consistency in Flows from each controller" )
1380 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1381 if all( tmp ):
1382 main.log.info( "Flow count is consistent across all ONOS nodes" )
1383 else:
1384 consistentFlows = False
1385 utilities.assert_equals(
1386 expect=True,
1387 actual=consistentFlows,
1388 onpass="The flow count is consistent across all ONOS nodes",
1389 onfail="ONOS nodes have different flow counts" )
1390
1391 if flowsResults and not consistentFlows:
1392 for i in range( main.numCtrls ):
1393 try:
1394 main.log.warn(
1395 "ONOS" + str( i + 1 ) + " flows: " +
1396 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1397 indent=4, separators=( ',', ': ' ) ) )
1398 except ( ValueError, TypeError ):
1399 main.log.warn(
1400 "ONOS" + str( i + 1 ) + " flows: " +
1401 repr( ONOSFlows[ i ] ) )
1402 elif flowsResults and consistentFlows:
1403 flowCheck = main.TRUE
1404 flowState = ONOSFlows[ 0 ]
1405
1406 main.step( "Get the OF Table entries" )
1407 global flows
1408 flows = []
1409 for i in range( 1, 29 ):
1410 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
1411 if flowCheck == main.FALSE:
1412 for table in flows:
1413 main.log.warn( table )
1414
1415
1416 main.step( "Start continuous pings" )
1417 main.Mininet2.pingLong(
1418 src=main.params[ 'PING' ][ 'source1' ],
1419 target=main.params[ 'PING' ][ 'target1' ],
1420 pingTime=500 )
1421 main.Mininet2.pingLong(
1422 src=main.params[ 'PING' ][ 'source2' ],
1423 target=main.params[ 'PING' ][ 'target2' ],
1424 pingTime=500 )
1425 main.Mininet2.pingLong(
1426 src=main.params[ 'PING' ][ 'source3' ],
1427 target=main.params[ 'PING' ][ 'target3' ],
1428 pingTime=500 )
1429 main.Mininet2.pingLong(
1430 src=main.params[ 'PING' ][ 'source4' ],
1431 target=main.params[ 'PING' ][ 'target4' ],
1432 pingTime=500 )
1433 main.Mininet2.pingLong(
1434 src=main.params[ 'PING' ][ 'source5' ],
1435 target=main.params[ 'PING' ][ 'target5' ],
1436 pingTime=500 )
1437 main.Mininet2.pingLong(
1438 src=main.params[ 'PING' ][ 'source6' ],
1439 target=main.params[ 'PING' ][ 'target6' ],
1440 pingTime=500 )
1441 main.Mininet2.pingLong(
1442 src=main.params[ 'PING' ][ 'source7' ],
1443 target=main.params[ 'PING' ][ 'target7' ],
1444 pingTime=500 )
1445 main.Mininet2.pingLong(
1446 src=main.params[ 'PING' ][ 'source8' ],
1447 target=main.params[ 'PING' ][ 'target8' ],
1448 pingTime=500 )
1449 main.Mininet2.pingLong(
1450 src=main.params[ 'PING' ][ 'source9' ],
1451 target=main.params[ 'PING' ][ 'target9' ],
1452 pingTime=500 )
1453 main.Mininet2.pingLong(
1454 src=main.params[ 'PING' ][ 'source10' ],
1455 target=main.params[ 'PING' ][ 'target10' ],
1456 pingTime=500 )
1457
1458 main.step( "Collecting topology information from ONOS" )
1459 devices = []
1460 threads = []
1461 for i in range( main.numCtrls ):
1462 t = main.Thread( target=main.CLIs[i].devices,
1463 name="devices-" + str( i ),
1464 args=[ ] )
1465 threads.append( t )
1466 t.start()
1467
1468 for t in threads:
1469 t.join()
1470 devices.append( t.result )
1471 hosts = []
1472 threads = []
1473 for i in range( main.numCtrls ):
1474 t = main.Thread( target=main.CLIs[i].hosts,
1475 name="hosts-" + str( i ),
1476 args=[ ] )
1477 threads.append( t )
1478 t.start()
1479
1480 for t in threads:
1481 t.join()
1482 try:
1483 hosts.append( json.loads( t.result ) )
1484 except ( ValueError, TypeError ):
1485
1486
1487 main.log.exception( "Error parsing json output of hosts" )
1488
1489 hosts.append( None )
1490
1491 ports = []
1492 threads = []
1493 for i in range( main.numCtrls ):
1494 t = main.Thread( target=main.CLIs[i].ports,
1495 name="ports-" + str( i ),
1496 args=[ ] )
1497 threads.append( t )
1498 t.start()
1499
1500 for t in threads:
1501 t.join()
1502 ports.append( t.result )
1503 links = []
1504 threads = []
1505 for i in range( main.numCtrls ):
1506 t = main.Thread( target=main.CLIs[i].links,
1507 name="links-" + str( i ),
1508 args=[ ] )
1509 threads.append( t )
1510 t.start()
1511
1512 for t in threads:
1513 t.join()
1514 links.append( t.result )
1515 clusters = []
1516 threads = []
1517 for i in range( main.numCtrls ):
1518 t = main.Thread( target=main.CLIs[i].clusters,
1519 name="clusters-" + str( i ),
1520 args=[ ] )
1521 threads.append( t )
1522 t.start()
1523
1524 for t in threads:
1525 t.join()
1526 clusters.append( t.result )
1527
1528
1529
1530 main.step( "Host view is consistent across ONOS nodes" )
1531 consistentHostsResult = main.TRUE
1532 for controller in range( len( hosts ) ):
1533 controllerStr = str( controller + 1 )
1534 if "Error" not in hosts[ controller ]:
1535 if hosts[ controller ] == hosts[ 0 ]:
1536 continue
1537 else:
1538 main.log.error( "hosts from ONOS" +
1539 controllerStr +
1540 " is inconsistent with ONOS1" )
1541 main.log.warn( repr( hosts[ controller ] ) )
1542 consistentHostsResult = main.FALSE
1543
1544 else:
1545 main.log.error( "Error in getting ONOS hosts from ONOS" +
1546 controllerStr )
1547 consistentHostsResult = main.FALSE
1548 main.log.warn( "ONOS" + controllerStr +
1549 " hosts response: " +
1550 repr( hosts[ controller ] ) )
1551 utilities.assert_equals(
1552 expect=main.TRUE,
1553 actual=consistentHostsResult,
1554 onpass="Hosts view is consistent across all ONOS nodes",
1555 onfail="ONOS nodes have different views of hosts" )
1556
1557 main.step( "Each host has an IP address" )
1558 ipResult = main.TRUE
1559 for controller in range( 0, len( hosts ) ):
1560 controllerStr = str( controller + 1 )
1561 for host in hosts[ controller ]:
1562 if not host.get( 'ipAddresses', [ ] ):
1563 main.log.error( "DEBUG:Error with host ips on controller" +
1564 controllerStr + ": " + str( host ) )
1565 ipResult = main.FALSE
1566 utilities.assert_equals(
1567 expect=main.TRUE,
1568 actual=ipResult,
1569 onpass="The ips of the hosts aren't empty",
1570 onfail="The ip of at least one host is missing" )
1571
1572
1573 main.step( "Cluster view is consistent across ONOS nodes" )
1574 consistentClustersResult = main.TRUE
1575 for controller in range( len( clusters ) ):
1576 controllerStr = str( controller + 1 )
1577 if "Error" not in clusters[ controller ]:
1578 if clusters[ controller ] == clusters[ 0 ]:
1579 continue
1580 else:
1581 main.log.error( "clusters from ONOS" + controllerStr +
1582 " is inconsistent with ONOS1" )
1583 consistentClustersResult = main.FALSE
1584
1585 else:
1586 main.log.error( "Error in getting dataplane clusters " +
1587 "from ONOS" + controllerStr )
1588 consistentClustersResult = main.FALSE
1589 main.log.warn( "ONOS" + controllerStr +
1590 " clusters response: " +
1591 repr( clusters[ controller ] ) )
1592 utilities.assert_equals(
1593 expect=main.TRUE,
1594 actual=consistentClustersResult,
1595 onpass="Clusters view is consistent across all ONOS nodes",
1596 onfail="ONOS nodes have different views of clusters" )
1597
1598 main.step( "Cluster view correct across ONOS nodes" )
1599 try:
1600 numClusters = len( json.loads( clusters[ 0 ] ) )
1601 except ( ValueError, TypeError ):
1602 main.log.exception( "Error parsing clusters[0]: " +
1603 repr( clusters[ 0 ] ) )
1604 clusterResults = main.FALSE
1605 if numClusters == 1:
1606 clusterResults = main.TRUE
1607 utilities.assert_equals(
1608 expect=1,
1609 actual=numClusters,
1610 onpass="ONOS shows 1 SCC",
1611 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1612
1613 main.step( "Comparing ONOS topology to MN" )
1614 devicesResults = main.TRUE
1615 linksResults = main.TRUE
1616 hostsResults = main.TRUE
1617 mnSwitches = main.Mininet1.getSwitches()
1618 mnLinks = main.Mininet1.getLinks()
1619 mnHosts = main.Mininet1.getHosts()
1620 for controller in range( main.numCtrls ):
1621 controllerStr = str( controller + 1 )
1622 if devices[ controller ] and ports[ controller ] and\
1623 "Error" not in devices[ controller ] and\
1624 "Error" not in ports[ controller ]:
1625 currentDevicesResult = main.Mininet1.compareSwitches(
1626 mnSwitches,
1627 json.loads( devices[ controller ] ),
1628 json.loads( ports[ controller ] ) )
1629 else:
1630 currentDevicesResult = main.FALSE
1631 utilities.assert_equals( expect=main.TRUE,
1632 actual=currentDevicesResult,
1633 onpass="ONOS" + controllerStr +
1634 " Switches view is correct",
1635 onfail="ONOS" + controllerStr +
1636 " Switches view is incorrect" )
1637 if links[ controller ] and "Error" not in links[ controller ]:
1638 currentLinksResult = main.Mininet1.compareLinks(
1639 mnSwitches, mnLinks,
1640 json.loads( links[ controller ] ) )
1641 else:
1642 currentLinksResult = main.FALSE
1643 utilities.assert_equals( expect=main.TRUE,
1644 actual=currentLinksResult,
1645 onpass="ONOS" + controllerStr +
1646 " links view is correct",
1647 onfail="ONOS" + controllerStr +
1648 " links view is incorrect" )
1649
1650 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1651 currentHostsResult = main.Mininet1.compareHosts(
1652 mnHosts,
1653 hosts[ controller ] )
1654 else:
1655 currentHostsResult = main.FALSE
1656 utilities.assert_equals( expect=main.TRUE,
1657 actual=currentHostsResult,
1658 onpass="ONOS" + controllerStr +
1659 " hosts exist in Mininet",
1660 onfail="ONOS" + controllerStr +
1661 " hosts don't match Mininet" )
1662
1663 devicesResults = devicesResults and currentDevicesResult
1664 linksResults = linksResults and currentLinksResult
1665 hostsResults = hostsResults and currentHostsResult
1666
1667 main.step( "Device information is correct" )
1668 utilities.assert_equals(
1669 expect=main.TRUE,
1670 actual=devicesResults,
1671 onpass="Device information is correct",
1672 onfail="Device information is incorrect" )
1673
1674 main.step( "Links are correct" )
1675 utilities.assert_equals(
1676 expect=main.TRUE,
1677 actual=linksResults,
1678 onpass="Link are correct",
1679 onfail="Links are incorrect" )
1680
1681 main.step( "Hosts are correct" )
1682 utilities.assert_equals(
1683 expect=main.TRUE,
1684 actual=hostsResults,
1685 onpass="Hosts are correct",
1686 onfail="Hosts are incorrect" )
1687
1688 - def CASE6( self, main ):
1689 """
1690 The Failure case.
1691 """
1692 import time
1693 assert main.numCtrls, "main.numCtrls not defined"
1694 assert main, "main not defined"
1695 assert utilities.assert_equals, "utilities.assert_equals not defined"
1696 assert main.CLIs, "main.CLIs not defined"
1697 assert main.nodes, "main.nodes not defined"
1698 try:
1699 labels
1700 except NameError:
1701 main.log.error( "labels not defined, setting to []" )
1702 global labels
1703 labels = []
1704 try:
1705 data
1706 except NameError:
1707 main.log.error( "data not defined, setting to []" )
1708 global data
1709 data = []
1710
1711 try:
1712 iCounterValue = 0
1713 except NameError:
1714 main.log.error( "iCounterValue not defined, setting to 0" )
1715 iCounterValue = 0
1716
1717 main.case( "Restart entire ONOS cluster" )
1718
1719 main.step( "Killing ONOS nodes" )
1720 killResults = main.TRUE
1721 killTime = time.time()
1722 for node in main.nodes:
1723 killed = main.ONOSbench.onosKill( node.ip_address )
1724 killResults = killResults and killed
1725 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1726 onpass="ONOS nodes killed",
1727 onfail="ONOS kill unsuccessful" )
1728
1729 main.step( "Checking if ONOS is up yet" )
1730 for i in range( 2 ):
1731 onosIsupResult = main.TRUE
1732 for node in main.nodes:
1733 started = main.ONOSbench.isup( node.ip_address )
1734 if not started:
1735 main.log.error( node.name + " didn't start!" )
1736 onosIsupResult = onosIsupResult and started
1737 if onosIsupResult == main.TRUE:
1738 break
1739 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1740 onpass="ONOS restarted",
1741 onfail="ONOS restart NOT successful" )
1742
1743 main.log.step( "Starting ONOS CLI sessions" )
1744 cliResults = main.TRUE
1745 threads = []
1746 for i in range( main.numCtrls ):
1747 t = main.Thread( target=main.CLIs[i].startOnosCli,
1748 name="startOnosCli-" + str( i ),
1749 args=[main.nodes[i].ip_address] )
1750 threads.append( t )
1751 t.start()
1752
1753 for t in threads:
1754 t.join()
1755 cliResults = cliResults and t.result
1756 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1757 onpass="ONOS cli started",
1758 onfail="ONOS clis did not restart" )
1759
1760
1761
1762 main.restartTime = time.time() - killTime
1763 main.log.debug( "Restart time: " + str( main.restartTime ) )
1764 labels.append( "Restart" )
1765 data.append( str( main.restartTime ) )
1766
1767
1768
1769 runResults = main.TRUE
1770 for cli in main.CLIs:
1771 run = main.CLIs[0].electionTestRun()
1772 if run != main.TRUE:
1773 main.log.error( "Error running for election on " + cli.name )
1774 runResults = runResults and run
1775 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1776 onpass="Reran for election",
1777 onfail="Failed to rerun for election" )
1778
1779
1780 time.sleep( 60 )
1781 main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) )
1782 main.log.debug( main.CLIs[0].leaders( jsonFormat=False ) )
1783 main.log.debug( main.CLIs[0].partitions( jsonFormat=False ) )
1784
1785 - def CASE7( self, main ):
1786 """
1787 Check state after ONOS failure
1788 """
1789 import json
1790 assert main.numCtrls, "main.numCtrls not defined"
1791 assert main, "main not defined"
1792 assert utilities.assert_equals, "utilities.assert_equals not defined"
1793 assert main.CLIs, "main.CLIs not defined"
1794 assert main.nodes, "main.nodes not defined"
1795 main.case( "Running ONOS Constant State Tests" )
1796
1797 main.step( "Check that each switch has a master" )
1798
1799 rolesNotNull = main.TRUE
1800 threads = []
1801 for i in range( main.numCtrls ):
1802 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1803 name="rolesNotNull-" + str( i ),
1804 args=[ ] )
1805 threads.append( t )
1806 t.start()
1807
1808 for t in threads:
1809 t.join()
1810 rolesNotNull = rolesNotNull and t.result
1811 utilities.assert_equals(
1812 expect=main.TRUE,
1813 actual=rolesNotNull,
1814 onpass="Each device has a master",
1815 onfail="Some devices don't have a master assigned" )
1816
1817 main.step( "Read device roles from ONOS" )
1818 ONOSMastership = []
1819 mastershipCheck = main.FALSE
1820 consistentMastership = True
1821 rolesResults = True
1822 threads = []
1823 for i in range( main.numCtrls ):
1824 t = main.Thread( target=main.CLIs[i].roles,
1825 name="roles-" + str( i ),
1826 args=[] )
1827 threads.append( t )
1828 t.start()
1829
1830 for t in threads:
1831 t.join()
1832 ONOSMastership.append( t.result )
1833
1834 for i in range( main.numCtrls ):
1835 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1836 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1837 " roles" )
1838 main.log.warn(
1839 "ONOS" + str( i + 1 ) + " mastership response: " +
1840 repr( ONOSMastership[i] ) )
1841 rolesResults = False
1842 utilities.assert_equals(
1843 expect=True,
1844 actual=rolesResults,
1845 onpass="No error in reading roles output",
1846 onfail="Error in reading roles from ONOS" )
1847
1848 main.step( "Check for consistency in roles from each controller" )
1849 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1850 main.log.info(
1851 "Switch roles are consistent across all ONOS nodes" )
1852 else:
1853 consistentMastership = False
1854 utilities.assert_equals(
1855 expect=True,
1856 actual=consistentMastership,
1857 onpass="Switch roles are consistent across all ONOS nodes",
1858 onfail="ONOS nodes have different views of switch roles" )
1859
1860 if rolesResults and not consistentMastership:
1861 for i in range( main.numCtrls ):
1862 main.log.warn(
1863 "ONOS" + str( i + 1 ) + " roles: ",
1864 json.dumps(
1865 json.loads( ONOSMastership[ i ] ),
1866 sort_keys=True,
1867 indent=4,
1868 separators=( ',', ': ' ) ) )
1869 elif rolesResults and not consistentMastership:
1870 mastershipCheck = main.TRUE
1871
1872 '''
1873 description2 = "Compare switch roles from before failure"
1874 main.step( description2 )
1875 try:
1876 currentJson = json.loads( ONOSMastership[0] )
1877 oldJson = json.loads( mastershipState )
1878 except ( ValueError, TypeError ):
1879 main.log.exception( "Something is wrong with parsing " +
1880 "ONOSMastership[0] or mastershipState" )
1881 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1882 main.log.error( "mastershipState" + repr( mastershipState ) )
1883 main.cleanup()
1884 main.exit()
1885 mastershipCheck = main.TRUE
1886 for i in range( 1, 29 ):
1887 switchDPID = str(
1888 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1889 current = [ switch[ 'master' ] for switch in currentJson
1890 if switchDPID in switch[ 'id' ] ]
1891 old = [ switch[ 'master' ] for switch in oldJson
1892 if switchDPID in switch[ 'id' ] ]
1893 if current == old:
1894 mastershipCheck = mastershipCheck and main.TRUE
1895 else:
1896 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1897 mastershipCheck = main.FALSE
1898 utilities.assert_equals(
1899 expect=main.TRUE,
1900 actual=mastershipCheck,
1901 onpass="Mastership of Switches was not changed",
1902 onfail="Mastership of some switches changed" )
1903 '''
1904
1905
1906 main.step( "Get the intents and compare across all nodes" )
1907 ONOSIntents = []
1908 intentCheck = main.FALSE
1909 consistentIntents = True
1910 intentsResults = True
1911 threads = []
1912 for i in range( main.numCtrls ):
1913 t = main.Thread( target=main.CLIs[i].intents,
1914 name="intents-" + str( i ),
1915 args=[],
1916 kwargs={ 'jsonFormat': True } )
1917 threads.append( t )
1918 t.start()
1919
1920 for t in threads:
1921 t.join()
1922 ONOSIntents.append( t.result )
1923
1924 for i in range( main.numCtrls ):
1925 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1926 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1927 " intents" )
1928 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1929 repr( ONOSIntents[ i ] ) )
1930 intentsResults = False
1931 utilities.assert_equals(
1932 expect=True,
1933 actual=intentsResults,
1934 onpass="No error in reading intents output",
1935 onfail="Error in reading intents from ONOS" )
1936
1937 main.step( "Check for consistency in Intents from each controller" )
1938 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1939 main.log.info( "Intents are consistent across all ONOS " +
1940 "nodes" )
1941 else:
1942 consistentIntents = False
1943
1944
1945
1946
1947
1948
1949
1950 title = " ID"
1951 for n in range( main.numCtrls ):
1952 title += " " * 10 + "ONOS" + str( n + 1 )
1953 main.log.warn( title )
1954
1955 keys = []
1956 for nodeStr in ONOSIntents:
1957 node = json.loads( nodeStr )
1958 for intent in node:
1959 keys.append( intent.get( 'id' ) )
1960 keys = set( keys )
1961 for key in keys:
1962 row = "%-13s" % key
1963 for nodeStr in ONOSIntents:
1964 node = json.loads( nodeStr )
1965 for intent in node:
1966 if intent.get( 'id' ) == key:
1967 row += "%-15s" % intent.get( 'state' )
1968 main.log.warn( row )
1969
1970
1971 utilities.assert_equals(
1972 expect=True,
1973 actual=consistentIntents,
1974 onpass="Intents are consistent across all ONOS nodes",
1975 onfail="ONOS nodes have different views of intents" )
1976 intentStates = []
1977 for node in ONOSIntents:
1978 nodeStates = []
1979
1980 try:
1981 for intent in json.loads( node ):
1982 nodeStates.append( intent[ 'state' ] )
1983 except ( ValueError, TypeError ):
1984 main.log.exception( "Error in parsing intents" )
1985 main.log.error( repr( node ) )
1986 intentStates.append( nodeStates )
1987 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1988 main.log.info( dict( out ) )
1989
1990 if intentsResults and not consistentIntents:
1991 for i in range( main.numCtrls ):
1992 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1993 main.log.warn( json.dumps(
1994 json.loads( ONOSIntents[ i ] ),
1995 sort_keys=True,
1996 indent=4,
1997 separators=( ',', ': ' ) ) )
1998 elif intentsResults and consistentIntents:
1999 intentCheck = main.TRUE
2000
2001
2002
2003 """
2004 main.step( "Compare current intents with intents before the failure" )
2005 # NOTE: this requires case 5 to pass for intentState to be set.
2006 # maybe we should stop the test if that fails?
2007 sameIntents = main.FALSE
2008 if intentState and intentState == ONOSIntents[ 0 ]:
2009 sameIntents = main.TRUE
2010 main.log.info( "Intents are consistent with before failure" )
2011 # TODO: possibly the states have changed? we may need to figure out
2012 # what the acceptable states are
2013 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2014 sameIntents = main.TRUE
2015 try:
2016 before = json.loads( intentState )
2017 after = json.loads( ONOSIntents[ 0 ] )
2018 for intent in before:
2019 if intent not in after:
2020 sameIntents = main.FALSE
2021 main.log.debug( "Intent is not currently in ONOS " +
2022 "(at least in the same form):" )
2023 main.log.debug( json.dumps( intent ) )
2024 except ( ValueError, TypeError ):
2025 main.log.exception( "Exception printing intents" )
2026 main.log.debug( repr( ONOSIntents[0] ) )
2027 main.log.debug( repr( intentState ) )
2028 if sameIntents == main.FALSE:
2029 try:
2030 main.log.debug( "ONOS intents before: " )
2031 main.log.debug( json.dumps( json.loads( intentState ),
2032 sort_keys=True, indent=4,
2033 separators=( ',', ': ' ) ) )
2034 main.log.debug( "Current ONOS intents: " )
2035 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2036 sort_keys=True, indent=4,
2037 separators=( ',', ': ' ) ) )
2038 except ( ValueError, TypeError ):
2039 main.log.exception( "Exception printing intents" )
2040 main.log.debug( repr( ONOSIntents[0] ) )
2041 main.log.debug( repr( intentState ) )
2042 utilities.assert_equals(
2043 expect=main.TRUE,
2044 actual=sameIntents,
2045 onpass="Intents are consistent with before failure",
2046 onfail="The Intents changed during failure" )
2047 intentCheck = intentCheck and sameIntents
2048 """
2049 main.step( "Get the OF Table entries and compare to before " +
2050 "component failure" )
2051 FlowTables = main.TRUE
2052 flows2 = []
2053 for i in range( 28 ):
2054 main.log.info( "Checking flow table on s" + str( i + 1 ) )
2055 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
2056 flows2.append( tmpFlows )
2057 tempResult = main.Mininet1.flowComp(
2058 flow1=flows[ i ],
2059 flow2=tmpFlows )
2060 FlowTables = FlowTables and tempResult
2061 if FlowTables == main.FALSE:
2062 main.log.info( "Differences in flow table for switch: s" +
2063 str( i + 1 ) )
2064 utilities.assert_equals(
2065 expect=main.TRUE,
2066 actual=FlowTables,
2067 onpass="No changes were found in the flow tables",
2068 onfail="Changes were found in the flow tables" )
2069
2070 main.Mininet2.pingLongKill()
2071 '''
2072 # main.step( "Check the continuous pings to ensure that no packets " +
2073 # "were dropped during component failure" )
2074 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2075 main.params[ 'TESTONIP' ] )
2076 LossInPings = main.FALSE
2077 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2078 for i in range( 8, 18 ):
2079 main.log.info(
2080 "Checking for a loss in pings along flow from s" +
2081 str( i ) )
2082 LossInPings = main.Mininet2.checkForLoss(
2083 "/tmp/ping.h" +
2084 str( i ) ) or LossInPings
2085 if LossInPings == main.TRUE:
2086 main.log.info( "Loss in ping detected" )
2087 elif LossInPings == main.ERROR:
2088 main.log.info( "There are multiple mininet process running" )
2089 elif LossInPings == main.FALSE:
2090 main.log.info( "No Loss in the pings" )
2091 main.log.info( "No loss of dataplane connectivity" )
2092 # utilities.assert_equals(
2093 # expect=main.FALSE,
2094 # actual=LossInPings,
2095 # onpass="No Loss of connectivity",
2096 # onfail="Loss of dataplane connectivity detected" )
2097
2098 # NOTE: Since intents are not persisted with IntnentStore,
2099 # we expect loss in dataplane connectivity
2100 LossInPings = main.FALSE
2101 '''
2102
2103 main.step( "Leadership Election is still functional" )
2104
2105 leaderList = []
2106 leaderResult = main.TRUE
2107 for cli in main.CLIs:
2108 leaderN = cli.electionTestLeader()
2109 leaderList.append( leaderN )
2110 if leaderN == main.FALSE:
2111
2112 main.log.error( "Something is wrong with " +
2113 "electionTestLeader function, check the" +
2114 " error logs" )
2115 leaderResult = main.FALSE
2116 elif leaderN is None:
2117 main.log.error( cli.name +
2118 " shows no leader for the election-app." )
2119 leaderResult = main.FALSE
2120 if len( set( leaderList ) ) != 1:
2121 leaderResult = main.FALSE
2122 main.log.error(
2123 "Inconsistent view of leader for the election test app" )
2124
2125 utilities.assert_equals(
2126 expect=main.TRUE,
2127 actual=leaderResult,
2128 onpass="Leadership election passed",
2129 onfail="Something went wrong with Leadership election" )
2130
2131 - def CASE8( self, main ):
2132 """
2133 Compare topo
2134 """
2135 import json
2136 import time
2137 assert main.numCtrls, "main.numCtrls not defined"
2138 assert main, "main not defined"
2139 assert utilities.assert_equals, "utilities.assert_equals not defined"
2140 assert main.CLIs, "main.CLIs not defined"
2141 assert main.nodes, "main.nodes not defined"
2142
2143 main.case( "Compare ONOS Topology view to Mininet topology" )
2144 main.caseExplanation = "Compare topology objects between Mininet" +\
2145 " and ONOS"
2146
2147 main.step( "Comparing ONOS topology to MN" )
2148 devicesResults = main.TRUE
2149 linksResults = main.TRUE
2150 hostsResults = main.TRUE
2151 hostAttachmentResults = True
2152 topoResult = main.FALSE
2153 elapsed = 0
2154 count = 0
2155 main.step( "Collecting topology information from ONOS" )
2156 startTime = time.time()
2157
2158 while topoResult == main.FALSE and elapsed < 60:
2159 count += 1
2160 cliStart = time.time()
2161 devices = []
2162 threads = []
2163 for i in range( main.numCtrls ):
2164 t = main.Thread( target=main.CLIs[i].devices,
2165 name="devices-" + str( i ),
2166 args=[ ] )
2167 threads.append( t )
2168 t.start()
2169
2170 for t in threads:
2171 t.join()
2172 devices.append( t.result )
2173 hosts = []
2174 ipResult = main.TRUE
2175 threads = []
2176 for i in range( main.numCtrls ):
2177 t = main.Thread( target=main.CLIs[i].hosts,
2178 name="hosts-" + str( i ),
2179 args=[ ] )
2180 threads.append( t )
2181 t.start()
2182
2183 for t in threads:
2184 t.join()
2185 try:
2186 hosts.append( json.loads( t.result ) )
2187 except ( ValueError, TypeError ):
2188 main.log.exception( "Error parsing hosts results" )
2189 main.log.error( repr( t.result ) )
2190 for controller in range( 0, len( hosts ) ):
2191 controllerStr = str( controller + 1 )
2192 for host in hosts[ controller ]:
2193 if host is None or host.get( 'ipAddresses', [] ) == []:
2194 main.log.error(
2195 "DEBUG:Error with host ipAddresses on controller" +
2196 controllerStr + ": " + str( host ) )
2197 ipResult = main.FALSE
2198 ports = []
2199 threads = []
2200 for i in range( main.numCtrls ):
2201 t = main.Thread( target=main.CLIs[i].ports,
2202 name="ports-" + str( i ),
2203 args=[ ] )
2204 threads.append( t )
2205 t.start()
2206
2207 for t in threads:
2208 t.join()
2209 ports.append( t.result )
2210 links = []
2211 threads = []
2212 for i in range( main.numCtrls ):
2213 t = main.Thread( target=main.CLIs[i].links,
2214 name="links-" + str( i ),
2215 args=[ ] )
2216 threads.append( t )
2217 t.start()
2218
2219 for t in threads:
2220 t.join()
2221 links.append( t.result )
2222 clusters = []
2223 threads = []
2224 for i in range( main.numCtrls ):
2225 t = main.Thread( target=main.CLIs[i].clusters,
2226 name="clusters-" + str( i ),
2227 args=[ ] )
2228 threads.append( t )
2229 t.start()
2230
2231 for t in threads:
2232 t.join()
2233 clusters.append( t.result )
2234
2235 elapsed = time.time() - startTime
2236 cliTime = time.time() - cliStart
2237 print "Elapsed time: " + str( elapsed )
2238 print "CLI time: " + str( cliTime )
2239
2240 mnSwitches = main.Mininet1.getSwitches()
2241 mnLinks = main.Mininet1.getLinks()
2242 mnHosts = main.Mininet1.getHosts()
2243 for controller in range( main.numCtrls ):
2244 controllerStr = str( controller + 1 )
2245 if devices[ controller ] and ports[ controller ] and\
2246 "Error" not in devices[ controller ] and\
2247 "Error" not in ports[ controller ]:
2248
2249 currentDevicesResult = main.Mininet1.compareSwitches(
2250 mnSwitches,
2251 json.loads( devices[ controller ] ),
2252 json.loads( ports[ controller ] ) )
2253 else:
2254 currentDevicesResult = main.FALSE
2255 utilities.assert_equals( expect=main.TRUE,
2256 actual=currentDevicesResult,
2257 onpass="ONOS" + controllerStr +
2258 " Switches view is correct",
2259 onfail="ONOS" + controllerStr +
2260 " Switches view is incorrect" )
2261
2262 if links[ controller ] and "Error" not in links[ controller ]:
2263 currentLinksResult = main.Mininet1.compareLinks(
2264 mnSwitches, mnLinks,
2265 json.loads( links[ controller ] ) )
2266 else:
2267 currentLinksResult = main.FALSE
2268 utilities.assert_equals( expect=main.TRUE,
2269 actual=currentLinksResult,
2270 onpass="ONOS" + controllerStr +
2271 " links view is correct",
2272 onfail="ONOS" + controllerStr +
2273 " links view is incorrect" )
2274
2275 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2276 currentHostsResult = main.Mininet1.compareHosts(
2277 mnHosts,
2278 hosts[ controller ] )
2279 else:
2280 currentHostsResult = main.FALSE
2281 utilities.assert_equals( expect=main.TRUE,
2282 actual=currentHostsResult,
2283 onpass="ONOS" + controllerStr +
2284 " hosts exist in Mininet",
2285 onfail="ONOS" + controllerStr +
2286 " hosts don't match Mininet" )
2287
2288 hostAttachment = True
2289 noHosts = False
2290
2291
2292 mappings = {}
2293 for i in range( 1, 29 ):
2294
2295 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2296 if i == 1:
2297 deviceId = "1000".zfill(16)
2298 elif i == 2:
2299 deviceId = "2000".zfill(16)
2300 elif i == 3:
2301 deviceId = "3000".zfill(16)
2302 elif i == 4:
2303 deviceId = "3004".zfill(16)
2304 elif i == 5:
2305 deviceId = "5000".zfill(16)
2306 elif i == 6:
2307 deviceId = "6000".zfill(16)
2308 elif i == 7:
2309 deviceId = "6007".zfill(16)
2310 elif i >= 8 and i <= 17:
2311 dpid = '3' + str( i ).zfill( 3 )
2312 deviceId = dpid.zfill(16)
2313 elif i >= 18 and i <= 27:
2314 dpid = '6' + str( i ).zfill( 3 )
2315 deviceId = dpid.zfill(16)
2316 elif i == 28:
2317 deviceId = "2800".zfill(16)
2318 mappings[ macId ] = deviceId
2319 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2320 if hosts[ controller ] == []:
2321 main.log.warn( "There are no hosts discovered" )
2322 noHosts = True
2323 else:
2324 for host in hosts[ controller ]:
2325 mac = None
2326 location = None
2327 device = None
2328 port = None
2329 try:
2330 mac = host.get( 'mac' )
2331 assert mac, "mac field could not be found for this host object"
2332
2333 location = host.get( 'location' )
2334 assert location, "location field could not be found for this host object"
2335
2336
2337 device = str( location.get( 'elementId' ) ).split(':')[1]
2338 assert device, "elementId field could not be found for this host location object"
2339
2340 port = location.get( 'port' )
2341 assert port, "port field could not be found for this host location object"
2342
2343
2344 if mac and device and port:
2345 if str( port ) != "1":
2346 main.log.error( "The attachment port is incorrect for " +
2347 "host " + str( mac ) +
2348 ". Expected: 1 Actual: " + str( port) )
2349 hostAttachment = False
2350 if device != mappings[ str( mac ) ]:
2351 main.log.error( "The attachment device is incorrect for " +
2352 "host " + str( mac ) +
2353 ". Expected: " + mappings[ str( mac ) ] +
2354 " Actual: " + device )
2355 hostAttachment = False
2356 else:
2357 hostAttachment = False
2358 except AssertionError:
2359 main.log.exception( "Json object not as expected" )
2360 main.log.error( repr( host ) )
2361 hostAttachment = False
2362 else:
2363 main.log.error( "No hosts json output or \"Error\"" +
2364 " in output. hosts = " +
2365 repr( hosts[ controller ] ) )
2366 if noHosts is False:
2367
2368
2369 hostAttachment = True
2370
2371
2372 devicesResults = devicesResults and currentDevicesResult
2373 linksResults = linksResults and currentLinksResult
2374 hostsResults = hostsResults and currentHostsResult
2375 hostAttachmentResults = hostAttachmentResults and\
2376 hostAttachment
2377 topoResult = ( devicesResults and linksResults
2378 and hostsResults and ipResult and
2379 hostAttachmentResults )
2380
2381
2382
2383
2384 main.step( "Hosts view is consistent across all ONOS nodes" )
2385 consistentHostsResult = main.TRUE
2386 for controller in range( len( hosts ) ):
2387 controllerStr = str( controller + 1 )
2388 if "Error" not in hosts[ controller ]:
2389 if hosts[ controller ] == hosts[ 0 ]:
2390 continue
2391 else:
2392 main.log.error( "hosts from ONOS" + controllerStr +
2393 " is inconsistent with ONOS1" )
2394 main.log.warn( repr( hosts[ controller ] ) )
2395 consistentHostsResult = main.FALSE
2396
2397 else:
2398 main.log.error( "Error in getting ONOS hosts from ONOS" +
2399 controllerStr )
2400 consistentHostsResult = main.FALSE
2401 main.log.warn( "ONOS" + controllerStr +
2402 " hosts response: " +
2403 repr( hosts[ controller ] ) )
2404 utilities.assert_equals(
2405 expect=main.TRUE,
2406 actual=consistentHostsResult,
2407 onpass="Hosts view is consistent across all ONOS nodes",
2408 onfail="ONOS nodes have different views of hosts" )
2409
2410 main.step( "Hosts information is correct" )
2411 hostsResults = hostsResults and ipResult
2412 utilities.assert_equals(
2413 expect=main.TRUE,
2414 actual=hostsResults,
2415 onpass="Host information is correct",
2416 onfail="Host information is incorrect" )
2417
2418 main.step( "Host attachment points to the network" )
2419 utilities.assert_equals(
2420 expect=True,
2421 actual=hostAttachmentResults,
2422 onpass="Hosts are correctly attached to the network",
2423 onfail="ONOS did not correctly attach hosts to the network" )
2424
2425
2426 main.step( "Clusters view is consistent across all ONOS nodes" )
2427 consistentClustersResult = main.TRUE
2428 for controller in range( len( clusters ) ):
2429 controllerStr = str( controller + 1 )
2430 if "Error" not in clusters[ controller ]:
2431 if clusters[ controller ] == clusters[ 0 ]:
2432 continue
2433 else:
2434 main.log.error( "clusters from ONOS" +
2435 controllerStr +
2436 " is inconsistent with ONOS1" )
2437 consistentClustersResult = main.FALSE
2438
2439 else:
2440 main.log.error( "Error in getting dataplane clusters " +
2441 "from ONOS" + controllerStr )
2442 consistentClustersResult = main.FALSE
2443 main.log.warn( "ONOS" + controllerStr +
2444 " clusters response: " +
2445 repr( clusters[ controller ] ) )
2446 utilities.assert_equals(
2447 expect=main.TRUE,
2448 actual=consistentClustersResult,
2449 onpass="Clusters view is consistent across all ONOS nodes",
2450 onfail="ONOS nodes have different views of clusters" )
2451
2452 main.step( "There is only one SCC" )
2453
2454 try:
2455 numClusters = len( json.loads( clusters[ 0 ] ) )
2456 except ( ValueError, TypeError ):
2457 main.log.exception( "Error parsing clusters[0]: " +
2458 repr( clusters[0] ) )
2459 clusterResults = main.FALSE
2460 if numClusters == 1:
2461 clusterResults = main.TRUE
2462 utilities.assert_equals(
2463 expect=1,
2464 actual=numClusters,
2465 onpass="ONOS shows 1 SCC",
2466 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2467
2468 topoResult = ( devicesResults and linksResults
2469 and hostsResults and consistentHostsResult
2470 and consistentClustersResult and clusterResults
2471 and ipResult and hostAttachmentResults )
2472
2473 topoResult = topoResult and int( count <= 2 )
2474 note = "note it takes about " + str( int( cliTime ) ) + \
2475 " seconds for the test to make all the cli calls to fetch " +\
2476 "the topology from each ONOS instance"
2477 main.log.info(
2478 "Very crass estimate for topology discovery/convergence( " +
2479 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2480 str( count ) + " tries" )
2481
2482 main.step( "Device information is correct" )
2483 utilities.assert_equals(
2484 expect=main.TRUE,
2485 actual=devicesResults,
2486 onpass="Device information is correct",
2487 onfail="Device information is incorrect" )
2488
2489 main.step( "Links are correct" )
2490 utilities.assert_equals(
2491 expect=main.TRUE,
2492 actual=linksResults,
2493 onpass="Link are correct",
2494 onfail="Links are incorrect" )
2495
2496
2497 main.step( "Checking ONOS nodes" )
2498 nodesOutput = []
2499 nodeResults = main.TRUE
2500 threads = []
2501 for i in range( main.numCtrls ):
2502 t = main.Thread( target=main.CLIs[i].nodes,
2503 name="nodes-" + str( i ),
2504 args=[ ] )
2505 threads.append( t )
2506 t.start()
2507
2508 for t in threads:
2509 t.join()
2510 nodesOutput.append( t.result )
2511 ips = [ node.ip_address for node in main.nodes ]
2512 for i in nodesOutput:
2513 try:
2514 current = json.loads( i )
2515 for node in current:
2516 currentResult = main.FALSE
2517 if node['ip'] in ips:
2518 if node['state'] == 'ACTIVE':
2519 currentResult = main.TRUE
2520 else:
2521 main.log.error( "Error in ONOS node availability" )
2522 main.log.error(
2523 json.dumps( current,
2524 sort_keys=True,
2525 indent=4,
2526 separators=( ',', ': ' ) ) )
2527 break
2528 nodeResults = nodeResults and currentResult
2529 except ( ValueError, TypeError ):
2530 main.log.error( "Error parsing nodes output" )
2531 main.log.warn( repr( i ) )
2532 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2533 onpass="Nodes check successful",
2534 onfail="Nodes check NOT successful" )
2535
2536 - def CASE9( self, main ):
2537 """
2538 Link s3-s28 down
2539 """
2540 import time
2541 assert main.numCtrls, "main.numCtrls not defined"
2542 assert main, "main not defined"
2543 assert utilities.assert_equals, "utilities.assert_equals not defined"
2544 assert main.CLIs, "main.CLIs not defined"
2545 assert main.nodes, "main.nodes not defined"
2546
2547
2548 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2549
2550 description = "Turn off a link to ensure that Link Discovery " +\
2551 "is working properly"
2552 main.case( description )
2553
2554 main.step( "Kill Link between s3 and s28" )
2555 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2556 main.log.info( "Waiting " + str( linkSleep ) +
2557 " seconds for link down to be discovered" )
2558 time.sleep( linkSleep )
2559 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2560 onpass="Link down successful",
2561 onfail="Failed to bring link down" )
2562
2563
2565 """
2566 Link s3-s28 up
2567 """
2568 import time
2569 assert main.numCtrls, "main.numCtrls not defined"
2570 assert main, "main not defined"
2571 assert utilities.assert_equals, "utilities.assert_equals not defined"
2572 assert main.CLIs, "main.CLIs not defined"
2573 assert main.nodes, "main.nodes not defined"
2574
2575
2576 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2577
2578 description = "Restore a link to ensure that Link Discovery is " + \
2579 "working properly"
2580 main.case( description )
2581
2582 main.step( "Bring link between s3 and s28 back up" )
2583 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2584 main.log.info( "Waiting " + str( linkSleep ) +
2585 " seconds for link up to be discovered" )
2586 time.sleep( linkSleep )
2587 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2588 onpass="Link up successful",
2589 onfail="Failed to bring link up" )
2590
2591
2593 """
2594 Switch Down
2595 """
2596
2597 import time
2598 assert main.numCtrls, "main.numCtrls not defined"
2599 assert main, "main not defined"
2600 assert utilities.assert_equals, "utilities.assert_equals not defined"
2601 assert main.CLIs, "main.CLIs not defined"
2602 assert main.nodes, "main.nodes not defined"
2603
2604 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2605
2606 description = "Killing a switch to ensure it is discovered correctly"
2607 main.case( description )
2608 switch = main.params[ 'kill' ][ 'switch' ]
2609 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2610
2611
2612 main.step( "Kill " + switch )
2613 main.log.info( "Deleting " + switch )
2614 main.Mininet1.delSwitch( switch )
2615 main.log.info( "Waiting " + str( switchSleep ) +
2616 " seconds for switch down to be discovered" )
2617 time.sleep( switchSleep )
2618 device = main.ONOScli1.getDevice( dpid=switchDPID )
2619
2620 main.log.warn( str( device ) )
2621 result = main.FALSE
2622 if device and device[ 'available' ] is False:
2623 result = main.TRUE
2624 utilities.assert_equals( expect=main.TRUE, actual=result,
2625 onpass="Kill switch successful",
2626 onfail="Failed to kill switch?" )
2627
2629 """
2630 Switch Up
2631 """
2632
2633 import time
2634 assert main.numCtrls, "main.numCtrls not defined"
2635 assert main, "main not defined"
2636 assert utilities.assert_equals, "utilities.assert_equals not defined"
2637 assert main.CLIs, "main.CLIs not defined"
2638 assert main.nodes, "main.nodes not defined"
2639 assert ONOS1Port, "ONOS1Port not defined"
2640 assert ONOS2Port, "ONOS2Port not defined"
2641 assert ONOS3Port, "ONOS3Port not defined"
2642 assert ONOS4Port, "ONOS4Port not defined"
2643 assert ONOS5Port, "ONOS5Port not defined"
2644 assert ONOS6Port, "ONOS6Port not defined"
2645 assert ONOS7Port, "ONOS7Port not defined"
2646
2647 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2648 switch = main.params[ 'kill' ][ 'switch' ]
2649 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2650 links = main.params[ 'kill' ][ 'links' ].split()
2651 description = "Adding a switch to ensure it is discovered correctly"
2652 main.case( description )
2653
2654 main.step( "Add back " + switch )
2655 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2656 for peer in links:
2657 main.Mininet1.addLink( switch, peer )
2658 ipList = []
2659 for i in range( main.numCtrls ):
2660 ipList.append( main.nodes[ i ].ip_address )
2661 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2662 main.log.info( "Waiting " + str( switchSleep ) +
2663 " seconds for switch up to be discovered" )
2664 time.sleep( switchSleep )
2665 device = main.ONOScli1.getDevice( dpid=switchDPID )
2666
2667 main.log.warn( str( device ) )
2668 result = main.FALSE
2669 if device and device[ 'available' ]:
2670 result = main.TRUE
2671 utilities.assert_equals( expect=main.TRUE, actual=result,
2672 onpass="add switch successful",
2673 onfail="Failed to add switch?" )
2674
2676 """
2677 Clean up
2678 """
2679 import os
2680 import time
2681 assert main.numCtrls, "main.numCtrls not defined"
2682 assert main, "main not defined"
2683 assert utilities.assert_equals, "utilities.assert_equals not defined"
2684 assert main.CLIs, "main.CLIs not defined"
2685 assert main.nodes, "main.nodes not defined"
2686
2687
2688 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2689 'blue': '\033[94m', 'green': '\033[92m',
2690 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2691 main.case( "Test Cleanup" )
2692 main.step( "Killing tcpdumps" )
2693 main.Mininet2.stopTcpdump()
2694
2695 testname = main.TEST
2696 if main.params[ 'BACKUP' ] == "True":
2697 main.step( "Copying MN pcap and ONOS log files to test station" )
2698 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2699 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
2700
2701
2702
2703
2704
2705
2706 logFolder = "/opt/onos/log/"
2707 logFiles = [ "karaf.log", "karaf.log.1" ]
2708
2709 dstDir = "~/packet_captures/"
2710 for f in logFiles:
2711 for node in main.nodes:
2712 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2713 ":" + logFolder + f + " " +
2714 teststationUser + "@" +
2715 teststationIP + ":" +
2716 dstDir + str( testname ) +
2717 "-" + node.name + "-" + f )
2718 main.ONOSbench.handle.expect( "\$" )
2719
2720
2721
2722 logFolder = "/opt/onos/var/"
2723 logFiles = [ "stderr.log", "stdout.log" ]
2724
2725 dstDir = "~/packet_captures/"
2726 for f in logFiles:
2727 for node in main.nodes:
2728 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2729 ":" + logFolder + f + " " +
2730 teststationUser + "@" +
2731 teststationIP + ":" +
2732 dstDir + str( testname ) +
2733 "-" + node.name + "-" + f )
2734 main.ONOSbench.handle.expect( "\$" )
2735
2736 time.sleep( 10 )
2737 main.step( "Packing and rotating pcap archives" )
2738 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
2739
2740 main.step( "Stopping Mininet" )
2741 mnResult = main.Mininet1.stopNet()
2742 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2743 onpass="Mininet stopped",
2744 onfail="MN cleanup NOT successful" )
2745
2746 main.step( "Checking ONOS Logs for errors" )
2747 for node in main.nodes:
2748 print colors[ 'purple' ] + "Checking logs for errors on " + \
2749 node.name + ":" + colors[ 'end' ]
2750 print main.ONOSbench.checkLogs( node.ip_address, restart=True )
2751
2752 try:
2753 timerLog = open( main.logdir + "/Timers.csv", 'w')
2754 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2755 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2756 timerLog.close()
2757 except NameError, e:
2758 main.log.exception(e)
2759
2761 """
2762 start election app on all onos nodes
2763 """
2764 assert main.numCtrls, "main.numCtrls not defined"
2765 assert main, "main not defined"
2766 assert utilities.assert_equals, "utilities.assert_equals not defined"
2767 assert main.CLIs, "main.CLIs not defined"
2768 assert main.nodes, "main.nodes not defined"
2769
2770 main.case("Start Leadership Election app")
2771 main.step( "Install leadership election app" )
2772 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2773 utilities.assert_equals(
2774 expect=main.TRUE,
2775 actual=appResult,
2776 onpass="Election app installed",
2777 onfail="Something went wrong with installing Leadership election" )
2778
2779 main.step( "Run for election on each node" )
2780 leaderResult = main.TRUE
2781 leaders = []
2782 for cli in main.CLIs:
2783 cli.electionTestRun()
2784 for cli in main.CLIs:
2785 leader = cli.electionTestLeader()
2786 if leader is None or leader == main.FALSE:
2787 main.log.error( cli.name + ": Leader for the election app " +
2788 "should be an ONOS node, instead got '" +
2789 str( leader ) + "'" )
2790 leaderResult = main.FALSE
2791 leaders.append( leader )
2792 utilities.assert_equals(
2793 expect=main.TRUE,
2794 actual=leaderResult,
2795 onpass="Successfully ran for leadership",
2796 onfail="Failed to run for leadership" )
2797
2798 main.step( "Check that each node shows the same leader" )
2799 sameLeader = main.TRUE
2800 if len( set( leaders ) ) != 1:
2801 sameLeader = main.FALSE
2802 main.log.error( "Results of electionTestLeader is order of CLIs:" +
2803 str( leaders ) )
2804 utilities.assert_equals(
2805 expect=main.TRUE,
2806 actual=sameLeader,
2807 onpass="Leadership is consistent for the election topic",
2808 onfail="Nodes have different leaders" )
2809
2811 """
2812 Check that Leadership Election is still functional
2813 """
2814 import time
2815 assert main.numCtrls, "main.numCtrls not defined"
2816 assert main, "main not defined"
2817 assert utilities.assert_equals, "utilities.assert_equals not defined"
2818 assert main.CLIs, "main.CLIs not defined"
2819 assert main.nodes, "main.nodes not defined"
2820
2821 leaderResult = main.TRUE
2822 description = "Check that Leadership Election is still functional"
2823 main.case( description )
2824
2825 main.step( "Run for election on each node" )
2826 leaderResult = main.TRUE
2827 leaders = []
2828 for cli in main.CLIs:
2829 cli.electionTestRun()
2830 for cli in main.CLIs:
2831 leader = cli.electionTestLeader()
2832 if leader is None or leader == main.FALSE:
2833 main.log.error( cli.name + ": Leader for the election app " +
2834 "should be an ONOS node, instead got '" +
2835 str( leader ) + "'" )
2836 leaderResult = main.FALSE
2837 leaders.append( leader )
2838 utilities.assert_equals(
2839 expect=main.TRUE,
2840 actual=leaderResult,
2841 onpass="Successfully ran for leadership",
2842 onfail="Failed to run for leadership" )
2843
2844 main.step( "Check that each node shows the same leader" )
2845 sameLeader = main.TRUE
2846 if len( set( leaders ) ) != 1:
2847 sameLeader = main.FALSE
2848 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
2849 str( leaders ) )
2850 utilities.assert_equals(
2851 expect=main.TRUE,
2852 actual=sameLeader,
2853 onpass="Leadership is consistent for the election topic",
2854 onfail="Nodes have different leaders" )
2855
2856 main.step( "Find current leader and withdraw" )
2857 leader = main.ONOScli1.electionTestLeader()
2858
2859 withdrawResult = main.FALSE
2860 if leader is None or leader == main.FALSE:
2861 main.log.error(
2862 "Leader for the election app should be an ONOS node," +
2863 "instead got '" + str( leader ) + "'" )
2864 leaderResult = main.FALSE
2865 oldLeader = None
2866 for i in range( len( main.CLIs ) ):
2867 if leader == main.nodes[ i ].ip_address:
2868 oldLeader = main.CLIs[ i ]
2869 break
2870 else:
2871 main.log.error( "Leader election, could not find current leader" )
2872 if oldLeader:
2873 withdrawResult = oldLeader.electionTestWithdraw()
2874 utilities.assert_equals(
2875 expect=main.TRUE,
2876 actual=withdrawResult,
2877 onpass="Node was withdrawn from election",
2878 onfail="Node was not withdrawn from election" )
2879
2880 main.step( "Make sure new leader is elected" )
2881
2882 leaderList = []
2883 for cli in main.CLIs:
2884 leaderN = cli.electionTestLeader()
2885 leaderList.append( leaderN )
2886 if leaderN == leader:
2887 main.log.error( cli.name + " still sees " + str( leader ) +
2888 " as leader after they withdrew" )
2889 leaderResult = main.FALSE
2890 elif leaderN == main.FALSE:
2891
2892
2893
2894 main.log.error( "Something is wrong with " +
2895 "electionTestLeader function, " +
2896 "check the error logs" )
2897 leaderResult = main.FALSE
2898 elif leaderN is None:
2899
2900 time.sleep(7)
2901 leaderN = cli.electionTestLeader()
2902 leaderList.pop()
2903 leaderList.append( leaderN )
2904 consistentLeader = main.FALSE
2905 if len( set( leaderList ) ) == 1:
2906 main.log.info( "Each Election-app sees '" +
2907 str( leaderList[ 0 ] ) +
2908 "' as the leader" )
2909 consistentLeader = main.TRUE
2910 else:
2911 main.log.error(
2912 "Inconsistent responses for leader of Election-app:" )
2913 for n in range( len( leaderList ) ):
2914 main.log.error( "ONOS" + str( n + 1 ) + " response: " +
2915 str( leaderList[ n ] ) )
2916 leaderResult = leaderResult and consistentLeader
2917 utilities.assert_equals(
2918 expect=main.TRUE,
2919 actual=leaderResult,
2920 onpass="Leadership election passed",
2921 onfail="Something went wrong with Leadership election" )
2922
2923 main.step( "Run for election on old leader( just so everyone " +
2924 "is in the hat )" )
2925 if oldLeader:
2926 runResult = oldLeader.electionTestRun()
2927 else:
2928 runResult = main.FALSE
2929 utilities.assert_equals(
2930 expect=main.TRUE,
2931 actual=runResult,
2932 onpass="App re-ran for election",
2933 onfail="App failed to run for election" )
2934
2935 main.step( "Leader did not change when old leader re-ran" )
2936 afterRun = main.ONOScli1.electionTestLeader()
2937
2938 if afterRun == leaderList[ 0 ]:
2939 afterResult = main.TRUE
2940 else:
2941 afterResult = main.FALSE
2942
2943 utilities.assert_equals(
2944 expect=main.TRUE,
2945 actual=afterResult,
2946 onpass="Old leader successfully re-ran for election",
2947 onfail="Something went wrong with Leadership election after " +
2948 "the old leader re-ran for election" )
2949
2951 """
2952 Install Distributed Primitives app
2953 """
2954 import time
2955 assert main.numCtrls, "main.numCtrls not defined"
2956 assert main, "main not defined"
2957 assert utilities.assert_equals, "utilities.assert_equals not defined"
2958 assert main.CLIs, "main.CLIs not defined"
2959 assert main.nodes, "main.nodes not defined"
2960
2961
2962 global pCounterName
2963 global iCounterName
2964 global pCounterValue
2965 global iCounterValue
2966 global onosSet
2967 global onosSetName
2968 pCounterName = "TestON-Partitions"
2969 iCounterName = "TestON-inMemory"
2970 pCounterValue = 0
2971 iCounterValue = 0
2972 onosSet = set([])
2973 onosSetName = "TestON-set"
2974
2975 description = "Install Primitives app"
2976 main.case( description )
2977 main.step( "Install Primitives app" )
2978 appName = "org.onosproject.distributedprimitives"
2979 appResults = main.CLIs[0].activateApp( appName )
2980 utilities.assert_equals( expect=main.TRUE,
2981 actual=appResults,
2982 onpass="Primitives app activated",
2983 onfail="Primitives app not activated" )
2984 time.sleep( 5 )
2985
2987 """
2988 Check for basic functionality with distributed primitives
2989 """
2990
2991 assert main.numCtrls, "main.numCtrls not defined"
2992 assert main, "main not defined"
2993 assert utilities.assert_equals, "utilities.assert_equals not defined"
2994 assert main.CLIs, "main.CLIs not defined"
2995 assert main.nodes, "main.nodes not defined"
2996 assert pCounterName, "pCounterName not defined"
2997 assert iCounterName, "iCounterName not defined"
2998 assert onosSetName, "onosSetName not defined"
2999
3000 try:
3001 pCounterValue
3002 except NameError:
3003 main.log.error( "pCounterValue not defined, setting to 0" )
3004 pCounterValue = 0
3005 try:
3006 iCounterValue
3007 except NameError:
3008 main.log.error( "iCounterValue not defined, setting to 0" )
3009 iCounterValue = 0
3010 try:
3011 onosSet
3012 except NameError:
3013 main.log.error( "onosSet not defined, setting to empty Set" )
3014 onosSet = set([])
3015
3016 addValue = "a"
3017 addAllValue = "a b c d e f"
3018 retainValue = "c d e f"
3019
3020 description = "Check for basic functionality with distributed " +\
3021 "primitives"
3022 main.case( description )
3023 main.caseExplanation = "Test the methods of the distributed " +\
3024 "primitives (counters and sets) throught the cli"
3025
3026
3027 main.step( "Increment then get a default counter on each node" )
3028 pCounters = []
3029 threads = []
3030 addedPValues = []
3031 for i in range( main.numCtrls ):
3032 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3033 name="counterAddAndGet-" + str( i ),
3034 args=[ pCounterName ] )
3035 pCounterValue += 1
3036 addedPValues.append( pCounterValue )
3037 threads.append( t )
3038 t.start()
3039
3040 for t in threads:
3041 t.join()
3042 pCounters.append( t.result )
3043
3044 pCounterResults = True
3045 for i in addedPValues:
3046 tmpResult = i in pCounters
3047 pCounterResults = pCounterResults and tmpResult
3048 if not tmpResult:
3049 main.log.error( str( i ) + " is not in partitioned "
3050 "counter incremented results" )
3051 utilities.assert_equals( expect=True,
3052 actual=pCounterResults,
3053 onpass="Default counter incremented",
3054 onfail="Error incrementing default" +
3055 " counter" )
3056
3057 main.step( "Get then Increment a default counter on each node" )
3058 pCounters = []
3059 threads = []
3060 addedPValues = []
3061 for i in range( main.numCtrls ):
3062 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3063 name="counterGetAndAdd-" + str( i ),
3064 args=[ pCounterName ] )
3065 addedPValues.append( pCounterValue )
3066 pCounterValue += 1
3067 threads.append( t )
3068 t.start()
3069
3070 for t in threads:
3071 t.join()
3072 pCounters.append( t.result )
3073
3074 pCounterResults = True
3075 for i in addedPValues:
3076 tmpResult = i in pCounters
3077 pCounterResults = pCounterResults and tmpResult
3078 if not tmpResult:
3079 main.log.error( str( i ) + " is not in partitioned "
3080 "counter incremented results" )
3081 utilities.assert_equals( expect=True,
3082 actual=pCounterResults,
3083 onpass="Default counter incremented",
3084 onfail="Error incrementing default" +
3085 " counter" )
3086
3087 main.step( "Counters we added have the correct values" )
3088 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3089 utilities.assert_equals( expect=main.TRUE,
3090 actual=incrementCheck,
3091 onpass="Added counters are correct",
3092 onfail="Added counters are incorrect" )
3093
3094 main.step( "Add -8 to then get a default counter on each node" )
3095 pCounters = []
3096 threads = []
3097 addedPValues = []
3098 for i in range( main.numCtrls ):
3099 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3100 name="counterIncrement-" + str( i ),
3101 args=[ pCounterName ],
3102 kwargs={ "delta": -8 } )
3103 pCounterValue += -8
3104 addedPValues.append( pCounterValue )
3105 threads.append( t )
3106 t.start()
3107
3108 for t in threads:
3109 t.join()
3110 pCounters.append( t.result )
3111
3112 pCounterResults = True
3113 for i in addedPValues:
3114 tmpResult = i in pCounters
3115 pCounterResults = pCounterResults and tmpResult
3116 if not tmpResult:
3117 main.log.error( str( i ) + " is not in partitioned "
3118 "counter incremented results" )
3119 utilities.assert_equals( expect=True,
3120 actual=pCounterResults,
3121 onpass="Default counter incremented",
3122 onfail="Error incrementing default" +
3123 " counter" )
3124
3125 main.step( "Add 5 to then get a default counter on each node" )
3126 pCounters = []
3127 threads = []
3128 addedPValues = []
3129 for i in range( main.numCtrls ):
3130 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3131 name="counterIncrement-" + str( i ),
3132 args=[ pCounterName ],
3133 kwargs={ "delta": 5 } )
3134 pCounterValue += 5
3135 addedPValues.append( pCounterValue )
3136 threads.append( t )
3137 t.start()
3138
3139 for t in threads:
3140 t.join()
3141 pCounters.append( t.result )
3142
3143 pCounterResults = True
3144 for i in addedPValues:
3145 tmpResult = i in pCounters
3146 pCounterResults = pCounterResults and tmpResult
3147 if not tmpResult:
3148 main.log.error( str( i ) + " is not in partitioned "
3149 "counter incremented results" )
3150 utilities.assert_equals( expect=True,
3151 actual=pCounterResults,
3152 onpass="Default counter incremented",
3153 onfail="Error incrementing default" +
3154 " counter" )
3155
3156 main.step( "Get then add 5 to a default counter on each node" )
3157 pCounters = []
3158 threads = []
3159 addedPValues = []
3160 for i in range( main.numCtrls ):
3161 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3162 name="counterIncrement-" + str( i ),
3163 args=[ pCounterName ],
3164 kwargs={ "delta": 5 } )
3165 addedPValues.append( pCounterValue )
3166 pCounterValue += 5
3167 threads.append( t )
3168 t.start()
3169
3170 for t in threads:
3171 t.join()
3172 pCounters.append( t.result )
3173
3174 pCounterResults = True
3175 for i in addedPValues:
3176 tmpResult = i in pCounters
3177 pCounterResults = pCounterResults and tmpResult
3178 if not tmpResult:
3179 main.log.error( str( i ) + " is not in partitioned "
3180 "counter incremented results" )
3181 utilities.assert_equals( expect=True,
3182 actual=pCounterResults,
3183 onpass="Default counter incremented",
3184 onfail="Error incrementing default" +
3185 " counter" )
3186
3187 main.step( "Counters we added have the correct values" )
3188 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3189 utilities.assert_equals( expect=main.TRUE,
3190 actual=incrementCheck,
3191 onpass="Added counters are correct",
3192 onfail="Added counters are incorrect" )
3193
3194
3195 main.step( "Increment and get an in-memory counter on each node" )
3196 iCounters = []
3197 addedIValues = []
3198 threads = []
3199 for i in range( main.numCtrls ):
3200 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3201 name="icounterIncrement-" + str( i ),
3202 args=[ iCounterName ],
3203 kwargs={ "inMemory": True } )
3204 iCounterValue += 1
3205 addedIValues.append( iCounterValue )
3206 threads.append( t )
3207 t.start()
3208
3209 for t in threads:
3210 t.join()
3211 iCounters.append( t.result )
3212
3213 iCounterResults = True
3214 for i in addedIValues:
3215 tmpResult = i in iCounters
3216 iCounterResults = iCounterResults and tmpResult
3217 if not tmpResult:
3218 main.log.error( str( i ) + " is not in the in-memory "
3219 "counter incremented results" )
3220 utilities.assert_equals( expect=True,
3221 actual=iCounterResults,
3222 onpass="In-memory counter incremented",
3223 onfail="Error incrementing in-memory" +
3224 " counter" )
3225
3226 main.step( "Get then Increment a in-memory counter on each node" )
3227 iCounters = []
3228 threads = []
3229 addedIValues = []
3230 for i in range( main.numCtrls ):
3231 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3232 name="counterGetAndAdd-" + str( i ),
3233 args=[ iCounterName ],
3234 kwargs={ "inMemory": True } )
3235 addedIValues.append( iCounterValue )
3236 iCounterValue += 1
3237 threads.append( t )
3238 t.start()
3239
3240 for t in threads:
3241 t.join()
3242 iCounters.append( t.result )
3243
3244 iCounterResults = True
3245 for i in addedIValues:
3246 tmpResult = i in iCounters
3247 iCounterResults = iCounterResults and tmpResult
3248 if not tmpResult:
3249 main.log.error( str( i ) + " is not in in-memory "
3250 "counter incremented results" )
3251 utilities.assert_equals( expect=True,
3252 actual=iCounterResults,
3253 onpass="In-memory counter incremented",
3254 onfail="Error incrementing in-memory" +
3255 " counter" )
3256
3257 main.step( "Counters we added have the correct values" )
3258 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3259 utilities.assert_equals( expect=main.TRUE,
3260 actual=incrementCheck,
3261 onpass="Added counters are correct",
3262 onfail="Added counters are incorrect" )
3263
3264 main.step( "Add -8 to then get a in-memory counter on each node" )
3265 iCounters = []
3266 threads = []
3267 addedIValues = []
3268 for i in range( main.numCtrls ):
3269 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3270 name="counterIncrement-" + str( i ),
3271 args=[ iCounterName ],
3272 kwargs={ "delta": -8, "inMemory": True } )
3273 iCounterValue += -8
3274 addedIValues.append( iCounterValue )
3275 threads.append( t )
3276 t.start()
3277
3278 for t in threads:
3279 t.join()
3280 iCounters.append( t.result )
3281
3282 iCounterResults = True
3283 for i in addedIValues:
3284 tmpResult = i in iCounters
3285 iCounterResults = iCounterResults and tmpResult
3286 if not tmpResult:
3287 main.log.error( str( i ) + " is not in in-memory "
3288 "counter incremented results" )
3289 utilities.assert_equals( expect=True,
3290 actual=pCounterResults,
3291 onpass="In-memory counter incremented",
3292 onfail="Error incrementing in-memory" +
3293 " counter" )
3294
3295 main.step( "Add 5 to then get a in-memory counter on each node" )
3296 iCounters = []
3297 threads = []
3298 addedIValues = []
3299 for i in range( main.numCtrls ):
3300 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3301 name="counterIncrement-" + str( i ),
3302 args=[ iCounterName ],
3303 kwargs={ "delta": 5, "inMemory": True } )
3304 iCounterValue += 5
3305 addedIValues.append( iCounterValue )
3306 threads.append( t )
3307 t.start()
3308
3309 for t in threads:
3310 t.join()
3311 iCounters.append( t.result )
3312
3313 iCounterResults = True
3314 for i in addedIValues:
3315 tmpResult = i in iCounters
3316 iCounterResults = iCounterResults and tmpResult
3317 if not tmpResult:
3318 main.log.error( str( i ) + " is not in in-memory "
3319 "counter incremented results" )
3320 utilities.assert_equals( expect=True,
3321 actual=pCounterResults,
3322 onpass="In-memory counter incremented",
3323 onfail="Error incrementing in-memory" +
3324 " counter" )
3325
3326 main.step( "Get then add 5 to a in-memory counter on each node" )
3327 iCounters = []
3328 threads = []
3329 addedIValues = []
3330 for i in range( main.numCtrls ):
3331 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3332 name="counterIncrement-" + str( i ),
3333 args=[ iCounterName ],
3334 kwargs={ "delta": 5, "inMemory": True } )
3335 addedIValues.append( iCounterValue )
3336 iCounterValue += 5
3337 threads.append( t )
3338 t.start()
3339
3340 for t in threads:
3341 t.join()
3342 iCounters.append( t.result )
3343
3344 iCounterResults = True
3345 for i in addedIValues:
3346 tmpResult = i in iCounters
3347 iCounterResults = iCounterResults and tmpResult
3348 if not tmpResult:
3349 main.log.error( str( i ) + " is not in in-memory "
3350 "counter incremented results" )
3351 utilities.assert_equals( expect=True,
3352 actual=iCounterResults,
3353 onpass="In-memory counter incremented",
3354 onfail="Error incrementing in-memory" +
3355 " counter" )
3356
3357 main.step( "Counters we added have the correct values" )
3358 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3359 utilities.assert_equals( expect=main.TRUE,
3360 actual=incrementCheck,
3361 onpass="Added counters are correct",
3362 onfail="Added counters are incorrect" )
3363
3364 main.step( "Check counters are consistant across nodes" )
3365 onosCounters = []
3366 threads = []
3367 for i in range( main.numCtrls ):
3368 t = main.Thread( target=main.CLIs[i].counters,
3369 name="counters-" + str( i ) )
3370 threads.append( t )
3371 t.start()
3372 for t in threads:
3373 t.join()
3374 onosCounters.append( t.result )
3375 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3376 if all( tmp ):
3377 main.log.info( "Counters are consistent across all nodes" )
3378 consistentCounterResults = main.TRUE
3379 else:
3380 main.log.error( "Counters are not consistent across all nodes" )
3381 consistentCounterResults = main.FALSE
3382 utilities.assert_equals( expect=main.TRUE,
3383 actual=consistentCounterResults,
3384 onpass="ONOS counters are consistent " +
3385 "across nodes",
3386 onfail="ONOS Counters are inconsistent " +
3387 "across nodes" )
3388
3389 main.step( "Counters we added have the correct values" )
3390 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3391 incrementCheck = incrementCheck and \
3392 main.Counters.counterCheck( iCounterName, iCounterValue )
3393 utilities.assert_equals( expect=main.TRUE,
3394 actual=incrementCheck,
3395 onpass="Added counters are correct",
3396 onfail="Added counters are incorrect" )
3397
3398
3399 main.step( "Distributed Set get" )
3400 size = len( onosSet )
3401 getResponses = []
3402 threads = []
3403 for i in range( main.numCtrls ):
3404 t = main.Thread( target=main.CLIs[i].setTestGet,
3405 name="setTestGet-" + str( i ),
3406 args=[ onosSetName ] )
3407 threads.append( t )
3408 t.start()
3409 for t in threads:
3410 t.join()
3411 getResponses.append( t.result )
3412
3413 getResults = main.TRUE
3414 for i in range( main.numCtrls ):
3415 if isinstance( getResponses[ i ], list):
3416 current = set( getResponses[ i ] )
3417 if len( current ) == len( getResponses[ i ] ):
3418
3419 if onosSet != current:
3420 main.log.error( "ONOS" + str( i + 1 ) +
3421 " has incorrect view" +
3422 " of set " + onosSetName + ":\n" +
3423 str( getResponses[ i ] ) )
3424 main.log.debug( "Expected: " + str( onosSet ) )
3425 main.log.debug( "Actual: " + str( current ) )
3426 getResults = main.FALSE
3427 else:
3428
3429 main.log.error( "ONOS" + str( i + 1 ) +
3430 " has repeat elements in" +
3431 " set " + onosSetName + ":\n" +
3432 str( getResponses[ i ] ) )
3433 getResults = main.FALSE
3434 elif getResponses[ i ] == main.ERROR:
3435 getResults = main.FALSE
3436 utilities.assert_equals( expect=main.TRUE,
3437 actual=getResults,
3438 onpass="Set elements are correct",
3439 onfail="Set elements are incorrect" )
3440
3441 main.step( "Distributed Set size" )
3442 sizeResponses = []
3443 threads = []
3444 for i in range( main.numCtrls ):
3445 t = main.Thread( target=main.CLIs[i].setTestSize,
3446 name="setTestSize-" + str( i ),
3447 args=[ onosSetName ] )
3448 threads.append( t )
3449 t.start()
3450 for t in threads:
3451 t.join()
3452 sizeResponses.append( t.result )
3453
3454 sizeResults = main.TRUE
3455 for i in range( main.numCtrls ):
3456 if size != sizeResponses[ i ]:
3457 sizeResults = main.FALSE
3458 main.log.error( "ONOS" + str( i + 1 ) +
3459 " expected a size of " + str( size ) +
3460 " for set " + onosSetName +
3461 " but got " + str( sizeResponses[ i ] ) )
3462 utilities.assert_equals( expect=main.TRUE,
3463 actual=sizeResults,
3464 onpass="Set sizes are correct",
3465 onfail="Set sizes are incorrect" )
3466
3467 main.step( "Distributed Set add()" )
3468 onosSet.add( addValue )
3469 addResponses = []
3470 threads = []
3471 for i in range( main.numCtrls ):
3472 t = main.Thread( target=main.CLIs[i].setTestAdd,
3473 name="setTestAdd-" + str( i ),
3474 args=[ onosSetName, addValue ] )
3475 threads.append( t )
3476 t.start()
3477 for t in threads:
3478 t.join()
3479 addResponses.append( t.result )
3480
3481
3482
3483
3484 addResults = main.TRUE
3485 for i in range( main.numCtrls ):
3486 if addResponses[ i ] == main.TRUE:
3487
3488 pass
3489 elif addResponses[ i ] == main.FALSE:
3490
3491 pass
3492 elif addResponses[ i ] == main.ERROR:
3493
3494 addResults = main.FALSE
3495 else:
3496
3497 addResults = main.FALSE
3498 if addResults != main.TRUE:
3499 main.log.error( "Error executing set add" )
3500
3501
3502 size = len( onosSet )
3503 getResponses = []
3504 threads = []
3505 for i in range( main.numCtrls ):
3506 t = main.Thread( target=main.CLIs[i].setTestGet,
3507 name="setTestGet-" + str( i ),
3508 args=[ onosSetName ] )
3509 threads.append( t )
3510 t.start()
3511 for t in threads:
3512 t.join()
3513 getResponses.append( t.result )
3514 getResults = main.TRUE
3515 for i in range( main.numCtrls ):
3516 if isinstance( getResponses[ i ], list):
3517 current = set( getResponses[ i ] )
3518 if len( current ) == len( getResponses[ i ] ):
3519
3520 if onosSet != current:
3521 main.log.error( "ONOS" + str( i + 1 ) +
3522 " has incorrect view" +
3523 " of set " + onosSetName + ":\n" +
3524 str( getResponses[ i ] ) )
3525 main.log.debug( "Expected: " + str( onosSet ) )
3526 main.log.debug( "Actual: " + str( current ) )
3527 getResults = main.FALSE
3528 else:
3529
3530 main.log.error( "ONOS" + str( i + 1 ) +
3531 " has repeat elements in" +
3532 " set " + onosSetName + ":\n" +
3533 str( getResponses[ i ] ) )
3534 getResults = main.FALSE
3535 elif getResponses[ i ] == main.ERROR:
3536 getResults = main.FALSE
3537 sizeResponses = []
3538 threads = []
3539 for i in range( main.numCtrls ):
3540 t = main.Thread( target=main.CLIs[i].setTestSize,
3541 name="setTestSize-" + str( i ),
3542 args=[ onosSetName ] )
3543 threads.append( t )
3544 t.start()
3545 for t in threads:
3546 t.join()
3547 sizeResponses.append( t.result )
3548 sizeResults = main.TRUE
3549 for i in range( main.numCtrls ):
3550 if size != sizeResponses[ i ]:
3551 sizeResults = main.FALSE
3552 main.log.error( "ONOS" + str( i + 1 ) +
3553 " expected a size of " + str( size ) +
3554 " for set " + onosSetName +
3555 " but got " + str( sizeResponses[ i ] ) )
3556 addResults = addResults and getResults and sizeResults
3557 utilities.assert_equals( expect=main.TRUE,
3558 actual=addResults,
3559 onpass="Set add correct",
3560 onfail="Set add was incorrect" )
3561
3562 main.step( "Distributed Set addAll()" )
3563 onosSet.update( addAllValue.split() )
3564 addResponses = []
3565 threads = []
3566 for i in range( main.numCtrls ):
3567 t = main.Thread( target=main.CLIs[i].setTestAdd,
3568 name="setTestAddAll-" + str( i ),
3569 args=[ onosSetName, addAllValue ] )
3570 threads.append( t )
3571 t.start()
3572 for t in threads:
3573 t.join()
3574 addResponses.append( t.result )
3575
3576
3577
3578
3579 addAllResults = main.TRUE
3580 for i in range( main.numCtrls ):
3581 if addResponses[ i ] == main.TRUE:
3582
3583 pass
3584 elif addResponses[ i ] == main.FALSE:
3585
3586 pass
3587 elif addResponses[ i ] == main.ERROR:
3588
3589 addAllResults = main.FALSE
3590 else:
3591
3592 addAllResults = main.FALSE
3593 if addAllResults != main.TRUE:
3594 main.log.error( "Error executing set addAll" )
3595
3596
3597 size = len( onosSet )
3598 getResponses = []
3599 threads = []
3600 for i in range( main.numCtrls ):
3601 t = main.Thread( target=main.CLIs[i].setTestGet,
3602 name="setTestGet-" + str( i ),
3603 args=[ onosSetName ] )
3604 threads.append( t )
3605 t.start()
3606 for t in threads:
3607 t.join()
3608 getResponses.append( t.result )
3609 getResults = main.TRUE
3610 for i in range( main.numCtrls ):
3611 if isinstance( getResponses[ i ], list):
3612 current = set( getResponses[ i ] )
3613 if len( current ) == len( getResponses[ i ] ):
3614
3615 if onosSet != current:
3616 main.log.error( "ONOS" + str( i + 1 ) +
3617 " has incorrect view" +
3618 " of set " + onosSetName + ":\n" +
3619 str( getResponses[ i ] ) )
3620 main.log.debug( "Expected: " + str( onosSet ) )
3621 main.log.debug( "Actual: " + str( current ) )
3622 getResults = main.FALSE
3623 else:
3624
3625 main.log.error( "ONOS" + str( i + 1 ) +
3626 " has repeat elements in" +
3627 " set " + onosSetName + ":\n" +
3628 str( getResponses[ i ] ) )
3629 getResults = main.FALSE
3630 elif getResponses[ i ] == main.ERROR:
3631 getResults = main.FALSE
3632 sizeResponses = []
3633 threads = []
3634 for i in range( main.numCtrls ):
3635 t = main.Thread( target=main.CLIs[i].setTestSize,
3636 name="setTestSize-" + str( i ),
3637 args=[ onosSetName ] )
3638 threads.append( t )
3639 t.start()
3640 for t in threads:
3641 t.join()
3642 sizeResponses.append( t.result )
3643 sizeResults = main.TRUE
3644 for i in range( main.numCtrls ):
3645 if size != sizeResponses[ i ]:
3646 sizeResults = main.FALSE
3647 main.log.error( "ONOS" + str( i + 1 ) +
3648 " expected a size of " + str( size ) +
3649 " for set " + onosSetName +
3650 " but got " + str( sizeResponses[ i ] ) )
3651 addAllResults = addAllResults and getResults and sizeResults
3652 utilities.assert_equals( expect=main.TRUE,
3653 actual=addAllResults,
3654 onpass="Set addAll correct",
3655 onfail="Set addAll was incorrect" )
3656
3657 main.step( "Distributed Set contains()" )
3658 containsResponses = []
3659 threads = []
3660 for i in range( main.numCtrls ):
3661 t = main.Thread( target=main.CLIs[i].setTestGet,
3662 name="setContains-" + str( i ),
3663 args=[ onosSetName ],
3664 kwargs={ "values": addValue } )
3665 threads.append( t )
3666 t.start()
3667 for t in threads:
3668 t.join()
3669
3670 containsResponses.append( t.result )
3671
3672 containsResults = main.TRUE
3673 for i in range( main.numCtrls ):
3674 if containsResponses[ i ] == main.ERROR:
3675 containsResults = main.FALSE
3676 else:
3677 containsResults = containsResults and\
3678 containsResponses[ i ][ 1 ]
3679 utilities.assert_equals( expect=main.TRUE,
3680 actual=containsResults,
3681 onpass="Set contains is functional",
3682 onfail="Set contains failed" )
3683
3684 main.step( "Distributed Set containsAll()" )
3685 containsAllResponses = []
3686 threads = []
3687 for i in range( main.numCtrls ):
3688 t = main.Thread( target=main.CLIs[i].setTestGet,
3689 name="setContainsAll-" + str( i ),
3690 args=[ onosSetName ],
3691 kwargs={ "values": addAllValue } )
3692 threads.append( t )
3693 t.start()
3694 for t in threads:
3695 t.join()
3696
3697 containsAllResponses.append( t.result )
3698
3699 containsAllResults = main.TRUE
3700 for i in range( main.numCtrls ):
3701 if containsResponses[ i ] == main.ERROR:
3702 containsResults = main.FALSE
3703 else:
3704 containsResults = containsResults and\
3705 containsResponses[ i ][ 1 ]
3706 utilities.assert_equals( expect=main.TRUE,
3707 actual=containsAllResults,
3708 onpass="Set containsAll is functional",
3709 onfail="Set containsAll failed" )
3710
3711 main.step( "Distributed Set remove()" )
3712 onosSet.remove( addValue )
3713 removeResponses = []
3714 threads = []
3715 for i in range( main.numCtrls ):
3716 t = main.Thread( target=main.CLIs[i].setTestRemove,
3717 name="setTestRemove-" + str( i ),
3718 args=[ onosSetName, addValue ] )
3719 threads.append( t )
3720 t.start()
3721 for t in threads:
3722 t.join()
3723 removeResponses.append( t.result )
3724
3725
3726
3727
3728 removeResults = main.TRUE
3729 for i in range( main.numCtrls ):
3730 if removeResponses[ i ] == main.TRUE:
3731
3732 pass
3733 elif removeResponses[ i ] == main.FALSE:
3734
3735 pass
3736 elif removeResponses[ i ] == main.ERROR:
3737
3738 removeResults = main.FALSE
3739 else:
3740
3741 removeResults = main.FALSE
3742 if removeResults != main.TRUE:
3743 main.log.error( "Error executing set remove" )
3744
3745
3746 size = len( onosSet )
3747 getResponses = []
3748 threads = []
3749 for i in range( main.numCtrls ):
3750 t = main.Thread( target=main.CLIs[i].setTestGet,
3751 name="setTestGet-" + str( i ),
3752 args=[ onosSetName ] )
3753 threads.append( t )
3754 t.start()
3755 for t in threads:
3756 t.join()
3757 getResponses.append( t.result )
3758 getResults = main.TRUE
3759 for i in range( main.numCtrls ):
3760 if isinstance( getResponses[ i ], list):
3761 current = set( getResponses[ i ] )
3762 if len( current ) == len( getResponses[ i ] ):
3763
3764 if onosSet != current:
3765 main.log.error( "ONOS" + str( i + 1 ) +
3766 " has incorrect view" +
3767 " of set " + onosSetName + ":\n" +
3768 str( getResponses[ i ] ) )
3769 main.log.debug( "Expected: " + str( onosSet ) )
3770 main.log.debug( "Actual: " + str( current ) )
3771 getResults = main.FALSE
3772 else:
3773
3774 main.log.error( "ONOS" + str( i + 1 ) +
3775 " has repeat elements in" +
3776 " set " + onosSetName + ":\n" +
3777 str( getResponses[ i ] ) )
3778 getResults = main.FALSE
3779 elif getResponses[ i ] == main.ERROR:
3780 getResults = main.FALSE
3781 sizeResponses = []
3782 threads = []
3783 for i in range( main.numCtrls ):
3784 t = main.Thread( target=main.CLIs[i].setTestSize,
3785 name="setTestSize-" + str( i ),
3786 args=[ onosSetName ] )
3787 threads.append( t )
3788 t.start()
3789 for t in threads:
3790 t.join()
3791 sizeResponses.append( t.result )
3792 sizeResults = main.TRUE
3793 for i in range( main.numCtrls ):
3794 if size != sizeResponses[ i ]:
3795 sizeResults = main.FALSE
3796 main.log.error( "ONOS" + str( i + 1 ) +
3797 " expected a size of " + str( size ) +
3798 " for set " + onosSetName +
3799 " but got " + str( sizeResponses[ i ] ) )
3800 removeResults = removeResults and getResults and sizeResults
3801 utilities.assert_equals( expect=main.TRUE,
3802 actual=removeResults,
3803 onpass="Set remove correct",
3804 onfail="Set remove was incorrect" )
3805
3806 main.step( "Distributed Set removeAll()" )
3807 onosSet.difference_update( addAllValue.split() )
3808 removeAllResponses = []
3809 threads = []
3810 try:
3811 for i in range( main.numCtrls ):
3812 t = main.Thread( target=main.CLIs[i].setTestRemove,
3813 name="setTestRemoveAll-" + str( i ),
3814 args=[ onosSetName, addAllValue ] )
3815 threads.append( t )
3816 t.start()
3817 for t in threads:
3818 t.join()
3819 removeAllResponses.append( t.result )
3820 except Exception, e:
3821 main.log.exception(e)
3822
3823
3824
3825
3826 removeAllResults = main.TRUE
3827 for i in range( main.numCtrls ):
3828 if removeAllResponses[ i ] == main.TRUE:
3829
3830 pass
3831 elif removeAllResponses[ i ] == main.FALSE:
3832
3833 pass
3834 elif removeAllResponses[ i ] == main.ERROR:
3835
3836 removeAllResults = main.FALSE
3837 else:
3838
3839 removeAllResults = main.FALSE
3840 if removeAllResults != main.TRUE:
3841 main.log.error( "Error executing set removeAll" )
3842
3843
3844 size = len( onosSet )
3845 getResponses = []
3846 threads = []
3847 for i in range( main.numCtrls ):
3848 t = main.Thread( target=main.CLIs[i].setTestGet,
3849 name="setTestGet-" + str( i ),
3850 args=[ onosSetName ] )
3851 threads.append( t )
3852 t.start()
3853 for t in threads:
3854 t.join()
3855 getResponses.append( t.result )
3856 getResults = main.TRUE
3857 for i in range( main.numCtrls ):
3858 if isinstance( getResponses[ i ], list):
3859 current = set( getResponses[ i ] )
3860 if len( current ) == len( getResponses[ i ] ):
3861
3862 if onosSet != current:
3863 main.log.error( "ONOS" + str( i + 1 ) +
3864 " has incorrect view" +
3865 " of set " + onosSetName + ":\n" +
3866 str( getResponses[ i ] ) )
3867 main.log.debug( "Expected: " + str( onosSet ) )
3868 main.log.debug( "Actual: " + str( current ) )
3869 getResults = main.FALSE
3870 else:
3871
3872 main.log.error( "ONOS" + str( i + 1 ) +
3873 " has repeat elements in" +
3874 " set " + onosSetName + ":\n" +
3875 str( getResponses[ i ] ) )
3876 getResults = main.FALSE
3877 elif getResponses[ i ] == main.ERROR:
3878 getResults = main.FALSE
3879 sizeResponses = []
3880 threads = []
3881 for i in range( main.numCtrls ):
3882 t = main.Thread( target=main.CLIs[i].setTestSize,
3883 name="setTestSize-" + str( i ),
3884 args=[ onosSetName ] )
3885 threads.append( t )
3886 t.start()
3887 for t in threads:
3888 t.join()
3889 sizeResponses.append( t.result )
3890 sizeResults = main.TRUE
3891 for i in range( main.numCtrls ):
3892 if size != sizeResponses[ i ]:
3893 sizeResults = main.FALSE
3894 main.log.error( "ONOS" + str( i + 1 ) +
3895 " expected a size of " + str( size ) +
3896 " for set " + onosSetName +
3897 " but got " + str( sizeResponses[ i ] ) )
3898 removeAllResults = removeAllResults and getResults and sizeResults
3899 utilities.assert_equals( expect=main.TRUE,
3900 actual=removeAllResults,
3901 onpass="Set removeAll correct",
3902 onfail="Set removeAll was incorrect" )
3903
3904 main.step( "Distributed Set addAll()" )
3905 onosSet.update( addAllValue.split() )
3906 addResponses = []
3907 threads = []
3908 for i in range( main.numCtrls ):
3909 t = main.Thread( target=main.CLIs[i].setTestAdd,
3910 name="setTestAddAll-" + str( i ),
3911 args=[ onosSetName, addAllValue ] )
3912 threads.append( t )
3913 t.start()
3914 for t in threads:
3915 t.join()
3916 addResponses.append( t.result )
3917
3918
3919
3920
3921 addAllResults = main.TRUE
3922 for i in range( main.numCtrls ):
3923 if addResponses[ i ] == main.TRUE:
3924
3925 pass
3926 elif addResponses[ i ] == main.FALSE:
3927
3928 pass
3929 elif addResponses[ i ] == main.ERROR:
3930
3931 addAllResults = main.FALSE
3932 else:
3933
3934 addAllResults = main.FALSE
3935 if addAllResults != main.TRUE:
3936 main.log.error( "Error executing set addAll" )
3937
3938
3939 size = len( onosSet )
3940 getResponses = []
3941 threads = []
3942 for i in range( main.numCtrls ):
3943 t = main.Thread( target=main.CLIs[i].setTestGet,
3944 name="setTestGet-" + str( i ),
3945 args=[ onosSetName ] )
3946 threads.append( t )
3947 t.start()
3948 for t in threads:
3949 t.join()
3950 getResponses.append( t.result )
3951 getResults = main.TRUE
3952 for i in range( main.numCtrls ):
3953 if isinstance( getResponses[ i ], list):
3954 current = set( getResponses[ i ] )
3955 if len( current ) == len( getResponses[ i ] ):
3956
3957 if onosSet != current:
3958 main.log.error( "ONOS" + str( i + 1 ) +
3959 " has incorrect view" +
3960 " of set " + onosSetName + ":\n" +
3961 str( getResponses[ i ] ) )
3962 main.log.debug( "Expected: " + str( onosSet ) )
3963 main.log.debug( "Actual: " + str( current ) )
3964 getResults = main.FALSE
3965 else:
3966
3967 main.log.error( "ONOS" + str( i + 1 ) +
3968 " has repeat elements in" +
3969 " set " + onosSetName + ":\n" +
3970 str( getResponses[ i ] ) )
3971 getResults = main.FALSE
3972 elif getResponses[ i ] == main.ERROR:
3973 getResults = main.FALSE
3974 sizeResponses = []
3975 threads = []
3976 for i in range( main.numCtrls ):
3977 t = main.Thread( target=main.CLIs[i].setTestSize,
3978 name="setTestSize-" + str( i ),
3979 args=[ onosSetName ] )
3980 threads.append( t )
3981 t.start()
3982 for t in threads:
3983 t.join()
3984 sizeResponses.append( t.result )
3985 sizeResults = main.TRUE
3986 for i in range( main.numCtrls ):
3987 if size != sizeResponses[ i ]:
3988 sizeResults = main.FALSE
3989 main.log.error( "ONOS" + str( i + 1 ) +
3990 " expected a size of " + str( size ) +
3991 " for set " + onosSetName +
3992 " but got " + str( sizeResponses[ i ] ) )
3993 addAllResults = addAllResults and getResults and sizeResults
3994 utilities.assert_equals( expect=main.TRUE,
3995 actual=addAllResults,
3996 onpass="Set addAll correct",
3997 onfail="Set addAll was incorrect" )
3998
3999 main.step( "Distributed Set clear()" )
4000 onosSet.clear()
4001 clearResponses = []
4002 threads = []
4003 for i in range( main.numCtrls ):
4004 t = main.Thread( target=main.CLIs[i].setTestRemove,
4005 name="setTestClear-" + str( i ),
4006 args=[ onosSetName, " "],
4007 kwargs={ "clear": True } )
4008 threads.append( t )
4009 t.start()
4010 for t in threads:
4011 t.join()
4012 clearResponses.append( t.result )
4013
4014
4015
4016
4017 clearResults = main.TRUE
4018 for i in range( main.numCtrls ):
4019 if clearResponses[ i ] == main.TRUE:
4020
4021 pass
4022 elif clearResponses[ i ] == main.FALSE:
4023
4024 pass
4025 elif clearResponses[ i ] == main.ERROR:
4026
4027 clearResults = main.FALSE
4028 else:
4029
4030 clearResults = main.FALSE
4031 if clearResults != main.TRUE:
4032 main.log.error( "Error executing set clear" )
4033
4034
4035 size = len( onosSet )
4036 getResponses = []
4037 threads = []
4038 for i in range( main.numCtrls ):
4039 t = main.Thread( target=main.CLIs[i].setTestGet,
4040 name="setTestGet-" + str( i ),
4041 args=[ onosSetName ] )
4042 threads.append( t )
4043 t.start()
4044 for t in threads:
4045 t.join()
4046 getResponses.append( t.result )
4047 getResults = main.TRUE
4048 for i in range( main.numCtrls ):
4049 if isinstance( getResponses[ i ], list):
4050 current = set( getResponses[ i ] )
4051 if len( current ) == len( getResponses[ i ] ):
4052
4053 if onosSet != current:
4054 main.log.error( "ONOS" + str( i + 1 ) +
4055 " has incorrect view" +
4056 " of set " + onosSetName + ":\n" +
4057 str( getResponses[ i ] ) )
4058 main.log.debug( "Expected: " + str( onosSet ) )
4059 main.log.debug( "Actual: " + str( current ) )
4060 getResults = main.FALSE
4061 else:
4062
4063 main.log.error( "ONOS" + str( i + 1 ) +
4064 " has repeat elements in" +
4065 " set " + onosSetName + ":\n" +
4066 str( getResponses[ i ] ) )
4067 getResults = main.FALSE
4068 elif getResponses[ i ] == main.ERROR:
4069 getResults = main.FALSE
4070 sizeResponses = []
4071 threads = []
4072 for i in range( main.numCtrls ):
4073 t = main.Thread( target=main.CLIs[i].setTestSize,
4074 name="setTestSize-" + str( i ),
4075 args=[ onosSetName ] )
4076 threads.append( t )
4077 t.start()
4078 for t in threads:
4079 t.join()
4080 sizeResponses.append( t.result )
4081 sizeResults = main.TRUE
4082 for i in range( main.numCtrls ):
4083 if size != sizeResponses[ i ]:
4084 sizeResults = main.FALSE
4085 main.log.error( "ONOS" + str( i + 1 ) +
4086 " expected a size of " + str( size ) +
4087 " for set " + onosSetName +
4088 " but got " + str( sizeResponses[ i ] ) )
4089 clearResults = clearResults and getResults and sizeResults
4090 utilities.assert_equals( expect=main.TRUE,
4091 actual=clearResults,
4092 onpass="Set clear correct",
4093 onfail="Set clear was incorrect" )
4094
4095 main.step( "Distributed Set addAll()" )
4096 onosSet.update( addAllValue.split() )
4097 addResponses = []
4098 threads = []
4099 for i in range( main.numCtrls ):
4100 t = main.Thread( target=main.CLIs[i].setTestAdd,
4101 name="setTestAddAll-" + str( i ),
4102 args=[ onosSetName, addAllValue ] )
4103 threads.append( t )
4104 t.start()
4105 for t in threads:
4106 t.join()
4107 addResponses.append( t.result )
4108
4109
4110
4111
4112 addAllResults = main.TRUE
4113 for i in range( main.numCtrls ):
4114 if addResponses[ i ] == main.TRUE:
4115
4116 pass
4117 elif addResponses[ i ] == main.FALSE:
4118
4119 pass
4120 elif addResponses[ i ] == main.ERROR:
4121
4122 addAllResults = main.FALSE
4123 else:
4124
4125 addAllResults = main.FALSE
4126 if addAllResults != main.TRUE:
4127 main.log.error( "Error executing set addAll" )
4128
4129
4130 size = len( onosSet )
4131 getResponses = []
4132 threads = []
4133 for i in range( main.numCtrls ):
4134 t = main.Thread( target=main.CLIs[i].setTestGet,
4135 name="setTestGet-" + str( i ),
4136 args=[ onosSetName ] )
4137 threads.append( t )
4138 t.start()
4139 for t in threads:
4140 t.join()
4141 getResponses.append( t.result )
4142 getResults = main.TRUE
4143 for i in range( main.numCtrls ):
4144 if isinstance( getResponses[ i ], list):
4145 current = set( getResponses[ i ] )
4146 if len( current ) == len( getResponses[ i ] ):
4147
4148 if onosSet != current:
4149 main.log.error( "ONOS" + str( i + 1 ) +
4150 " has incorrect view" +
4151 " of set " + onosSetName + ":\n" +
4152 str( getResponses[ i ] ) )
4153 main.log.debug( "Expected: " + str( onosSet ) )
4154 main.log.debug( "Actual: " + str( current ) )
4155 getResults = main.FALSE
4156 else:
4157
4158 main.log.error( "ONOS" + str( i + 1 ) +
4159 " has repeat elements in" +
4160 " set " + onosSetName + ":\n" +
4161 str( getResponses[ i ] ) )
4162 getResults = main.FALSE
4163 elif getResponses[ i ] == main.ERROR:
4164 getResults = main.FALSE
4165 sizeResponses = []
4166 threads = []
4167 for i in range( main.numCtrls ):
4168 t = main.Thread( target=main.CLIs[i].setTestSize,
4169 name="setTestSize-" + str( i ),
4170 args=[ onosSetName ] )
4171 threads.append( t )
4172 t.start()
4173 for t in threads:
4174 t.join()
4175 sizeResponses.append( t.result )
4176 sizeResults = main.TRUE
4177 for i in range( main.numCtrls ):
4178 if size != sizeResponses[ i ]:
4179 sizeResults = main.FALSE
4180 main.log.error( "ONOS" + str( i + 1 ) +
4181 " expected a size of " + str( size ) +
4182 " for set " + onosSetName +
4183 " but got " + str( sizeResponses[ i ] ) )
4184 addAllResults = addAllResults and getResults and sizeResults
4185 utilities.assert_equals( expect=main.TRUE,
4186 actual=addAllResults,
4187 onpass="Set addAll correct",
4188 onfail="Set addAll was incorrect" )
4189
4190 main.step( "Distributed Set retain()" )
4191 onosSet.intersection_update( retainValue.split() )
4192 retainResponses = []
4193 threads = []
4194 for i in range( main.numCtrls ):
4195 t = main.Thread( target=main.CLIs[i].setTestRemove,
4196 name="setTestRetain-" + str( i ),
4197 args=[ onosSetName, retainValue ],
4198 kwargs={ "retain": True } )
4199 threads.append( t )
4200 t.start()
4201 for t in threads:
4202 t.join()
4203 retainResponses.append( t.result )
4204
4205
4206
4207
4208 retainResults = main.TRUE
4209 for i in range( main.numCtrls ):
4210 if retainResponses[ i ] == main.TRUE:
4211
4212 pass
4213 elif retainResponses[ i ] == main.FALSE:
4214
4215 pass
4216 elif retainResponses[ i ] == main.ERROR:
4217
4218 retainResults = main.FALSE
4219 else:
4220
4221 retainResults = main.FALSE
4222 if retainResults != main.TRUE:
4223 main.log.error( "Error executing set retain" )
4224
4225
4226 size = len( onosSet )
4227 getResponses = []
4228 threads = []
4229 for i in range( main.numCtrls ):
4230 t = main.Thread( target=main.CLIs[i].setTestGet,
4231 name="setTestGet-" + str( i ),
4232 args=[ onosSetName ] )
4233 threads.append( t )
4234 t.start()
4235 for t in threads:
4236 t.join()
4237 getResponses.append( t.result )
4238 getResults = main.TRUE
4239 for i in range( main.numCtrls ):
4240 if isinstance( getResponses[ i ], list):
4241 current = set( getResponses[ i ] )
4242 if len( current ) == len( getResponses[ i ] ):
4243
4244 if onosSet != current:
4245 main.log.error( "ONOS" + str( i + 1 ) +
4246 " has incorrect view" +
4247 " of set " + onosSetName + ":\n" +
4248 str( getResponses[ i ] ) )
4249 main.log.debug( "Expected: " + str( onosSet ) )
4250 main.log.debug( "Actual: " + str( current ) )
4251 getResults = main.FALSE
4252 else:
4253
4254 main.log.error( "ONOS" + str( i + 1 ) +
4255 " has repeat elements in" +
4256 " set " + onosSetName + ":\n" +
4257 str( getResponses[ i ] ) )
4258 getResults = main.FALSE
4259 elif getResponses[ i ] == main.ERROR:
4260 getResults = main.FALSE
4261 sizeResponses = []
4262 threads = []
4263 for i in range( main.numCtrls ):
4264 t = main.Thread( target=main.CLIs[i].setTestSize,
4265 name="setTestSize-" + str( i ),
4266 args=[ onosSetName ] )
4267 threads.append( t )
4268 t.start()
4269 for t in threads:
4270 t.join()
4271 sizeResponses.append( t.result )
4272 sizeResults = main.TRUE
4273 for i in range( main.numCtrls ):
4274 if size != sizeResponses[ i ]:
4275 sizeResults = main.FALSE
4276 main.log.error( "ONOS" + str( i + 1 ) +
4277 " expected a size of " +
4278 str( size ) + " for set " + onosSetName +
4279 " but got " + str( sizeResponses[ i ] ) )
4280 retainResults = retainResults and getResults and sizeResults
4281 utilities.assert_equals( expect=main.TRUE,
4282 actual=retainResults,
4283 onpass="Set retain correct",
4284 onfail="Set retain was incorrect" )
4285