1 """
2 Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6 List of test cases:
7 CASE1: Compile ONOS and push it to the test machines
8 CASE2: Assign devices to controllers
9 CASE21: Assign mastership to controllers
10 CASE3: Assign intents
11 CASE4: Ping across added host intents
12 CASE5: Reading state of ONOS
13 CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14 CASE7: Check state after control plane failure
15 CASE8: Compare topo
16 CASE9: Link s3-s28 down
17 CASE10: Link s3-s28 up
18 CASE11: Switch down
19 CASE12: Switch up
20 CASE13: Clean up
21 CASE14: start election app on all onos nodes
22 CASE15: Check that Leadership Election is still functional
23 CASE16: Install Distributed Primitives app
24 CASE17: Check for basic functionality with distributed primitives
25 """
26
27
29
32
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
51 import imp
52 main.log.info( "ONOS HA Sanity test - initialization" )
53 main.case( "Setting up test environment" )
54 main.caseExplanation = "Setup the test environment including " +\
55 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57
58
59
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
66 main.numCtrls = int( main.params[ 'num_controllers' ] )
67 if main.ONOSbench.maxNodes:
68 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
70
71
72 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79
80
81
82 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
83 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
84 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
85 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
86 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
87 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
88 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
89
90 try:
91 fileName = "Counters"
92 path = main.params[ 'imports' ][ 'path' ]
93 main.Counters = imp.load_source( fileName,
94 path + fileName + ".py" )
95 except Exception as e:
96 main.log.exception( e )
97 main.cleanup()
98 main.exit()
99
100 main.CLIs = []
101 main.nodes = []
102 ipList = []
103 for i in range( 1, main.numCtrls + 1 ):
104 try:
105 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
106 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
107 ipList.append( main.nodes[ -1 ].ip_address )
108 except AttributeError:
109 break
110
111 main.step( "Create cell file" )
112 cellAppString = main.params[ 'ENV' ][ 'appString' ]
113 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
114 main.Mininet1.ip_address,
115 cellAppString, ipList )
116 main.step( "Applying cell variable to environment" )
117 cellResult = main.ONOSbench.setCell( cellName )
118 verifyResult = main.ONOSbench.verifyCell()
119
120
121 main.log.info( "Removing raft logs" )
122 main.ONOSbench.onosRemoveRaftLogs()
123
124 main.log.info( "Uninstalling ONOS" )
125 for node in main.nodes:
126 main.ONOSbench.onosUninstall( node.ip_address )
127
128
129 main.log.info( "Killing any ONOS processes" )
130 killResults = main.TRUE
131 for node in main.nodes:
132 killed = main.ONOSbench.onosKill( node.ip_address )
133 killResults = killResults and killed
134
135 cleanInstallResult = main.TRUE
136 gitPullResult = main.TRUE
137
138 main.step( "Starting Mininet" )
139
140
141 topoName = "obelisk.py"
142 filePath = main.ONOSbench.home + "/tools/test/topos/"
143 main.ONOSbench.copyMininetFile( topoName, filePath,
144 main.Mininet1.user_name,
145 main.Mininet1.ip_address )
146 mnResult = main.Mininet1.startNet( )
147 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
148 onpass="Mininet Started",
149 onfail="Error starting Mininet" )
150
151 main.step( "Git checkout and pull " + gitBranch )
152 if PULLCODE:
153 main.ONOSbench.gitCheckout( gitBranch )
154 gitPullResult = main.ONOSbench.gitPull()
155
156 utilities.assert_lesser( expect=0, actual=gitPullResult,
157 onpass="Git pull successful",
158 onfail="Git pull failed" )
159 main.ONOSbench.getVersion( report=True )
160
161 main.step( "Using mvn clean install" )
162 cleanInstallResult = main.TRUE
163 if PULLCODE and gitPullResult == main.TRUE:
164 cleanInstallResult = main.ONOSbench.cleanInstall()
165 else:
166 main.log.warn( "Did not pull new code so skipping mvn " +
167 "clean install" )
168 utilities.assert_equals( expect=main.TRUE,
169 actual=cleanInstallResult,
170 onpass="MCI successful",
171 onfail="MCI failed" )
172
173
174
175
176
177 job = "HAsanity"
178 plotName = "Plot-HA"
179 graphs = '<ac:structured-macro ac:name="html">\n'
180 graphs += '<ac:plain-text-body><![CDATA[\n'
181 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
182 '/plot/' + plotName + '/getPlot?index=0' +\
183 '&width=500&height=300"' +\
184 'noborder="0" width="500" height="300" scrolling="yes" ' +\
185 'seamless="seamless"></iframe>\n'
186 graphs += ']]></ac:plain-text-body>\n'
187 graphs += '</ac:structured-macro>\n'
188 main.log.wiki(graphs)
189
190 main.step( "Creating ONOS package" )
191 packageResult = main.ONOSbench.onosPackage()
192 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
193 onpass="ONOS package successful",
194 onfail="ONOS package failed" )
195
196 main.step( "Installing ONOS package" )
197 onosInstallResult = main.TRUE
198 for node in main.nodes:
199 tmpResult = main.ONOSbench.onosInstall( options="-f",
200 node=node.ip_address )
201 onosInstallResult = onosInstallResult and tmpResult
202 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
203 onpass="ONOS install successful",
204 onfail="ONOS install failed" )
205
206 main.step( "Checking if ONOS is up yet" )
207 for i in range( 2 ):
208 onosIsupResult = main.TRUE
209 for node in main.nodes:
210 started = main.ONOSbench.isup( node.ip_address )
211 if not started:
212 main.log.error( node.name + " didn't start!" )
213 main.ONOSbench.onosStop( node.ip_address )
214 main.ONOSbench.onosStart( node.ip_address )
215 onosIsupResult = onosIsupResult and started
216 if onosIsupResult == main.TRUE:
217 break
218 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
219 onpass="ONOS startup successful",
220 onfail="ONOS startup failed" )
221
222 main.log.step( "Starting ONOS CLI sessions" )
223 cliResults = main.TRUE
224 threads = []
225 for i in range( main.numCtrls ):
226 t = main.Thread( target=main.CLIs[i].startOnosCli,
227 name="startOnosCli-" + str( i ),
228 args=[main.nodes[i].ip_address] )
229 threads.append( t )
230 t.start()
231
232 for t in threads:
233 t.join()
234 cliResults = cliResults and t.result
235 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
236 onpass="ONOS cli startup successful",
237 onfail="ONOS cli startup failed" )
238
239 if main.params[ 'tcpdump' ].lower() == "true":
240 main.step( "Start Packet Capture MN" )
241 main.Mininet2.startTcpdump(
242 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
243 + "-MN.pcap",
244 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
245 port=main.params[ 'MNtcpdump' ][ 'port' ] )
246
247 main.step( "App Ids check" )
248 appCheck = main.TRUE
249 threads = []
250 for i in range( main.numCtrls ):
251 t = main.Thread( target=main.CLIs[i].appToIDCheck,
252 name="appToIDCheck-" + str( i ),
253 args=[] )
254 threads.append( t )
255 t.start()
256
257 for t in threads:
258 t.join()
259 appCheck = appCheck and t.result
260 if appCheck != main.TRUE:
261 main.log.warn( main.CLIs[0].apps() )
262 main.log.warn( main.CLIs[0].appIDs() )
263 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
264 onpass="App Ids seem to be correct",
265 onfail="Something is wrong with app Ids" )
266
267 if cliResults == main.FALSE:
268 main.log.error( "Failed to start ONOS, stopping test" )
269 main.cleanup()
270 main.exit()
271
272 - def CASE2( self, main ):
273 """
274 Assign devices to controllers
275 """
276 import re
277 assert main.numCtrls, "main.numCtrls not defined"
278 assert main, "main not defined"
279 assert utilities.assert_equals, "utilities.assert_equals not defined"
280 assert main.CLIs, "main.CLIs not defined"
281 assert main.nodes, "main.nodes not defined"
282 assert ONOS1Port, "ONOS1Port not defined"
283 assert ONOS2Port, "ONOS2Port not defined"
284 assert ONOS3Port, "ONOS3Port not defined"
285 assert ONOS4Port, "ONOS4Port not defined"
286 assert ONOS5Port, "ONOS5Port not defined"
287 assert ONOS6Port, "ONOS6Port not defined"
288 assert ONOS7Port, "ONOS7Port not defined"
289
290 main.case( "Assigning devices to controllers" )
291 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
292 "and check that an ONOS node becomes the " +\
293 "master of the device."
294 main.step( "Assign switches to controllers" )
295
296 ipList = []
297 for i in range( main.numCtrls ):
298 ipList.append( main.nodes[ i ].ip_address )
299 swList = []
300 for i in range( 1, 29 ):
301 swList.append( "s" + str( i ) )
302 main.Mininet1.assignSwController( sw=swList, ip=ipList )
303
304 mastershipCheck = main.TRUE
305 for i in range( 1, 29 ):
306 response = main.Mininet1.getSwController( "s" + str( i ) )
307 try:
308 main.log.info( str( response ) )
309 except Exception:
310 main.log.info( repr( response ) )
311 for node in main.nodes:
312 if re.search( "tcp:" + node.ip_address, response ):
313 mastershipCheck = mastershipCheck and main.TRUE
314 else:
315 main.log.error( "Error, node " + node.ip_address + " is " +
316 "not in the list of controllers s" +
317 str( i ) + " is connecting to." )
318 mastershipCheck = main.FALSE
319 utilities.assert_equals(
320 expect=main.TRUE,
321 actual=mastershipCheck,
322 onpass="Switch mastership assigned correctly",
323 onfail="Switches not assigned correctly to controllers" )
324
326 """
327 Assign mastership to controllers
328 """
329 import time
330 assert main.numCtrls, "main.numCtrls not defined"
331 assert main, "main not defined"
332 assert utilities.assert_equals, "utilities.assert_equals not defined"
333 assert main.CLIs, "main.CLIs not defined"
334 assert main.nodes, "main.nodes not defined"
335 assert ONOS1Port, "ONOS1Port not defined"
336 assert ONOS2Port, "ONOS2Port not defined"
337 assert ONOS3Port, "ONOS3Port not defined"
338 assert ONOS4Port, "ONOS4Port not defined"
339 assert ONOS5Port, "ONOS5Port not defined"
340 assert ONOS6Port, "ONOS6Port not defined"
341 assert ONOS7Port, "ONOS7Port not defined"
342
343 main.case( "Assigning Controller roles for switches" )
344 main.caseExplanation = "Check that ONOS is connected to each " +\
345 "device. Then manually assign" +\
346 " mastership to specific ONOS nodes using" +\
347 " 'device-role'"
348 main.step( "Assign mastership of switches to specific controllers" )
349
350 roleCall = main.TRUE
351
352 ipList = [ ]
353 deviceList = []
354 try:
355
356
357
358 for i in range( 1, 29 ):
359
360 if i == 1:
361 c = 0
362 ip = main.nodes[ c ].ip_address
363 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
364 elif i == 2:
365 c = 1 % main.numCtrls
366 ip = main.nodes[ c ].ip_address
367 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
368 elif i == 3:
369 c = 1 % main.numCtrls
370 ip = main.nodes[ c ].ip_address
371 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
372 elif i == 4:
373 c = 3 % main.numCtrls
374 ip = main.nodes[ c ].ip_address
375 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
376 elif i == 5:
377 c = 2 % main.numCtrls
378 ip = main.nodes[ c ].ip_address
379 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
380 elif i == 6:
381 c = 2 % main.numCtrls
382 ip = main.nodes[ c ].ip_address
383 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
384 elif i == 7:
385 c = 5 % main.numCtrls
386 ip = main.nodes[ c ].ip_address
387 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
388 elif i >= 8 and i <= 17:
389 c = 4 % main.numCtrls
390 ip = main.nodes[ c ].ip_address
391 dpid = '3' + str( i ).zfill( 3 )
392 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
393 elif i >= 18 and i <= 27:
394 c = 6 % main.numCtrls
395 ip = main.nodes[ c ].ip_address
396 dpid = '6' + str( i ).zfill( 3 )
397 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
398 elif i == 28:
399 c = 0
400 ip = main.nodes[ c ].ip_address
401 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
402 else:
403 main.log.error( "You didn't write an else statement for " +
404 "switch s" + str( i ) )
405 roleCall = main.FALSE
406
407 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
408
409 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
410 ip )
411 ipList.append( ip )
412 deviceList.append( deviceId )
413 except ( AttributeError, AssertionError ):
414 main.log.exception( "Something is wrong with ONOS device view" )
415 main.log.info( main.ONOScli1.devices() )
416 utilities.assert_equals(
417 expect=main.TRUE,
418 actual=roleCall,
419 onpass="Re-assigned switch mastership to designated controller",
420 onfail="Something wrong with deviceRole calls" )
421
422 main.step( "Check mastership was correctly assigned" )
423 roleCheck = main.TRUE
424
425
426 time.sleep( 5 )
427 for i in range( len( ipList ) ):
428 ip = ipList[i]
429 deviceId = deviceList[i]
430
431 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
432 if ip in master:
433 roleCheck = roleCheck and main.TRUE
434 else:
435 roleCheck = roleCheck and main.FALSE
436 main.log.error( "Error, controller " + ip + " is not" +
437 " master " + "of device " +
438 str( deviceId ) + ". Master is " +
439 repr( master ) + "." )
440 utilities.assert_equals(
441 expect=main.TRUE,
442 actual=roleCheck,
443 onpass="Switches were successfully reassigned to designated " +
444 "controller",
445 onfail="Switches were not successfully reassigned" )
446
447 - def CASE3( self, main ):
448 """
449 Assign intents
450 """
451 import time
452 import json
453 assert main.numCtrls, "main.numCtrls not defined"
454 assert main, "main not defined"
455 assert utilities.assert_equals, "utilities.assert_equals not defined"
456 assert main.CLIs, "main.CLIs not defined"
457 assert main.nodes, "main.nodes not defined"
458 main.case( "Adding host Intents" )
459 main.caseExplanation = "Discover hosts by using pingall then " +\
460 "assign predetermined host-to-host intents." +\
461 " After installation, check that the intent" +\
462 " is distributed to all nodes and the state" +\
463 " is INSTALLED"
464
465
466 main.step( "Install reactive forwarding app" )
467 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
468 utilities.assert_equals( expect=main.TRUE, actual=installResults,
469 onpass="Install fwd successful",
470 onfail="Install fwd failed" )
471
472 main.step( "Check app ids" )
473 appCheck = main.TRUE
474 threads = []
475 for i in range( main.numCtrls ):
476 t = main.Thread( target=main.CLIs[i].appToIDCheck,
477 name="appToIDCheck-" + str( i ),
478 args=[] )
479 threads.append( t )
480 t.start()
481
482 for t in threads:
483 t.join()
484 appCheck = appCheck and t.result
485 if appCheck != main.TRUE:
486 main.log.warn( main.CLIs[0].apps() )
487 main.log.warn( main.CLIs[0].appIDs() )
488 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
489 onpass="App Ids seem to be correct",
490 onfail="Something is wrong with app Ids" )
491
492 main.step( "Discovering Hosts( Via pingall for now )" )
493
494
495 pingResult = main.FALSE
496 for i in range(2):
497 time1 = time.time()
498 pingResult = main.Mininet1.pingall()
499 if i == 0:
500 utilities.assert_equals(
501 expect=main.TRUE,
502 actual=pingResult,
503 onpass="Reactive Pingall test passed",
504 onfail="Reactive Pingall failed, " +
505 "one or more ping pairs failed" )
506 time2 = time.time()
507 main.log.info( "Time for pingall: %2f seconds" %
508 ( time2 - time1 ) )
509
510 time.sleep( 11 )
511
512 main.step( "Uninstall reactive forwarding app" )
513 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
514 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
515 onpass="Uninstall fwd successful",
516 onfail="Uninstall fwd failed" )
517 '''
518 main.Mininet1.handle.sendline( "py [ h.cmd( \"arping -c 1 10.1.1.1 \" ) for h in net.hosts ] ")
519 import time
520 time.sleep(60)
521 '''
522
523 main.step( "Check app ids" )
524 threads = []
525 appCheck2 = main.TRUE
526 for i in range( main.numCtrls ):
527 t = main.Thread( target=main.CLIs[i].appToIDCheck,
528 name="appToIDCheck-" + str( i ),
529 args=[] )
530 threads.append( t )
531 t.start()
532
533 for t in threads:
534 t.join()
535 appCheck2 = appCheck2 and t.result
536 if appCheck2 != main.TRUE:
537 main.log.warn( main.CLIs[0].apps() )
538 main.log.warn( main.CLIs[0].appIDs() )
539 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
540 onpass="App Ids seem to be correct",
541 onfail="Something is wrong with app Ids" )
542
543 main.step( "Add host intents via cli" )
544 intentIds = []
545
546
547 intentAddResult = True
548 hostResult = main.TRUE
549 for i in range( 8, 18 ):
550 main.log.info( "Adding host intent between h" + str( i ) +
551 " and h" + str( i + 10 ) )
552 host1 = "00:00:00:00:00:" + \
553 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
554 host2 = "00:00:00:00:00:" + \
555 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
556
557 host1Dict = main.ONOScli1.getHost( host1 )
558 host2Dict = main.ONOScli1.getHost( host2 )
559 host1Id = None
560 host2Id = None
561 if host1Dict and host2Dict:
562 host1Id = host1Dict.get( 'id', None )
563 host2Id = host2Dict.get( 'id', None )
564 if host1Id and host2Id:
565 nodeNum = ( i % main.numCtrls )
566 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
567 if tmpId:
568 main.log.info( "Added intent with id: " + tmpId )
569 intentIds.append( tmpId )
570 else:
571 main.log.error( "addHostIntent returned: " +
572 repr( tmpId ) )
573 else:
574 main.log.error( "Error, getHost() failed for h" + str( i ) +
575 " and/or h" + str( i + 10 ) )
576 hosts = main.CLIs[ 0 ].hosts()
577 main.log.warn( "Hosts output: " )
578 try:
579 main.log.warn( json.dumps( json.loads( hosts ),
580 sort_keys=True,
581 indent=4,
582 separators=( ',', ': ' ) ) )
583 except ( ValueError, TypeError ):
584 main.log.warn( repr( hosts ) )
585 hostResult = main.FALSE
586 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
587 onpass="Found a host id for each host",
588 onfail="Error looking up host ids" )
589
590 intentStart = time.time()
591 onosIds = main.ONOScli1.getAllIntentsId()
592 main.log.info( "Submitted intents: " + str( intentIds ) )
593 main.log.info( "Intents in ONOS: " + str( onosIds ) )
594 for intent in intentIds:
595 if intent in onosIds:
596 pass
597 else:
598 intentAddResult = False
599 if intentAddResult:
600 intentStop = time.time()
601 else:
602 intentStop = None
603
604 intents = main.ONOScli1.intents()
605 intentStates = []
606 installedCheck = True
607 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
608 count = 0
609 try:
610 for intent in json.loads( intents ):
611 state = intent.get( 'state', None )
612 if "INSTALLED" not in state:
613 installedCheck = False
614 intentId = intent.get( 'id', None )
615 intentStates.append( ( intentId, state ) )
616 except ( ValueError, TypeError ):
617 main.log.exception( "Error parsing intents" )
618
619 tmplist = [ i for i, s in intentStates ]
620 missingIntents = False
621 for i in intentIds:
622 if i not in tmplist:
623 intentStates.append( ( i, " - " ) )
624 missingIntents = True
625 intentStates.sort()
626 for i, s in intentStates:
627 count += 1
628 main.log.info( "%-6s%-15s%-15s" %
629 ( str( count ), str( i ), str( s ) ) )
630 leaders = main.ONOScli1.leaders()
631 try:
632 missing = False
633 if leaders:
634 parsedLeaders = json.loads( leaders )
635 main.log.warn( json.dumps( parsedLeaders,
636 sort_keys=True,
637 indent=4,
638 separators=( ',', ': ' ) ) )
639
640 topics = []
641 for i in range( 14 ):
642 topics.append( "intent-partition-" + str( i ) )
643 main.log.debug( topics )
644 ONOStopics = [ j['topic'] for j in parsedLeaders ]
645 for topic in topics:
646 if topic not in ONOStopics:
647 main.log.error( "Error: " + topic +
648 " not in leaders" )
649 missing = True
650 else:
651 main.log.error( "leaders() returned None" )
652 except ( ValueError, TypeError ):
653 main.log.exception( "Error parsing leaders" )
654 main.log.error( repr( leaders ) )
655
656 if missing:
657 for node in main.CLIs:
658 response = node.leaders( jsonFormat=False)
659 main.log.warn( str( node.name ) + " leaders output: \n" +
660 str( response ) )
661
662 partitions = main.ONOScli1.partitions()
663 try:
664 if partitions :
665 parsedPartitions = json.loads( partitions )
666 main.log.warn( json.dumps( parsedPartitions,
667 sort_keys=True,
668 indent=4,
669 separators=( ',', ': ' ) ) )
670
671
672 else:
673 main.log.error( "partitions() returned None" )
674 except ( ValueError, TypeError ):
675 main.log.exception( "Error parsing partitions" )
676 main.log.error( repr( partitions ) )
677 pendingMap = main.ONOScli1.pendingMap()
678 try:
679 if pendingMap :
680 parsedPending = json.loads( pendingMap )
681 main.log.warn( json.dumps( parsedPending,
682 sort_keys=True,
683 indent=4,
684 separators=( ',', ': ' ) ) )
685
686 else:
687 main.log.error( "pendingMap() returned None" )
688 except ( ValueError, TypeError ):
689 main.log.exception( "Error parsing pending map" )
690 main.log.error( repr( pendingMap ) )
691
692 intentAddResult = bool( intentAddResult and not missingIntents and
693 installedCheck )
694 if not intentAddResult:
695 main.log.error( "Error in pushing host intents to ONOS" )
696
697 main.step( "Intent Anti-Entropy dispersion" )
698 for i in range(100):
699 correct = True
700 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
701 for cli in main.CLIs:
702 onosIds = []
703 ids = cli.getAllIntentsId()
704 onosIds.append( ids )
705 main.log.debug( "Intents in " + cli.name + ": " +
706 str( sorted( onosIds ) ) )
707 if sorted( ids ) != sorted( intentIds ):
708 main.log.warn( "Set of intent IDs doesn't match" )
709 correct = False
710 break
711 else:
712 intents = json.loads( cli.intents() )
713 for intent in intents:
714 if intent[ 'state' ] != "INSTALLED":
715 main.log.warn( "Intent " + intent[ 'id' ] +
716 " is " + intent[ 'state' ] )
717 correct = False
718 break
719 if correct:
720 break
721 else:
722 time.sleep(1)
723 if not intentStop:
724 intentStop = time.time()
725 global gossipTime
726 gossipTime = intentStop - intentStart
727 main.log.info( "It took about " + str( gossipTime ) +
728 " seconds for all intents to appear in each node" )
729
730
731 utilities.assert_greater_equals(
732 expect=40, actual=gossipTime,
733 onpass="ECM anti-entropy for intents worked within " +
734 "expected time",
735 onfail="Intent ECM anti-entropy took too long" )
736 if gossipTime <= 40:
737 intentAddResult = True
738
739 if not intentAddResult or "key" in pendingMap:
740 import time
741 installedCheck = True
742 main.log.info( "Sleeping 60 seconds to see if intents are found" )
743 time.sleep( 60 )
744 onosIds = main.ONOScli1.getAllIntentsId()
745 main.log.info( "Submitted intents: " + str( intentIds ) )
746 main.log.info( "Intents in ONOS: " + str( onosIds ) )
747
748 intents = main.ONOScli1.intents()
749 intentStates = []
750 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
751 count = 0
752 try:
753 for intent in json.loads( intents ):
754
755 state = intent.get( 'state', None )
756 if "INSTALLED" not in state:
757 installedCheck = False
758 intentId = intent.get( 'id', None )
759 intentStates.append( ( intentId, state ) )
760 except ( ValueError, TypeError ):
761 main.log.exception( "Error parsing intents" )
762
763 tmplist = [ i for i, s in intentStates ]
764 for i in intentIds:
765 if i not in tmplist:
766 intentStates.append( ( i, " - " ) )
767 intentStates.sort()
768 for i, s in intentStates:
769 count += 1
770 main.log.info( "%-6s%-15s%-15s" %
771 ( str( count ), str( i ), str( s ) ) )
772 leaders = main.ONOScli1.leaders()
773 try:
774 missing = False
775 if leaders:
776 parsedLeaders = json.loads( leaders )
777 main.log.warn( json.dumps( parsedLeaders,
778 sort_keys=True,
779 indent=4,
780 separators=( ',', ': ' ) ) )
781
782
783 topics = []
784 for i in range( 14 ):
785 topics.append( "intent-partition-" + str( i ) )
786
787 topics.append( "org.onosproject.election" )
788 main.log.debug( topics )
789 ONOStopics = [ j['topic'] for j in parsedLeaders ]
790 for topic in topics:
791 if topic not in ONOStopics:
792 main.log.error( "Error: " + topic +
793 " not in leaders" )
794 missing = True
795 else:
796 main.log.error( "leaders() returned None" )
797 except ( ValueError, TypeError ):
798 main.log.exception( "Error parsing leaders" )
799 main.log.error( repr( leaders ) )
800
801 if missing:
802 for node in main.CLIs:
803 response = node.leaders( jsonFormat=False)
804 main.log.warn( str( node.name ) + " leaders output: \n" +
805 str( response ) )
806
807 partitions = main.ONOScli1.partitions()
808 try:
809 if partitions :
810 parsedPartitions = json.loads( partitions )
811 main.log.warn( json.dumps( parsedPartitions,
812 sort_keys=True,
813 indent=4,
814 separators=( ',', ': ' ) ) )
815
816
817 else:
818 main.log.error( "partitions() returned None" )
819 except ( ValueError, TypeError ):
820 main.log.exception( "Error parsing partitions" )
821 main.log.error( repr( partitions ) )
822 pendingMap = main.ONOScli1.pendingMap()
823 try:
824 if pendingMap :
825 parsedPending = json.loads( pendingMap )
826 main.log.warn( json.dumps( parsedPending,
827 sort_keys=True,
828 indent=4,
829 separators=( ',', ': ' ) ) )
830
831 else:
832 main.log.error( "pendingMap() returned None" )
833 except ( ValueError, TypeError ):
834 main.log.exception( "Error parsing pending map" )
835 main.log.error( repr( pendingMap ) )
836
837 - def CASE4( self, main ):
838 """
839 Ping across added host intents
840 """
841 import json
842 import time
843 assert main.numCtrls, "main.numCtrls not defined"
844 assert main, "main not defined"
845 assert utilities.assert_equals, "utilities.assert_equals not defined"
846 assert main.CLIs, "main.CLIs not defined"
847 assert main.nodes, "main.nodes not defined"
848 main.case( "Verify connectivity by sendind traffic across Intents" )
849 main.caseExplanation = "Ping across added host intents to check " +\
850 "functionality and check the state of " +\
851 "the intent"
852 main.step( "Ping across added host intents" )
853 PingResult = main.TRUE
854 for i in range( 8, 18 ):
855 ping = main.Mininet1.pingHost( src="h" + str( i ),
856 target="h" + str( i + 10 ) )
857 PingResult = PingResult and ping
858 if ping == main.FALSE:
859 main.log.warn( "Ping failed between h" + str( i ) +
860 " and h" + str( i + 10 ) )
861 elif ping == main.TRUE:
862 main.log.info( "Ping test passed!" )
863
864 if PingResult == main.FALSE:
865 main.log.error(
866 "Intents have not been installed correctly, pings failed." )
867
868 main.log.warn( "ONOS1 intents: " )
869 try:
870 tmpIntents = main.ONOScli1.intents()
871 main.log.warn( json.dumps( json.loads( tmpIntents ),
872 sort_keys=True,
873 indent=4,
874 separators=( ',', ': ' ) ) )
875 except ( ValueError, TypeError ):
876 main.log.warn( repr( tmpIntents ) )
877 utilities.assert_equals(
878 expect=main.TRUE,
879 actual=PingResult,
880 onpass="Intents have been installed correctly and pings work",
881 onfail="Intents have not been installed correctly, pings failed." )
882
883 main.step( "Check Intent state" )
884 installedCheck = False
885 loopCount = 0
886 while not installedCheck and loopCount < 40:
887 installedCheck = True
888
889 intents = main.ONOScli1.intents()
890 intentStates = []
891 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
892 count = 0
893
894 try:
895 for intent in json.loads( intents ):
896 state = intent.get( 'state', None )
897 if "INSTALLED" not in state:
898 installedCheck = False
899 intentId = intent.get( 'id', None )
900 intentStates.append( ( intentId, state ) )
901 except ( ValueError, TypeError ):
902 main.log.exception( "Error parsing intents." )
903
904 intentStates.sort()
905 for i, s in intentStates:
906 count += 1
907 main.log.info( "%-6s%-15s%-15s" %
908 ( str( count ), str( i ), str( s ) ) )
909 if not installedCheck:
910 time.sleep( 1 )
911 loopCount += 1
912 utilities.assert_equals( expect=True, actual=installedCheck,
913 onpass="Intents are all INSTALLED",
914 onfail="Intents are not all in " +
915 "INSTALLED state" )
916
917 main.step( "Check leadership of topics" )
918 leaders = main.ONOScli1.leaders()
919 topicCheck = main.TRUE
920 try:
921 if leaders:
922 parsedLeaders = json.loads( leaders )
923 main.log.warn( json.dumps( parsedLeaders,
924 sort_keys=True,
925 indent=4,
926 separators=( ',', ': ' ) ) )
927
928
929
930 topics = []
931 for i in range( 14 ):
932 topics.append( "intent-partition-" + str( i ) )
933
934
935
936 main.log.debug( topics )
937 ONOStopics = [ j['topic'] for j in parsedLeaders ]
938 for topic in topics:
939 if topic not in ONOStopics:
940 main.log.error( "Error: " + topic +
941 " not in leaders" )
942 topicCheck = main.FALSE
943 else:
944 main.log.error( "leaders() returned None" )
945 topicCheck = main.FALSE
946 except ( ValueError, TypeError ):
947 topicCheck = main.FALSE
948 main.log.exception( "Error parsing leaders" )
949 main.log.error( repr( leaders ) )
950
951
952 if topicCheck:
953 for node in main.CLIs:
954 response = node.leaders( jsonFormat=False)
955 main.log.warn( str( node.name ) + " leaders output: \n" +
956 str( response ) )
957
958 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
959 onpass="intent Partitions is in leaders",
960 onfail="Some topics were lost " )
961
962 partitions = main.ONOScli1.partitions()
963 try:
964 if partitions :
965 parsedPartitions = json.loads( partitions )
966 main.log.warn( json.dumps( parsedPartitions,
967 sort_keys=True,
968 indent=4,
969 separators=( ',', ': ' ) ) )
970
971
972 else:
973 main.log.error( "partitions() returned None" )
974 except ( ValueError, TypeError ):
975 main.log.exception( "Error parsing partitions" )
976 main.log.error( repr( partitions ) )
977
978 pendingMap = main.ONOScli1.pendingMap()
979 try:
980 if pendingMap :
981 parsedPending = json.loads( pendingMap )
982 main.log.warn( json.dumps( parsedPending,
983 sort_keys=True,
984 indent=4,
985 separators=( ',', ': ' ) ) )
986
987 else:
988 main.log.error( "pendingMap() returned None" )
989 except ( ValueError, TypeError ):
990 main.log.exception( "Error parsing pending map" )
991 main.log.error( repr( pendingMap ) )
992
993 if not installedCheck:
994 main.log.info( "Waiting 60 seconds to see if the state of " +
995 "intents change" )
996 time.sleep( 60 )
997
998 intents = main.ONOScli1.intents()
999 intentStates = []
1000 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1001 count = 0
1002
1003 try:
1004 for intent in json.loads( intents ):
1005 state = intent.get( 'state', None )
1006 if "INSTALLED" not in state:
1007 installedCheck = False
1008 intentId = intent.get( 'id', None )
1009 intentStates.append( ( intentId, state ) )
1010 except ( ValueError, TypeError ):
1011 main.log.exception( "Error parsing intents." )
1012 intentStates.sort()
1013 for i, s in intentStates:
1014 count += 1
1015 main.log.info( "%-6s%-15s%-15s" %
1016 ( str( count ), str( i ), str( s ) ) )
1017 leaders = main.ONOScli1.leaders()
1018 try:
1019 missing = False
1020 if leaders:
1021 parsedLeaders = json.loads( leaders )
1022 main.log.warn( json.dumps( parsedLeaders,
1023 sort_keys=True,
1024 indent=4,
1025 separators=( ',', ': ' ) ) )
1026
1027
1028 topics = []
1029 for i in range( 14 ):
1030 topics.append( "intent-partition-" + str( i ) )
1031
1032 topics.append( "org.onosproject.election" )
1033 main.log.debug( topics )
1034 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1035 for topic in topics:
1036 if topic not in ONOStopics:
1037 main.log.error( "Error: " + topic +
1038 " not in leaders" )
1039 missing = True
1040 else:
1041 main.log.error( "leaders() returned None" )
1042 except ( ValueError, TypeError ):
1043 main.log.exception( "Error parsing leaders" )
1044 main.log.error( repr( leaders ) )
1045 if missing:
1046 for node in main.CLIs:
1047 response = node.leaders( jsonFormat=False)
1048 main.log.warn( str( node.name ) + " leaders output: \n" +
1049 str( response ) )
1050
1051 partitions = main.ONOScli1.partitions()
1052 try:
1053 if partitions :
1054 parsedPartitions = json.loads( partitions )
1055 main.log.warn( json.dumps( parsedPartitions,
1056 sort_keys=True,
1057 indent=4,
1058 separators=( ',', ': ' ) ) )
1059
1060
1061 else:
1062 main.log.error( "partitions() returned None" )
1063 except ( ValueError, TypeError ):
1064 main.log.exception( "Error parsing partitions" )
1065 main.log.error( repr( partitions ) )
1066 pendingMap = main.ONOScli1.pendingMap()
1067 try:
1068 if pendingMap :
1069 parsedPending = json.loads( pendingMap )
1070 main.log.warn( json.dumps( parsedPending,
1071 sort_keys=True,
1072 indent=4,
1073 separators=( ',', ': ' ) ) )
1074
1075 else:
1076 main.log.error( "pendingMap() returned None" )
1077 except ( ValueError, TypeError ):
1078 main.log.exception( "Error parsing pending map" )
1079 main.log.error( repr( pendingMap ) )
1080
1081 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
1082 main.step( "Wait a minute then ping again" )
1083
1084 PingResult = main.TRUE
1085 for i in range( 8, 18 ):
1086 ping = main.Mininet1.pingHost( src="h" + str( i ),
1087 target="h" + str( i + 10 ) )
1088 PingResult = PingResult and ping
1089 if ping == main.FALSE:
1090 main.log.warn( "Ping failed between h" + str( i ) +
1091 " and h" + str( i + 10 ) )
1092 elif ping == main.TRUE:
1093 main.log.info( "Ping test passed!" )
1094
1095 if PingResult == main.FALSE:
1096 main.log.error(
1097 "Intents have not been installed correctly, pings failed." )
1098
1099 main.log.warn( "ONOS1 intents: " )
1100 try:
1101 tmpIntents = main.ONOScli1.intents()
1102 main.log.warn( json.dumps( json.loads( tmpIntents ),
1103 sort_keys=True,
1104 indent=4,
1105 separators=( ',', ': ' ) ) )
1106 except ( ValueError, TypeError ):
1107 main.log.warn( repr( tmpIntents ) )
1108 utilities.assert_equals(
1109 expect=main.TRUE,
1110 actual=PingResult,
1111 onpass="Intents have been installed correctly and pings work",
1112 onfail="Intents have not been installed correctly, pings failed." )
1113
1114 - def CASE5( self, main ):
1115 """
1116 Reading state of ONOS
1117 """
1118 import json
1119 import time
1120 assert main.numCtrls, "main.numCtrls not defined"
1121 assert main, "main not defined"
1122 assert utilities.assert_equals, "utilities.assert_equals not defined"
1123 assert main.CLIs, "main.CLIs not defined"
1124 assert main.nodes, "main.nodes not defined"
1125
1126 main.case( "Setting up and gathering data for current state" )
1127
1128
1129
1130
1131 main.step( "Check that each switch has a master" )
1132 global mastershipState
1133 mastershipState = '[]'
1134
1135
1136 rolesNotNull = main.TRUE
1137 threads = []
1138 for i in range( main.numCtrls ):
1139 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1140 name="rolesNotNull-" + str( i ),
1141 args=[] )
1142 threads.append( t )
1143 t.start()
1144
1145 for t in threads:
1146 t.join()
1147 rolesNotNull = rolesNotNull and t.result
1148 utilities.assert_equals(
1149 expect=main.TRUE,
1150 actual=rolesNotNull,
1151 onpass="Each device has a master",
1152 onfail="Some devices don't have a master assigned" )
1153
1154 main.step( "Get the Mastership of each switch from each controller" )
1155 ONOSMastership = []
1156 mastershipCheck = main.FALSE
1157 consistentMastership = True
1158 rolesResults = True
1159 threads = []
1160 for i in range( main.numCtrls ):
1161 t = main.Thread( target=main.CLIs[i].roles,
1162 name="roles-" + str( i ),
1163 args=[] )
1164 threads.append( t )
1165 t.start()
1166
1167 for t in threads:
1168 t.join()
1169 ONOSMastership.append( t.result )
1170
1171 for i in range( main.numCtrls ):
1172 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1173 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1174 " roles" )
1175 main.log.warn(
1176 "ONOS" + str( i + 1 ) + " mastership response: " +
1177 repr( ONOSMastership[i] ) )
1178 rolesResults = False
1179 utilities.assert_equals(
1180 expect=True,
1181 actual=rolesResults,
1182 onpass="No error in reading roles output",
1183 onfail="Error in reading roles from ONOS" )
1184
1185 main.step( "Check for consistency in roles from each controller" )
1186 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1187 main.log.info(
1188 "Switch roles are consistent across all ONOS nodes" )
1189 else:
1190 consistentMastership = False
1191 utilities.assert_equals(
1192 expect=True,
1193 actual=consistentMastership,
1194 onpass="Switch roles are consistent across all ONOS nodes",
1195 onfail="ONOS nodes have different views of switch roles" )
1196
1197 if rolesResults and not consistentMastership:
1198 for i in range( main.numCtrls ):
1199 try:
1200 main.log.warn(
1201 "ONOS" + str( i + 1 ) + " roles: ",
1202 json.dumps(
1203 json.loads( ONOSMastership[ i ] ),
1204 sort_keys=True,
1205 indent=4,
1206 separators=( ',', ': ' ) ) )
1207 except ( ValueError, TypeError ):
1208 main.log.warn( repr( ONOSMastership[ i ] ) )
1209 elif rolesResults and consistentMastership:
1210 mastershipCheck = main.TRUE
1211 mastershipState = ONOSMastership[ 0 ]
1212
1213 main.step( "Get the intents from each controller" )
1214 global intentState
1215 intentState = []
1216 ONOSIntents = []
1217 intentCheck = main.FALSE
1218 consistentIntents = True
1219 intentsResults = True
1220 threads = []
1221 for i in range( main.numCtrls ):
1222 t = main.Thread( target=main.CLIs[i].intents,
1223 name="intents-" + str( i ),
1224 args=[],
1225 kwargs={ 'jsonFormat': True } )
1226 threads.append( t )
1227 t.start()
1228
1229 for t in threads:
1230 t.join()
1231 ONOSIntents.append( t.result )
1232
1233 for i in range( main.numCtrls ):
1234 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1235 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1236 " intents" )
1237 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1238 repr( ONOSIntents[ i ] ) )
1239 intentsResults = False
1240 utilities.assert_equals(
1241 expect=True,
1242 actual=intentsResults,
1243 onpass="No error in reading intents output",
1244 onfail="Error in reading intents from ONOS" )
1245
1246 main.step( "Check for consistency in Intents from each controller" )
1247 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1248 main.log.info( "Intents are consistent across all ONOS " +
1249 "nodes" )
1250 else:
1251 consistentIntents = False
1252 main.log.error( "Intents not consistent" )
1253 utilities.assert_equals(
1254 expect=True,
1255 actual=consistentIntents,
1256 onpass="Intents are consistent across all ONOS nodes",
1257 onfail="ONOS nodes have different views of intents" )
1258
1259 if intentsResults:
1260
1261
1262
1263
1264
1265
1266 title = " Id"
1267 for n in range( main.numCtrls ):
1268 title += " " * 10 + "ONOS" + str( n + 1 )
1269 main.log.warn( title )
1270
1271 keys = []
1272 try:
1273
1274 for nodeStr in ONOSIntents:
1275 node = json.loads( nodeStr )
1276 for intent in node:
1277 keys.append( intent.get( 'id' ) )
1278 keys = set( keys )
1279
1280 for key in keys:
1281 row = "%-13s" % key
1282 for nodeStr in ONOSIntents:
1283 node = json.loads( nodeStr )
1284 for intent in node:
1285 if intent.get( 'id', "Error" ) == key:
1286 row += "%-15s" % intent.get( 'state' )
1287 main.log.warn( row )
1288
1289 except ValueError as e:
1290 main.log.exception( e )
1291 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1292
1293 if intentsResults and not consistentIntents:
1294
1295 n = len(ONOSIntents)
1296 main.log.debug( "ONOS" + str( n ) + " intents: " )
1297 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1298 sort_keys=True,
1299 indent=4,
1300 separators=( ',', ': ' ) ) )
1301 for i in range( main.numCtrls ):
1302 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1303 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1304 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1305 sort_keys=True,
1306 indent=4,
1307 separators=( ',', ': ' ) ) )
1308 else:
1309 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
1310 str( n ) + " intents" )
1311 elif intentsResults and consistentIntents:
1312 intentCheck = main.TRUE
1313 intentState = ONOSIntents[ 0 ]
1314
1315 main.step( "Get the flows from each controller" )
1316 global flowState
1317 flowState = []
1318 ONOSFlows = []
1319 ONOSFlowsJson = []
1320 flowCheck = main.FALSE
1321 consistentFlows = True
1322 flowsResults = True
1323 threads = []
1324 for i in range( main.numCtrls ):
1325 t = main.Thread( target=main.CLIs[i].flows,
1326 name="flows-" + str( i ),
1327 args=[],
1328 kwargs={ 'jsonFormat': True } )
1329 threads.append( t )
1330 t.start()
1331
1332
1333 time.sleep(30)
1334 for t in threads:
1335 t.join()
1336 result = t.result
1337 ONOSFlows.append( result )
1338
1339 for i in range( main.numCtrls ):
1340 num = str( i + 1 )
1341 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1342 main.log.error( "Error in getting ONOS" + num + " flows" )
1343 main.log.warn( "ONOS" + num + " flows response: " +
1344 repr( ONOSFlows[ i ] ) )
1345 flowsResults = False
1346 ONOSFlowsJson.append( None )
1347 else:
1348 try:
1349 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1350 except ( ValueError, TypeError ):
1351
1352 main.log.exception( "Error in parsing ONOS" + num +
1353 " response as json." )
1354 main.log.error( repr( ONOSFlows[ i ] ) )
1355 ONOSFlowsJson.append( None )
1356 flowsResults = False
1357 utilities.assert_equals(
1358 expect=True,
1359 actual=flowsResults,
1360 onpass="No error in reading flows output",
1361 onfail="Error in reading flows from ONOS" )
1362
1363 main.step( "Check for consistency in Flows from each controller" )
1364 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1365 if all( tmp ):
1366 main.log.info( "Flow count is consistent across all ONOS nodes" )
1367 else:
1368 consistentFlows = False
1369 utilities.assert_equals(
1370 expect=True,
1371 actual=consistentFlows,
1372 onpass="The flow count is consistent across all ONOS nodes",
1373 onfail="ONOS nodes have different flow counts" )
1374
1375 if flowsResults and not consistentFlows:
1376 for i in range( main.numCtrls ):
1377 try:
1378 main.log.warn(
1379 "ONOS" + str( i + 1 ) + " flows: " +
1380 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1381 indent=4, separators=( ',', ': ' ) ) )
1382 except ( ValueError, TypeError ):
1383 main.log.warn(
1384 "ONOS" + str( i + 1 ) + " flows: " +
1385 repr( ONOSFlows[ i ] ) )
1386 elif flowsResults and consistentFlows:
1387 flowCheck = main.TRUE
1388 flowState = ONOSFlows[ 0 ]
1389
1390 main.step( "Get the OF Table entries" )
1391 global flows
1392 flows = []
1393 for i in range( 1, 29 ):
1394 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
1395 if flowCheck == main.FALSE:
1396 for table in flows:
1397 main.log.warn( table )
1398
1399
1400 main.step( "Start continuous pings" )
1401 main.Mininet2.pingLong(
1402 src=main.params[ 'PING' ][ 'source1' ],
1403 target=main.params[ 'PING' ][ 'target1' ],
1404 pingTime=500 )
1405 main.Mininet2.pingLong(
1406 src=main.params[ 'PING' ][ 'source2' ],
1407 target=main.params[ 'PING' ][ 'target2' ],
1408 pingTime=500 )
1409 main.Mininet2.pingLong(
1410 src=main.params[ 'PING' ][ 'source3' ],
1411 target=main.params[ 'PING' ][ 'target3' ],
1412 pingTime=500 )
1413 main.Mininet2.pingLong(
1414 src=main.params[ 'PING' ][ 'source4' ],
1415 target=main.params[ 'PING' ][ 'target4' ],
1416 pingTime=500 )
1417 main.Mininet2.pingLong(
1418 src=main.params[ 'PING' ][ 'source5' ],
1419 target=main.params[ 'PING' ][ 'target5' ],
1420 pingTime=500 )
1421 main.Mininet2.pingLong(
1422 src=main.params[ 'PING' ][ 'source6' ],
1423 target=main.params[ 'PING' ][ 'target6' ],
1424 pingTime=500 )
1425 main.Mininet2.pingLong(
1426 src=main.params[ 'PING' ][ 'source7' ],
1427 target=main.params[ 'PING' ][ 'target7' ],
1428 pingTime=500 )
1429 main.Mininet2.pingLong(
1430 src=main.params[ 'PING' ][ 'source8' ],
1431 target=main.params[ 'PING' ][ 'target8' ],
1432 pingTime=500 )
1433 main.Mininet2.pingLong(
1434 src=main.params[ 'PING' ][ 'source9' ],
1435 target=main.params[ 'PING' ][ 'target9' ],
1436 pingTime=500 )
1437 main.Mininet2.pingLong(
1438 src=main.params[ 'PING' ][ 'source10' ],
1439 target=main.params[ 'PING' ][ 'target10' ],
1440 pingTime=500 )
1441
1442 main.step( "Collecting topology information from ONOS" )
1443 devices = []
1444 threads = []
1445 for i in range( main.numCtrls ):
1446 t = main.Thread( target=main.CLIs[i].devices,
1447 name="devices-" + str( i ),
1448 args=[ ] )
1449 threads.append( t )
1450 t.start()
1451
1452 for t in threads:
1453 t.join()
1454 devices.append( t.result )
1455 hosts = []
1456 threads = []
1457 for i in range( main.numCtrls ):
1458 t = main.Thread( target=main.CLIs[i].hosts,
1459 name="hosts-" + str( i ),
1460 args=[ ] )
1461 threads.append( t )
1462 t.start()
1463
1464 for t in threads:
1465 t.join()
1466 try:
1467 hosts.append( json.loads( t.result ) )
1468 except ( ValueError, TypeError ):
1469
1470
1471 main.log.exception( "Error parsing json output of hosts" )
1472
1473 hosts.append( None )
1474
1475 ports = []
1476 threads = []
1477 for i in range( main.numCtrls ):
1478 t = main.Thread( target=main.CLIs[i].ports,
1479 name="ports-" + str( i ),
1480 args=[ ] )
1481 threads.append( t )
1482 t.start()
1483
1484 for t in threads:
1485 t.join()
1486 ports.append( t.result )
1487 links = []
1488 threads = []
1489 for i in range( main.numCtrls ):
1490 t = main.Thread( target=main.CLIs[i].links,
1491 name="links-" + str( i ),
1492 args=[ ] )
1493 threads.append( t )
1494 t.start()
1495
1496 for t in threads:
1497 t.join()
1498 links.append( t.result )
1499 clusters = []
1500 threads = []
1501 for i in range( main.numCtrls ):
1502 t = main.Thread( target=main.CLIs[i].clusters,
1503 name="clusters-" + str( i ),
1504 args=[ ] )
1505 threads.append( t )
1506 t.start()
1507
1508 for t in threads:
1509 t.join()
1510 clusters.append( t.result )
1511
1512
1513
1514 main.step( "Host view is consistent across ONOS nodes" )
1515 consistentHostsResult = main.TRUE
1516 for controller in range( len( hosts ) ):
1517 controllerStr = str( controller + 1 )
1518 if "Error" not in hosts[ controller ]:
1519 if hosts[ controller ] == hosts[ 0 ]:
1520 continue
1521 else:
1522 main.log.error( "hosts from ONOS" +
1523 controllerStr +
1524 " is inconsistent with ONOS1" )
1525 main.log.warn( repr( hosts[ controller ] ) )
1526 consistentHostsResult = main.FALSE
1527
1528 else:
1529 main.log.error( "Error in getting ONOS hosts from ONOS" +
1530 controllerStr )
1531 consistentHostsResult = main.FALSE
1532 main.log.warn( "ONOS" + controllerStr +
1533 " hosts response: " +
1534 repr( hosts[ controller ] ) )
1535 utilities.assert_equals(
1536 expect=main.TRUE,
1537 actual=consistentHostsResult,
1538 onpass="Hosts view is consistent across all ONOS nodes",
1539 onfail="ONOS nodes have different views of hosts" )
1540
1541 main.step( "Each host has an IP address" )
1542 ipResult = main.TRUE
1543 for controller in range( 0, len( hosts ) ):
1544 controllerStr = str( controller + 1 )
1545 for host in hosts[ controller ]:
1546 if not host.get( 'ipAddresses', [ ] ):
1547 main.log.error( "DEBUG:Error with host ips on controller" +
1548 controllerStr + ": " + str( host ) )
1549 ipResult = main.FALSE
1550 utilities.assert_equals(
1551 expect=main.TRUE,
1552 actual=ipResult,
1553 onpass="The ips of the hosts aren't empty",
1554 onfail="The ip of at least one host is missing" )
1555
1556
1557 main.step( "Cluster view is consistent across ONOS nodes" )
1558 consistentClustersResult = main.TRUE
1559 for controller in range( len( clusters ) ):
1560 controllerStr = str( controller + 1 )
1561 if "Error" not in clusters[ controller ]:
1562 if clusters[ controller ] == clusters[ 0 ]:
1563 continue
1564 else:
1565 main.log.error( "clusters from ONOS" + controllerStr +
1566 " is inconsistent with ONOS1" )
1567 consistentClustersResult = main.FALSE
1568
1569 else:
1570 main.log.error( "Error in getting dataplane clusters " +
1571 "from ONOS" + controllerStr )
1572 consistentClustersResult = main.FALSE
1573 main.log.warn( "ONOS" + controllerStr +
1574 " clusters response: " +
1575 repr( clusters[ controller ] ) )
1576 utilities.assert_equals(
1577 expect=main.TRUE,
1578 actual=consistentClustersResult,
1579 onpass="Clusters view is consistent across all ONOS nodes",
1580 onfail="ONOS nodes have different views of clusters" )
1581
1582 main.step( "Cluster view correct across ONOS nodes" )
1583 try:
1584 numClusters = len( json.loads( clusters[ 0 ] ) )
1585 except ( ValueError, TypeError ):
1586 main.log.exception( "Error parsing clusters[0]: " +
1587 repr( clusters[ 0 ] ) )
1588 clusterResults = main.FALSE
1589 if numClusters == 1:
1590 clusterResults = main.TRUE
1591 utilities.assert_equals(
1592 expect=1,
1593 actual=numClusters,
1594 onpass="ONOS shows 1 SCC",
1595 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1596
1597 main.step( "Comparing ONOS topology to MN" )
1598 devicesResults = main.TRUE
1599 linksResults = main.TRUE
1600 hostsResults = main.TRUE
1601 mnSwitches = main.Mininet1.getSwitches()
1602 mnLinks = main.Mininet1.getLinks()
1603 mnHosts = main.Mininet1.getHosts()
1604 for controller in range( main.numCtrls ):
1605 controllerStr = str( controller + 1 )
1606 if devices[ controller ] and ports[ controller ] and\
1607 "Error" not in devices[ controller ] and\
1608 "Error" not in ports[ controller ]:
1609
1610 currentDevicesResult = main.Mininet1.compareSwitches(
1611 mnSwitches,
1612 json.loads( devices[ controller ] ),
1613 json.loads( ports[ controller ] ) )
1614 else:
1615 currentDevicesResult = main.FALSE
1616 utilities.assert_equals( expect=main.TRUE,
1617 actual=currentDevicesResult,
1618 onpass="ONOS" + controllerStr +
1619 " Switches view is correct",
1620 onfail="ONOS" + controllerStr +
1621 " Switches view is incorrect" )
1622 if links[ controller ] and "Error" not in links[ controller ]:
1623 currentLinksResult = main.Mininet1.compareLinks(
1624 mnSwitches, mnLinks,
1625 json.loads( links[ controller ] ) )
1626 else:
1627 currentLinksResult = main.FALSE
1628 utilities.assert_equals( expect=main.TRUE,
1629 actual=currentLinksResult,
1630 onpass="ONOS" + controllerStr +
1631 " links view is correct",
1632 onfail="ONOS" + controllerStr +
1633 " links view is incorrect" )
1634
1635 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1636 currentHostsResult = main.Mininet1.compareHosts(
1637 mnHosts,
1638 hosts[ controller ] )
1639 else:
1640 currentHostsResult = main.FALSE
1641 utilities.assert_equals( expect=main.TRUE,
1642 actual=currentHostsResult,
1643 onpass="ONOS" + controllerStr +
1644 " hosts exist in Mininet",
1645 onfail="ONOS" + controllerStr +
1646 " hosts don't match Mininet" )
1647
1648 devicesResults = devicesResults and currentDevicesResult
1649 linksResults = linksResults and currentLinksResult
1650 hostsResults = hostsResults and currentHostsResult
1651
1652 main.step( "Device information is correct" )
1653 utilities.assert_equals(
1654 expect=main.TRUE,
1655 actual=devicesResults,
1656 onpass="Device information is correct",
1657 onfail="Device information is incorrect" )
1658
1659 main.step( "Links are correct" )
1660 utilities.assert_equals(
1661 expect=main.TRUE,
1662 actual=linksResults,
1663 onpass="Link are correct",
1664 onfail="Links are incorrect" )
1665
1666 main.step( "Hosts are correct" )
1667 utilities.assert_equals(
1668 expect=main.TRUE,
1669 actual=hostsResults,
1670 onpass="Hosts are correct",
1671 onfail="Hosts are incorrect" )
1672
1673 - def CASE6( self, main ):
1674 """
1675 The Failure case. Since this is the Sanity test, we do nothing.
1676 """
1677 import time
1678 assert main.numCtrls, "main.numCtrls not defined"
1679 assert main, "main not defined"
1680 assert utilities.assert_equals, "utilities.assert_equals not defined"
1681 assert main.CLIs, "main.CLIs not defined"
1682 assert main.nodes, "main.nodes not defined"
1683 main.case( "Wait 60 seconds instead of inducing a failure" )
1684 time.sleep( 60 )
1685 utilities.assert_equals(
1686 expect=main.TRUE,
1687 actual=main.TRUE,
1688 onpass="Sleeping 60 seconds",
1689 onfail="Something is terribly wrong with my math" )
1690
1691 - def CASE7( self, main ):
1692 """
1693 Check state after ONOS failure
1694 """
1695 import json
1696 assert main.numCtrls, "main.numCtrls not defined"
1697 assert main, "main not defined"
1698 assert utilities.assert_equals, "utilities.assert_equals not defined"
1699 assert main.CLIs, "main.CLIs not defined"
1700 assert main.nodes, "main.nodes not defined"
1701 main.case( "Running ONOS Constant State Tests" )
1702
1703 main.step( "Check that each switch has a master" )
1704
1705 rolesNotNull = main.TRUE
1706 threads = []
1707 for i in range( main.numCtrls ):
1708 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1709 name="rolesNotNull-" + str( i ),
1710 args=[ ] )
1711 threads.append( t )
1712 t.start()
1713
1714 for t in threads:
1715 t.join()
1716 rolesNotNull = rolesNotNull and t.result
1717 utilities.assert_equals(
1718 expect=main.TRUE,
1719 actual=rolesNotNull,
1720 onpass="Each device has a master",
1721 onfail="Some devices don't have a master assigned" )
1722
1723 main.step( "Read device roles from ONOS" )
1724 ONOSMastership = []
1725 mastershipCheck = main.FALSE
1726 consistentMastership = True
1727 rolesResults = True
1728 threads = []
1729 for i in range( main.numCtrls ):
1730 t = main.Thread( target=main.CLIs[i].roles,
1731 name="roles-" + str( i ),
1732 args=[] )
1733 threads.append( t )
1734 t.start()
1735
1736 for t in threads:
1737 t.join()
1738 ONOSMastership.append( t.result )
1739
1740 for i in range( main.numCtrls ):
1741 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1742 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1743 " roles" )
1744 main.log.warn(
1745 "ONOS" + str( i + 1 ) + " mastership response: " +
1746 repr( ONOSMastership[i] ) )
1747 rolesResults = False
1748 utilities.assert_equals(
1749 expect=True,
1750 actual=rolesResults,
1751 onpass="No error in reading roles output",
1752 onfail="Error in reading roles from ONOS" )
1753
1754 main.step( "Check for consistency in roles from each controller" )
1755 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1756 main.log.info(
1757 "Switch roles are consistent across all ONOS nodes" )
1758 else:
1759 consistentMastership = False
1760 utilities.assert_equals(
1761 expect=True,
1762 actual=consistentMastership,
1763 onpass="Switch roles are consistent across all ONOS nodes",
1764 onfail="ONOS nodes have different views of switch roles" )
1765
1766 if rolesResults and not consistentMastership:
1767 for i in range( main.numCtrls ):
1768 main.log.warn(
1769 "ONOS" + str( i + 1 ) + " roles: ",
1770 json.dumps(
1771 json.loads( ONOSMastership[ i ] ),
1772 sort_keys=True,
1773 indent=4,
1774 separators=( ',', ': ' ) ) )
1775 elif rolesResults and not consistentMastership:
1776 mastershipCheck = main.TRUE
1777
1778 description2 = "Compare switch roles from before failure"
1779 main.step( description2 )
1780 try:
1781 currentJson = json.loads( ONOSMastership[0] )
1782 oldJson = json.loads( mastershipState )
1783 except ( ValueError, TypeError ):
1784 main.log.exception( "Something is wrong with parsing " +
1785 "ONOSMastership[0] or mastershipState" )
1786 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1787 main.log.error( "mastershipState" + repr( mastershipState ) )
1788 main.cleanup()
1789 main.exit()
1790 mastershipCheck = main.TRUE
1791 for i in range( 1, 29 ):
1792 switchDPID = str(
1793 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1794 current = [ switch[ 'master' ] for switch in currentJson
1795 if switchDPID in switch[ 'id' ] ]
1796 old = [ switch[ 'master' ] for switch in oldJson
1797 if switchDPID in switch[ 'id' ] ]
1798 if current == old:
1799 mastershipCheck = mastershipCheck and main.TRUE
1800 else:
1801 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1802 mastershipCheck = main.FALSE
1803 utilities.assert_equals(
1804 expect=main.TRUE,
1805 actual=mastershipCheck,
1806 onpass="Mastership of Switches was not changed",
1807 onfail="Mastership of some switches changed" )
1808 mastershipCheck = mastershipCheck and consistentMastership
1809
1810 main.step( "Get the intents and compare across all nodes" )
1811 ONOSIntents = []
1812 intentCheck = main.FALSE
1813 consistentIntents = True
1814 intentsResults = True
1815 threads = []
1816 for i in range( main.numCtrls ):
1817 t = main.Thread( target=main.CLIs[i].intents,
1818 name="intents-" + str( i ),
1819 args=[],
1820 kwargs={ 'jsonFormat': True } )
1821 threads.append( t )
1822 t.start()
1823
1824 for t in threads:
1825 t.join()
1826 ONOSIntents.append( t.result )
1827
1828 for i in range( main.numCtrls ):
1829 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1830 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1831 " intents" )
1832 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1833 repr( ONOSIntents[ i ] ) )
1834 intentsResults = False
1835 utilities.assert_equals(
1836 expect=True,
1837 actual=intentsResults,
1838 onpass="No error in reading intents output",
1839 onfail="Error in reading intents from ONOS" )
1840
1841 main.step( "Check for consistency in Intents from each controller" )
1842 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1843 main.log.info( "Intents are consistent across all ONOS " +
1844 "nodes" )
1845 else:
1846 consistentIntents = False
1847
1848
1849
1850
1851
1852
1853
1854 title = " ID"
1855 for n in range( main.numCtrls ):
1856 title += " " * 10 + "ONOS" + str( n + 1 )
1857 main.log.warn( title )
1858
1859 keys = []
1860 for nodeStr in ONOSIntents:
1861 node = json.loads( nodeStr )
1862 for intent in node:
1863 keys.append( intent.get( 'id' ) )
1864 keys = set( keys )
1865 for key in keys:
1866 row = "%-13s" % key
1867 for nodeStr in ONOSIntents:
1868 node = json.loads( nodeStr )
1869 for intent in node:
1870 if intent.get( 'id' ) == key:
1871 row += "%-15s" % intent.get( 'state' )
1872 main.log.warn( row )
1873
1874
1875 utilities.assert_equals(
1876 expect=True,
1877 actual=consistentIntents,
1878 onpass="Intents are consistent across all ONOS nodes",
1879 onfail="ONOS nodes have different views of intents" )
1880 intentStates = []
1881 for node in ONOSIntents:
1882 nodeStates = []
1883
1884 try:
1885 for intent in json.loads( node ):
1886 nodeStates.append( intent[ 'state' ] )
1887 except ( ValueError, TypeError ):
1888 main.log.exception( "Error in parsing intents" )
1889 main.log.error( repr( node ) )
1890 intentStates.append( nodeStates )
1891 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1892 main.log.info( dict( out ) )
1893
1894 if intentsResults and not consistentIntents:
1895 for i in range( main.numCtrls ):
1896 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1897 main.log.warn( json.dumps(
1898 json.loads( ONOSIntents[ i ] ),
1899 sort_keys=True,
1900 indent=4,
1901 separators=( ',', ': ' ) ) )
1902 elif intentsResults and consistentIntents:
1903 intentCheck = main.TRUE
1904
1905
1906
1907 main.step( "Compare current intents with intents before the failure" )
1908
1909
1910 sameIntents = main.FALSE
1911 if intentState and intentState == ONOSIntents[ 0 ]:
1912 sameIntents = main.TRUE
1913 main.log.info( "Intents are consistent with before failure" )
1914
1915
1916 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1917 sameIntents = main.TRUE
1918 try:
1919 before = json.loads( intentState )
1920 after = json.loads( ONOSIntents[ 0 ] )
1921 for intent in before:
1922 if intent not in after:
1923 sameIntents = main.FALSE
1924 main.log.debug( "Intent is not currently in ONOS " +
1925 "(at least in the same form):" )
1926 main.log.debug( json.dumps( intent ) )
1927 except ( ValueError, TypeError ):
1928 main.log.exception( "Exception printing intents" )
1929 main.log.debug( repr( ONOSIntents[0] ) )
1930 main.log.debug( repr( intentState ) )
1931 if sameIntents == main.FALSE:
1932 try:
1933 main.log.debug( "ONOS intents before: " )
1934 main.log.debug( json.dumps( json.loads( intentState ),
1935 sort_keys=True, indent=4,
1936 separators=( ',', ': ' ) ) )
1937 main.log.debug( "Current ONOS intents: " )
1938 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1939 sort_keys=True, indent=4,
1940 separators=( ',', ': ' ) ) )
1941 except ( ValueError, TypeError ):
1942 main.log.exception( "Exception printing intents" )
1943 main.log.debug( repr( ONOSIntents[0] ) )
1944 main.log.debug( repr( intentState ) )
1945 utilities.assert_equals(
1946 expect=main.TRUE,
1947 actual=sameIntents,
1948 onpass="Intents are consistent with before failure",
1949 onfail="The Intents changed during failure" )
1950 intentCheck = intentCheck and sameIntents
1951
1952 main.step( "Get the OF Table entries and compare to before " +
1953 "component failure" )
1954 FlowTables = main.TRUE
1955 flows2 = []
1956 for i in range( 28 ):
1957 main.log.info( "Checking flow table on s" + str( i + 1 ) )
1958 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
1959 flows2.append( tmpFlows )
1960 tempResult = main.Mininet1.flowComp(
1961 flow1=flows[ i ],
1962 flow2=tmpFlows )
1963 FlowTables = FlowTables and tempResult
1964 if FlowTables == main.FALSE:
1965 main.log.info( "Differences in flow table for switch: s" +
1966 str( i + 1 ) )
1967 utilities.assert_equals(
1968 expect=main.TRUE,
1969 actual=FlowTables,
1970 onpass="No changes were found in the flow tables",
1971 onfail="Changes were found in the flow tables" )
1972
1973 main.Mininet2.pingLongKill()
1974 '''
1975 main.step( "Check the continuous pings to ensure that no packets " +
1976 "were dropped during component failure" )
1977 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1978 main.params[ 'TESTONIP' ] )
1979 LossInPings = main.FALSE
1980 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1981 for i in range( 8, 18 ):
1982 main.log.info(
1983 "Checking for a loss in pings along flow from s" +
1984 str( i ) )
1985 LossInPings = main.Mininet2.checkForLoss(
1986 "/tmp/ping.h" +
1987 str( i ) ) or LossInPings
1988 if LossInPings == main.TRUE:
1989 main.log.info( "Loss in ping detected" )
1990 elif LossInPings == main.ERROR:
1991 main.log.info( "There are multiple mininet process running" )
1992 elif LossInPings == main.FALSE:
1993 main.log.info( "No Loss in the pings" )
1994 main.log.info( "No loss of dataplane connectivity" )
1995 utilities.assert_equals(
1996 expect=main.FALSE,
1997 actual=LossInPings,
1998 onpass="No Loss of connectivity",
1999 onfail="Loss of dataplane connectivity detected" )
2000 '''
2001
2002 main.step( "Leadership Election is still functional" )
2003
2004
2005
2006 leader = main.nodes[ 0 ].ip_address
2007 leaderResult = main.TRUE
2008 for cli in main.CLIs:
2009 leaderN = cli.electionTestLeader()
2010
2011 if leaderN == leader:
2012
2013
2014
2015 pass
2016 elif leaderN == main.FALSE:
2017
2018 main.log.error( "Something is wrong with " +
2019 "electionTestLeader function, check the" +
2020 " error logs" )
2021 leaderResult = main.FALSE
2022 elif leader != leaderN:
2023 leaderResult = main.FALSE
2024 main.log.error( cli.name + " sees " + str( leaderN ) +
2025 " as the leader of the election app. " +
2026 "Leader should be " + str( leader ) )
2027 utilities.assert_equals(
2028 expect=main.TRUE,
2029 actual=leaderResult,
2030 onpass="Leadership election passed",
2031 onfail="Something went wrong with Leadership election" )
2032
2033 - def CASE8( self, main ):
2034 """
2035 Compare topo
2036 """
2037 import json
2038 import time
2039 assert main.numCtrls, "main.numCtrls not defined"
2040 assert main, "main not defined"
2041 assert utilities.assert_equals, "utilities.assert_equals not defined"
2042 assert main.CLIs, "main.CLIs not defined"
2043 assert main.nodes, "main.nodes not defined"
2044
2045 main.case( "Compare ONOS Topology view to Mininet topology" )
2046 main.caseExplanation = "Compare topology objects between Mininet" +\
2047 " and ONOS"
2048
2049 main.step( "Comparing ONOS topology to MN" )
2050 devicesResults = main.TRUE
2051 linksResults = main.TRUE
2052 hostsResults = main.TRUE
2053 hostAttachmentResults = True
2054 topoResult = main.FALSE
2055 elapsed = 0
2056 count = 0
2057 main.step( "Collecting topology information from ONOS" )
2058 startTime = time.time()
2059
2060 while topoResult == main.FALSE and elapsed < 60:
2061 count += 1
2062 cliStart = time.time()
2063 devices = []
2064 threads = []
2065 for i in range( main.numCtrls ):
2066 t = main.Thread( target=main.CLIs[i].devices,
2067 name="devices-" + str( i ),
2068 args=[ ] )
2069 threads.append( t )
2070 t.start()
2071
2072 for t in threads:
2073 t.join()
2074 devices.append( t.result )
2075 hosts = []
2076 ipResult = main.TRUE
2077 threads = []
2078 for i in range( main.numCtrls ):
2079 t = main.Thread( target=main.CLIs[i].hosts,
2080 name="hosts-" + str( i ),
2081 args=[ ] )
2082 threads.append( t )
2083 t.start()
2084
2085 for t in threads:
2086 t.join()
2087 try:
2088 hosts.append( json.loads( t.result ) )
2089 except ( ValueError, TypeError ):
2090 main.log.exception( "Error parsing hosts results" )
2091 main.log.error( repr( t.result ) )
2092 for controller in range( 0, len( hosts ) ):
2093 controllerStr = str( controller + 1 )
2094 for host in hosts[ controller ]:
2095 if host is None or host.get( 'ipAddresses', [] ) == []:
2096 main.log.error(
2097 "DEBUG:Error with host ipAddresses on controller" +
2098 controllerStr + ": " + str( host ) )
2099 ipResult = main.FALSE
2100 ports = []
2101 threads = []
2102 for i in range( main.numCtrls ):
2103 t = main.Thread( target=main.CLIs[i].ports,
2104 name="ports-" + str( i ),
2105 args=[ ] )
2106 threads.append( t )
2107 t.start()
2108
2109 for t in threads:
2110 t.join()
2111 ports.append( t.result )
2112 links = []
2113 threads = []
2114 for i in range( main.numCtrls ):
2115 t = main.Thread( target=main.CLIs[i].links,
2116 name="links-" + str( i ),
2117 args=[ ] )
2118 threads.append( t )
2119 t.start()
2120
2121 for t in threads:
2122 t.join()
2123 links.append( t.result )
2124 clusters = []
2125 threads = []
2126 for i in range( main.numCtrls ):
2127 t = main.Thread( target=main.CLIs[i].clusters,
2128 name="clusters-" + str( i ),
2129 args=[ ] )
2130 threads.append( t )
2131 t.start()
2132
2133 for t in threads:
2134 t.join()
2135 clusters.append( t.result )
2136
2137 elapsed = time.time() - startTime
2138 cliTime = time.time() - cliStart
2139 print "Elapsed time: " + str( elapsed )
2140 print "CLI time: " + str( cliTime )
2141
2142 mnSwitches = main.Mininet1.getSwitches()
2143 mnLinks = main.Mininet1.getLinks()
2144 mnHosts = main.Mininet1.getHosts()
2145 for controller in range( main.numCtrls ):
2146 controllerStr = str( controller + 1 )
2147 if devices[ controller ] and ports[ controller ] and\
2148 "Error" not in devices[ controller ] and\
2149 "Error" not in ports[ controller ]:
2150
2151 currentDevicesResult = main.Mininet1.compareSwitches(
2152 mnSwitches,
2153 json.loads( devices[ controller ] ),
2154 json.loads( ports[ controller ] ) )
2155 else:
2156 currentDevicesResult = main.FALSE
2157 utilities.assert_equals( expect=main.TRUE,
2158 actual=currentDevicesResult,
2159 onpass="ONOS" + controllerStr +
2160 " Switches view is correct",
2161 onfail="ONOS" + controllerStr +
2162 " Switches view is incorrect" )
2163
2164 if links[ controller ] and "Error" not in links[ controller ]:
2165 currentLinksResult = main.Mininet1.compareLinks(
2166 mnSwitches, mnLinks,
2167 json.loads( links[ controller ] ) )
2168 else:
2169 currentLinksResult = main.FALSE
2170 utilities.assert_equals( expect=main.TRUE,
2171 actual=currentLinksResult,
2172 onpass="ONOS" + controllerStr +
2173 " links view is correct",
2174 onfail="ONOS" + controllerStr +
2175 " links view is incorrect" )
2176
2177 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2178 currentHostsResult = main.Mininet1.compareHosts(
2179 mnHosts,
2180 hosts[ controller ] )
2181 else:
2182 currentHostsResult = main.FALSE
2183 utilities.assert_equals( expect=main.TRUE,
2184 actual=currentHostsResult,
2185 onpass="ONOS" + controllerStr +
2186 " hosts exist in Mininet",
2187 onfail="ONOS" + controllerStr +
2188 " hosts don't match Mininet" )
2189
2190 hostAttachment = True
2191 zeroHosts = False
2192
2193
2194 mappings = {}
2195 for i in range( 1, 29 ):
2196
2197 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2198 if i == 1:
2199 deviceId = "1000".zfill(16)
2200 elif i == 2:
2201 deviceId = "2000".zfill(16)
2202 elif i == 3:
2203 deviceId = "3000".zfill(16)
2204 elif i == 4:
2205 deviceId = "3004".zfill(16)
2206 elif i == 5:
2207 deviceId = "5000".zfill(16)
2208 elif i == 6:
2209 deviceId = "6000".zfill(16)
2210 elif i == 7:
2211 deviceId = "6007".zfill(16)
2212 elif i >= 8 and i <= 17:
2213 dpid = '3' + str( i ).zfill( 3 )
2214 deviceId = dpid.zfill(16)
2215 elif i >= 18 and i <= 27:
2216 dpid = '6' + str( i ).zfill( 3 )
2217 deviceId = dpid.zfill(16)
2218 elif i == 28:
2219 deviceId = "2800".zfill(16)
2220 mappings[ macId ] = deviceId
2221 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2222 if hosts[ controller ] == []:
2223 main.log.warn( "There are no hosts discovered" )
2224 zeroHosts = True
2225 else:
2226 for host in hosts[ controller ]:
2227 mac = None
2228 location = None
2229 device = None
2230 port = None
2231 try:
2232 mac = host.get( 'mac' )
2233 assert mac, "mac field could not be found for this host object"
2234
2235 location = host.get( 'location' )
2236 assert location, "location field could not be found for this host object"
2237
2238
2239 device = str( location.get( 'elementId' ) ).split(':')[1]
2240 assert device, "elementId field could not be found for this host location object"
2241
2242 port = location.get( 'port' )
2243 assert port, "port field could not be found for this host location object"
2244
2245
2246 if mac and device and port:
2247 if str( port ) != "1":
2248 main.log.error( "The attachment port is incorrect for " +
2249 "host " + str( mac ) +
2250 ". Expected: 1 Actual: " + str( port) )
2251 hostAttachment = False
2252 if device != mappings[ str( mac ) ]:
2253 main.log.error( "The attachment device is incorrect for " +
2254 "host " + str( mac ) +
2255 ". Expected: " + mappings[ str( mac ) ] +
2256 " Actual: " + device )
2257 hostAttachment = False
2258 else:
2259 hostAttachment = False
2260 except AssertionError:
2261 main.log.exception( "Json object not as expected" )
2262 main.log.error( repr( host ) )
2263 hostAttachment = False
2264 else:
2265 main.log.error( "No hosts json output or \"Error\"" +
2266 " in output. hosts = " +
2267 repr( hosts[ controller ] ) )
2268 if zeroHosts is False:
2269 hostAttachment = True
2270
2271
2272 devicesResults = devicesResults and currentDevicesResult
2273 linksResults = linksResults and currentLinksResult
2274 hostsResults = hostsResults and currentHostsResult
2275 hostAttachmentResults = hostAttachmentResults and\
2276 hostAttachment
2277 topoResult = ( devicesResults and linksResults
2278 and hostsResults and ipResult and
2279 hostAttachmentResults )
2280
2281
2282
2283
2284 main.step( "Hosts view is consistent across all ONOS nodes" )
2285 consistentHostsResult = main.TRUE
2286 for controller in range( len( hosts ) ):
2287 controllerStr = str( controller + 1 )
2288 if "Error" not in hosts[ controller ]:
2289 if hosts[ controller ] == hosts[ 0 ]:
2290 continue
2291 else:
2292 main.log.error( "hosts from ONOS" + controllerStr +
2293 " is inconsistent with ONOS1" )
2294 main.log.warn( repr( hosts[ controller ] ) )
2295 consistentHostsResult = main.FALSE
2296
2297 else:
2298 main.log.error( "Error in getting ONOS hosts from ONOS" +
2299 controllerStr )
2300 consistentHostsResult = main.FALSE
2301 main.log.warn( "ONOS" + controllerStr +
2302 " hosts response: " +
2303 repr( hosts[ controller ] ) )
2304 utilities.assert_equals(
2305 expect=main.TRUE,
2306 actual=consistentHostsResult,
2307 onpass="Hosts view is consistent across all ONOS nodes",
2308 onfail="ONOS nodes have different views of hosts" )
2309
2310 main.step( "Hosts information is correct" )
2311 hostsResults = hostsResults and ipResult
2312 utilities.assert_equals(
2313 expect=main.TRUE,
2314 actual=hostsResults,
2315 onpass="Host information is correct",
2316 onfail="Host information is incorrect" )
2317
2318 main.step( "Host attachment points to the network" )
2319 utilities.assert_equals(
2320 expect=True,
2321 actual=hostAttachmentResults,
2322 onpass="Hosts are correctly attached to the network",
2323 onfail="ONOS did not correctly attach hosts to the network" )
2324
2325
2326 main.step( "Clusters view is consistent across all ONOS nodes" )
2327 consistentClustersResult = main.TRUE
2328 for controller in range( len( clusters ) ):
2329 controllerStr = str( controller + 1 )
2330 if "Error" not in clusters[ controller ]:
2331 if clusters[ controller ] == clusters[ 0 ]:
2332 continue
2333 else:
2334 main.log.error( "clusters from ONOS" +
2335 controllerStr +
2336 " is inconsistent with ONOS1" )
2337 consistentClustersResult = main.FALSE
2338
2339 else:
2340 main.log.error( "Error in getting dataplane clusters " +
2341 "from ONOS" + controllerStr )
2342 consistentClustersResult = main.FALSE
2343 main.log.warn( "ONOS" + controllerStr +
2344 " clusters response: " +
2345 repr( clusters[ controller ] ) )
2346 utilities.assert_equals(
2347 expect=main.TRUE,
2348 actual=consistentClustersResult,
2349 onpass="Clusters view is consistent across all ONOS nodes",
2350 onfail="ONOS nodes have different views of clusters" )
2351
2352 main.step( "There is only one SCC" )
2353
2354 try:
2355 numClusters = len( json.loads( clusters[ 0 ] ) )
2356 except ( ValueError, TypeError ):
2357 main.log.exception( "Error parsing clusters[0]: " +
2358 repr( clusters[0] ) )
2359 clusterResults = main.FALSE
2360 if numClusters == 1:
2361 clusterResults = main.TRUE
2362 utilities.assert_equals(
2363 expect=1,
2364 actual=numClusters,
2365 onpass="ONOS shows 1 SCC",
2366 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2367
2368 topoResult = ( devicesResults and linksResults
2369 and hostsResults and consistentHostsResult
2370 and consistentClustersResult and clusterResults
2371 and ipResult and hostAttachmentResults )
2372
2373 topoResult = topoResult and int( count <= 2 )
2374 note = "note it takes about " + str( int( cliTime ) ) + \
2375 " seconds for the test to make all the cli calls to fetch " +\
2376 "the topology from each ONOS instance"
2377 main.log.info(
2378 "Very crass estimate for topology discovery/convergence( " +
2379 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2380 str( count ) + " tries" )
2381
2382 main.step( "Device information is correct" )
2383 utilities.assert_equals(
2384 expect=main.TRUE,
2385 actual=devicesResults,
2386 onpass="Device information is correct",
2387 onfail="Device information is incorrect" )
2388
2389 main.step( "Links are correct" )
2390 utilities.assert_equals(
2391 expect=main.TRUE,
2392 actual=linksResults,
2393 onpass="Link are correct",
2394 onfail="Links are incorrect" )
2395
2396 main.step( "Hosts are correct" )
2397 utilities.assert_equals(
2398 expect=main.TRUE,
2399 actual=hostsResults,
2400 onpass="Hosts are correct",
2401 onfail="Hosts are incorrect" )
2402
2403
2404 main.step( "Checking ONOS nodes" )
2405 nodesOutput = []
2406 nodeResults = main.TRUE
2407 threads = []
2408 for i in range( main.numCtrls ):
2409 t = main.Thread( target=main.CLIs[i].nodes,
2410 name="nodes-" + str( i ),
2411 args=[ ] )
2412 threads.append( t )
2413 t.start()
2414
2415 for t in threads:
2416 t.join()
2417 nodesOutput.append( t.result )
2418 ips = [ node.ip_address for node in main.nodes ]
2419 for i in nodesOutput:
2420 try:
2421 current = json.loads( i )
2422 for node in current:
2423 currentResult = main.FALSE
2424 if node['ip'] in ips:
2425 if node['state'] == 'ACTIVE':
2426 currentResult = main.TRUE
2427 else:
2428 main.log.error( "Error in ONOS node availability" )
2429 main.log.error(
2430 json.dumps( current,
2431 sort_keys=True,
2432 indent=4,
2433 separators=( ',', ': ' ) ) )
2434 break
2435 nodeResults = nodeResults and currentResult
2436 except ( ValueError, TypeError ):
2437 main.log.error( "Error parsing nodes output" )
2438 main.log.warn( repr( i ) )
2439 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2440 onpass="Nodes check successful",
2441 onfail="Nodes check NOT successful" )
2442
2443 - def CASE9( self, main ):
2444 """
2445 Link s3-s28 down
2446 """
2447 import time
2448 assert main.numCtrls, "main.numCtrls not defined"
2449 assert main, "main not defined"
2450 assert utilities.assert_equals, "utilities.assert_equals not defined"
2451 assert main.CLIs, "main.CLIs not defined"
2452 assert main.nodes, "main.nodes not defined"
2453
2454
2455 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2456
2457 description = "Turn off a link to ensure that Link Discovery " +\
2458 "is working properly"
2459 main.case( description )
2460
2461 main.step( "Kill Link between s3 and s28" )
2462 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2463 main.log.info( "Waiting " + str( linkSleep ) +
2464 " seconds for link down to be discovered" )
2465 time.sleep( linkSleep )
2466 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2467 onpass="Link down successful",
2468 onfail="Failed to bring link down" )
2469
2470
2472 """
2473 Link s3-s28 up
2474 """
2475 import time
2476 assert main.numCtrls, "main.numCtrls not defined"
2477 assert main, "main not defined"
2478 assert utilities.assert_equals, "utilities.assert_equals not defined"
2479 assert main.CLIs, "main.CLIs not defined"
2480 assert main.nodes, "main.nodes not defined"
2481
2482
2483 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2484
2485 description = "Restore a link to ensure that Link Discovery is " + \
2486 "working properly"
2487 main.case( description )
2488
2489 main.step( "Bring link between s3 and s28 back up" )
2490 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2491 main.log.info( "Waiting " + str( linkSleep ) +
2492 " seconds for link up to be discovered" )
2493 time.sleep( linkSleep )
2494 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2495 onpass="Link up successful",
2496 onfail="Failed to bring link up" )
2497
2498
2500 """
2501 Switch Down
2502 """
2503
2504 import time
2505 assert main.numCtrls, "main.numCtrls not defined"
2506 assert main, "main not defined"
2507 assert utilities.assert_equals, "utilities.assert_equals not defined"
2508 assert main.CLIs, "main.CLIs not defined"
2509 assert main.nodes, "main.nodes not defined"
2510
2511 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2512
2513 description = "Killing a switch to ensure it is discovered correctly"
2514 main.case( description )
2515 switch = main.params[ 'kill' ][ 'switch' ]
2516 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2517
2518
2519 main.step( "Kill " + switch )
2520 main.log.info( "Deleting " + switch )
2521 main.Mininet1.delSwitch( switch )
2522 main.log.info( "Waiting " + str( switchSleep ) +
2523 " seconds for switch down to be discovered" )
2524 time.sleep( switchSleep )
2525 device = main.ONOScli1.getDevice( dpid=switchDPID )
2526
2527 main.log.warn( str( device ) )
2528 result = main.FALSE
2529 if device and device[ 'available' ] is False:
2530 result = main.TRUE
2531 utilities.assert_equals( expect=main.TRUE, actual=result,
2532 onpass="Kill switch successful",
2533 onfail="Failed to kill switch?" )
2534
2536 """
2537 Switch Up
2538 """
2539
2540 import time
2541 assert main.numCtrls, "main.numCtrls not defined"
2542 assert main, "main not defined"
2543 assert utilities.assert_equals, "utilities.assert_equals not defined"
2544 assert main.CLIs, "main.CLIs not defined"
2545 assert main.nodes, "main.nodes not defined"
2546 assert ONOS1Port, "ONOS1Port not defined"
2547 assert ONOS2Port, "ONOS2Port not defined"
2548 assert ONOS3Port, "ONOS3Port not defined"
2549 assert ONOS4Port, "ONOS4Port not defined"
2550 assert ONOS5Port, "ONOS5Port not defined"
2551 assert ONOS6Port, "ONOS6Port not defined"
2552 assert ONOS7Port, "ONOS7Port not defined"
2553
2554 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2555 switch = main.params[ 'kill' ][ 'switch' ]
2556 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2557 links = main.params[ 'kill' ][ 'links' ].split()
2558 description = "Adding a switch to ensure it is discovered correctly"
2559 main.case( description )
2560
2561 main.step( "Add back " + switch )
2562 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2563 for peer in links:
2564 main.Mininet1.addLink( switch, peer )
2565 ipList = []
2566 for i in range( main.numCtrls ):
2567 ipList.append( main.nodes[ i ].ip_address )
2568 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2569 main.log.info( "Waiting " + str( switchSleep ) +
2570 " seconds for switch up to be discovered" )
2571 time.sleep( switchSleep )
2572 device = main.ONOScli1.getDevice( dpid=switchDPID )
2573
2574 main.log.warn( str( device ) )
2575 result = main.FALSE
2576 if device and device[ 'available' ]:
2577 result = main.TRUE
2578 utilities.assert_equals( expect=main.TRUE, actual=result,
2579 onpass="add switch successful",
2580 onfail="Failed to add switch?" )
2581
2583 """
2584 Clean up
2585 """
2586 import os
2587 import time
2588 assert main.numCtrls, "main.numCtrls not defined"
2589 assert main, "main not defined"
2590 assert utilities.assert_equals, "utilities.assert_equals not defined"
2591 assert main.CLIs, "main.CLIs not defined"
2592 assert main.nodes, "main.nodes not defined"
2593
2594
2595 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2596 'blue': '\033[94m', 'green': '\033[92m',
2597 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2598 main.case( "Test Cleanup" )
2599 main.step( "Killing tcpdumps" )
2600 main.Mininet2.stopTcpdump()
2601
2602 testname = main.TEST
2603 if main.params[ 'BACKUP' ] == "True":
2604 main.step( "Copying MN pcap and ONOS log files to test station" )
2605 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2606 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
2607
2608
2609
2610
2611
2612
2613 logFolder = "/opt/onos/log/"
2614 logFiles = [ "karaf.log", "karaf.log.1" ]
2615
2616 dstDir = "~/packet_captures/"
2617 for f in logFiles:
2618 for node in main.nodes:
2619 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2620 ":" + logFolder + f + " " +
2621 teststationUser + "@" +
2622 teststationIP + ":" +
2623 dstDir + str( testname ) +
2624 "-" + node.name + "-" + f )
2625 main.ONOSbench.handle.expect( "\$" )
2626
2627
2628
2629 logFolder = "/opt/onos/var/"
2630 logFiles = [ "stderr.log", "stdout.log" ]
2631
2632 dstDir = "~/packet_captures/"
2633 for f in logFiles:
2634 for node in main.nodes:
2635 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2636 ":" + logFolder + f + " " +
2637 teststationUser + "@" +
2638 teststationIP + ":" +
2639 dstDir + str( testname ) +
2640 "-" + node.name + "-" + f )
2641 main.ONOSbench.handle.expect( "\$" )
2642
2643 time.sleep( 10 )
2644 main.step( "Packing and rotating pcap archives" )
2645 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
2646
2647 main.step( "Stopping Mininet" )
2648 mnResult = main.Mininet1.stopNet()
2649 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2650 onpass="Mininet stopped",
2651 onfail="MN cleanup NOT successful" )
2652
2653 main.step( "Checking ONOS Logs for errors" )
2654 for node in main.nodes:
2655 print colors[ 'purple' ] + "Checking logs for errors on " + \
2656 node.name + ":" + colors[ 'end' ]
2657 print main.ONOSbench.checkLogs( node.ip_address )
2658
2659 try:
2660 timerLog = open( main.logdir + "/Timers.csv", 'w')
2661
2662 labels = "Gossip Intents"
2663 data = str( gossipTime )
2664 timerLog.write( labels + "\n" + data )
2665 timerLog.close()
2666 except NameError, e:
2667 main.log.exception(e)
2668
2670 """
2671 start election app on all onos nodes
2672 """
2673 assert main.numCtrls, "main.numCtrls not defined"
2674 assert main, "main not defined"
2675 assert utilities.assert_equals, "utilities.assert_equals not defined"
2676 assert main.CLIs, "main.CLIs not defined"
2677 assert main.nodes, "main.nodes not defined"
2678
2679 main.case("Start Leadership Election app")
2680 main.step( "Install leadership election app" )
2681 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2682 utilities.assert_equals(
2683 expect=main.TRUE,
2684 actual=appResult,
2685 onpass="Election app installed",
2686 onfail="Something went wrong with installing Leadership election" )
2687
2688 main.step( "Run for election on each node" )
2689 leaderResult = main.TRUE
2690 leaders = []
2691 for cli in main.CLIs:
2692 cli.electionTestRun()
2693 for cli in main.CLIs:
2694 leader = cli.electionTestLeader()
2695 if leader is None or leader == main.FALSE:
2696 main.log.error( cli.name + ": Leader for the election app " +
2697 "should be an ONOS node, instead got '" +
2698 str( leader ) + "'" )
2699 leaderResult = main.FALSE
2700 leaders.append( leader )
2701 utilities.assert_equals(
2702 expect=main.TRUE,
2703 actual=leaderResult,
2704 onpass="Successfully ran for leadership",
2705 onfail="Failed to run for leadership" )
2706
2707 main.step( "Check that each node shows the same leader" )
2708 sameLeader = main.TRUE
2709 if len( set( leaders ) ) != 1:
2710 sameLeader = main.FALSE
2711 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
2712 str( leaders ) )
2713 utilities.assert_equals(
2714 expect=main.TRUE,
2715 actual=sameLeader,
2716 onpass="Leadership is consistent for the election topic",
2717 onfail="Nodes have different leaders" )
2718
2720 """
2721 Check that Leadership Election is still functional
2722 """
2723 import time
2724 assert main.numCtrls, "main.numCtrls not defined"
2725 assert main, "main not defined"
2726 assert utilities.assert_equals, "utilities.assert_equals not defined"
2727 assert main.CLIs, "main.CLIs not defined"
2728 assert main.nodes, "main.nodes not defined"
2729
2730 leaderResult = main.TRUE
2731 description = "Check that Leadership Election is still functional"
2732 main.case( description )
2733
2734 main.step( "Check that each node shows the same leader" )
2735 sameLeader = main.TRUE
2736 leaders = []
2737 for cli in main.CLIs:
2738 leader = cli.electionTestLeader()
2739 leaders.append( leader )
2740 if len( set( leaders ) ) != 1:
2741 sameLeader = main.FALSE
2742 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
2743 str( leaders ) )
2744 utilities.assert_equals(
2745 expect=main.TRUE,
2746 actual=sameLeader,
2747 onpass="Leadership is consistent for the election topic",
2748 onfail="Nodes have different leaders" )
2749
2750 main.step( "Find current leader and withdraw" )
2751 leader = main.ONOScli1.electionTestLeader()
2752
2753 withdrawResult = main.FALSE
2754 if leader is None or leader == main.FALSE:
2755 main.log.error(
2756 "Leader for the election app should be an ONOS node," +
2757 "instead got '" + str( leader ) + "'" )
2758 leaderResult = main.FALSE
2759 oldLeader = None
2760 for i in range( len( main.CLIs ) ):
2761 if leader == main.nodes[ i ].ip_address:
2762 oldLeader = main.CLIs[ i ]
2763 break
2764 else:
2765 main.log.error( "Leader election, could not find current leader" )
2766 if oldLeader:
2767 withdrawResult = oldLeader.electionTestWithdraw()
2768 utilities.assert_equals(
2769 expect=main.TRUE,
2770 actual=withdrawResult,
2771 onpass="Node was withdrawn from election",
2772 onfail="Node was not withdrawn from election" )
2773
2774 main.step( "Make sure new leader is elected" )
2775
2776 leaderList = []
2777 for cli in main.CLIs:
2778 leaderN = cli.electionTestLeader()
2779 leaderList.append( leaderN )
2780 if leaderN == leader:
2781 main.log.error( cli.name + " still sees " + str( leader ) +
2782 " as leader after they withdrew" )
2783 leaderResult = main.FALSE
2784 elif leaderN == main.FALSE:
2785
2786
2787
2788 main.log.error( "Something is wrong with " +
2789 "electionTestLeader function, " +
2790 "check the error logs" )
2791 leaderResult = main.FALSE
2792 elif leaderN is None:
2793
2794 time.sleep(7)
2795 leaderN = cli.electionTestLeader()
2796 leaderList.pop()
2797 leaderList.append( leaderN )
2798 consistentLeader = main.FALSE
2799 if len( set( leaderList ) ) == 1:
2800 main.log.info( "Each Election-app sees '" +
2801 str( leaderList[ 0 ] ) +
2802 "' as the leader" )
2803 consistentLeader = main.TRUE
2804 else:
2805 main.log.error(
2806 "Inconsistent responses for leader of Election-app:" )
2807 for n in range( len( leaderList ) ):
2808 main.log.error( "ONOS" + str( n + 1 ) + " response: " +
2809 str( leaderList[ n ] ) )
2810 leaderResult = leaderResult and consistentLeader
2811 utilities.assert_equals(
2812 expect=main.TRUE,
2813 actual=leaderResult,
2814 onpass="Leadership election passed",
2815 onfail="Something went wrong with Leadership election" )
2816
2817 main.step( "Run for election on old leader( just so everyone " +
2818 "is in the hat )" )
2819 if oldLeader:
2820 runResult = oldLeader.electionTestRun()
2821 else:
2822 runResult = main.FALSE
2823 utilities.assert_equals(
2824 expect=main.TRUE,
2825 actual=runResult,
2826 onpass="App re-ran for election",
2827 onfail="App failed to run for election" )
2828
2829 main.step( "Leader did not change when old leader re-ran" )
2830 afterRun = main.ONOScli1.electionTestLeader()
2831
2832 if afterRun == leaderList[ 0 ]:
2833 afterResult = main.TRUE
2834 else:
2835 afterResult = main.FALSE
2836
2837 utilities.assert_equals(
2838 expect=main.TRUE,
2839 actual=afterResult,
2840 onpass="Old leader successfully re-ran for election",
2841 onfail="Something went wrong with Leadership election after " +
2842 "the old leader re-ran for election" )
2843
2845 """
2846 Install Distributed Primitives app
2847 """
2848 import time
2849 assert main.numCtrls, "main.numCtrls not defined"
2850 assert main, "main not defined"
2851 assert utilities.assert_equals, "utilities.assert_equals not defined"
2852 assert main.CLIs, "main.CLIs not defined"
2853 assert main.nodes, "main.nodes not defined"
2854
2855
2856 global pCounterName
2857 global iCounterName
2858 global pCounterValue
2859 global iCounterValue
2860 global onosSet
2861 global onosSetName
2862 pCounterName = "TestON-Partitions"
2863 iCounterName = "TestON-inMemory"
2864 pCounterValue = 0
2865 iCounterValue = 0
2866 onosSet = set([])
2867 onosSetName = "TestON-set"
2868
2869 description = "Install Primitives app"
2870 main.case( description )
2871 main.step( "Install Primitives app" )
2872 appName = "org.onosproject.distributedprimitives"
2873 appResults = main.CLIs[0].activateApp( appName )
2874 utilities.assert_equals( expect=main.TRUE,
2875 actual=appResults,
2876 onpass="Primitives app activated",
2877 onfail="Primitives app not activated" )
2878 time.sleep( 5 )
2879
2881 """
2882 Check for basic functionality with distributed primitives
2883 """
2884
2885 assert main.numCtrls, "main.numCtrls not defined"
2886 assert main, "main not defined"
2887 assert utilities.assert_equals, "utilities.assert_equals not defined"
2888 assert main.CLIs, "main.CLIs not defined"
2889 assert main.nodes, "main.nodes not defined"
2890 assert pCounterName, "pCounterName not defined"
2891 assert iCounterName, "iCounterName not defined"
2892 assert onosSetName, "onosSetName not defined"
2893
2894 try:
2895 pCounterValue
2896 except NameError:
2897 main.log.error( "pCounterValue not defined, setting to 0" )
2898 pCounterValue = 0
2899 try:
2900 iCounterValue
2901 except NameError:
2902 main.log.error( "iCounterValue not defined, setting to 0" )
2903 iCounterValue = 0
2904 try:
2905 onosSet
2906 except NameError:
2907 main.log.error( "onosSet not defined, setting to empty Set" )
2908 onosSet = set([])
2909
2910 addValue = "a"
2911 addAllValue = "a b c d e f"
2912 retainValue = "c d e f"
2913
2914 description = "Check for basic functionality with distributed " +\
2915 "primitives"
2916 main.case( description )
2917 main.caseExplanation = "Test the methods of the distributed " +\
2918 "primitives (counters and sets) throught the cli"
2919
2920
2921 main.step( "Increment then get a default counter on each node" )
2922 pCounters = []
2923 threads = []
2924 addedPValues = []
2925 for i in range( main.numCtrls ):
2926 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2927 name="counterAddAndGet-" + str( i ),
2928 args=[ pCounterName ] )
2929 pCounterValue += 1
2930 addedPValues.append( pCounterValue )
2931 threads.append( t )
2932 t.start()
2933
2934 for t in threads:
2935 t.join()
2936 pCounters.append( t.result )
2937
2938 pCounterResults = True
2939 for i in addedPValues:
2940 tmpResult = i in pCounters
2941 pCounterResults = pCounterResults and tmpResult
2942 if not tmpResult:
2943 main.log.error( str( i ) + " is not in partitioned "
2944 "counter incremented results" )
2945 utilities.assert_equals( expect=True,
2946 actual=pCounterResults,
2947 onpass="Default counter incremented",
2948 onfail="Error incrementing default" +
2949 " counter" )
2950
2951 main.step( "Get then Increment a default counter on each node" )
2952 pCounters = []
2953 threads = []
2954 addedPValues = []
2955 for i in range( main.numCtrls ):
2956 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2957 name="counterGetAndAdd-" + str( i ),
2958 args=[ pCounterName ] )
2959 addedPValues.append( pCounterValue )
2960 pCounterValue += 1
2961 threads.append( t )
2962 t.start()
2963
2964 for t in threads:
2965 t.join()
2966 pCounters.append( t.result )
2967
2968 pCounterResults = True
2969 for i in addedPValues:
2970 tmpResult = i in pCounters
2971 pCounterResults = pCounterResults and tmpResult
2972 if not tmpResult:
2973 main.log.error( str( i ) + " is not in partitioned "
2974 "counter incremented results" )
2975 utilities.assert_equals( expect=True,
2976 actual=pCounterResults,
2977 onpass="Default counter incremented",
2978 onfail="Error incrementing default" +
2979 " counter" )
2980
2981 main.step( "Counters we added have the correct values" )
2982 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2983 utilities.assert_equals( expect=main.TRUE,
2984 actual=incrementCheck,
2985 onpass="Added counters are correct",
2986 onfail="Added counters are incorrect" )
2987
2988 main.step( "Add -8 to then get a default counter on each node" )
2989 pCounters = []
2990 threads = []
2991 addedPValues = []
2992 for i in range( main.numCtrls ):
2993 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2994 name="counterIncrement-" + str( i ),
2995 args=[ pCounterName ],
2996 kwargs={ "delta": -8 } )
2997 pCounterValue += -8
2998 addedPValues.append( pCounterValue )
2999 threads.append( t )
3000 t.start()
3001
3002 for t in threads:
3003 t.join()
3004 pCounters.append( t.result )
3005
3006 pCounterResults = True
3007 for i in addedPValues:
3008 tmpResult = i in pCounters
3009 pCounterResults = pCounterResults and tmpResult
3010 if not tmpResult:
3011 main.log.error( str( i ) + " is not in partitioned "
3012 "counter incremented results" )
3013 utilities.assert_equals( expect=True,
3014 actual=pCounterResults,
3015 onpass="Default counter incremented",
3016 onfail="Error incrementing default" +
3017 " counter" )
3018
3019 main.step( "Add 5 to then get a default counter on each node" )
3020 pCounters = []
3021 threads = []
3022 addedPValues = []
3023 for i in range( main.numCtrls ):
3024 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3025 name="counterIncrement-" + str( i ),
3026 args=[ pCounterName ],
3027 kwargs={ "delta": 5 } )
3028 pCounterValue += 5
3029 addedPValues.append( pCounterValue )
3030 threads.append( t )
3031 t.start()
3032
3033 for t in threads:
3034 t.join()
3035 pCounters.append( t.result )
3036
3037 pCounterResults = True
3038 for i in addedPValues:
3039 tmpResult = i in pCounters
3040 pCounterResults = pCounterResults and tmpResult
3041 if not tmpResult:
3042 main.log.error( str( i ) + " is not in partitioned "
3043 "counter incremented results" )
3044 utilities.assert_equals( expect=True,
3045 actual=pCounterResults,
3046 onpass="Default counter incremented",
3047 onfail="Error incrementing default" +
3048 " counter" )
3049
3050 main.step( "Get then add 5 to a default counter on each node" )
3051 pCounters = []
3052 threads = []
3053 addedPValues = []
3054 for i in range( main.numCtrls ):
3055 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3056 name="counterIncrement-" + str( i ),
3057 args=[ pCounterName ],
3058 kwargs={ "delta": 5 } )
3059 addedPValues.append( pCounterValue )
3060 pCounterValue += 5
3061 threads.append( t )
3062 t.start()
3063
3064 for t in threads:
3065 t.join()
3066 pCounters.append( t.result )
3067
3068 pCounterResults = True
3069 for i in addedPValues:
3070 tmpResult = i in pCounters
3071 pCounterResults = pCounterResults and tmpResult
3072 if not tmpResult:
3073 main.log.error( str( i ) + " is not in partitioned "
3074 "counter incremented results" )
3075 utilities.assert_equals( expect=True,
3076 actual=pCounterResults,
3077 onpass="Default counter incremented",
3078 onfail="Error incrementing default" +
3079 " counter" )
3080
3081 main.step( "Counters we added have the correct values" )
3082 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3083 utilities.assert_equals( expect=main.TRUE,
3084 actual=incrementCheck,
3085 onpass="Added counters are correct",
3086 onfail="Added counters are incorrect" )
3087
3088
3089 main.step( "Increment and get an in-memory counter on each node" )
3090 iCounters = []
3091 addedIValues = []
3092 threads = []
3093 for i in range( main.numCtrls ):
3094 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3095 name="icounterIncrement-" + str( i ),
3096 args=[ iCounterName ],
3097 kwargs={ "inMemory": True } )
3098 iCounterValue += 1
3099 addedIValues.append( iCounterValue )
3100 threads.append( t )
3101 t.start()
3102
3103 for t in threads:
3104 t.join()
3105 iCounters.append( t.result )
3106
3107 iCounterResults = True
3108 for i in addedIValues:
3109 tmpResult = i in iCounters
3110 iCounterResults = iCounterResults and tmpResult
3111 if not tmpResult:
3112 main.log.error( str( i ) + " is not in the in-memory "
3113 "counter incremented results" )
3114 utilities.assert_equals( expect=True,
3115 actual=iCounterResults,
3116 onpass="In-memory counter incremented",
3117 onfail="Error incrementing in-memory" +
3118 " counter" )
3119
3120 main.step( "Get then Increment a in-memory counter on each node" )
3121 iCounters = []
3122 threads = []
3123 addedIValues = []
3124 for i in range( main.numCtrls ):
3125 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3126 name="counterGetAndAdd-" + str( i ),
3127 args=[ iCounterName ],
3128 kwargs={ "inMemory": True } )
3129 addedIValues.append( iCounterValue )
3130 iCounterValue += 1
3131 threads.append( t )
3132 t.start()
3133
3134 for t in threads:
3135 t.join()
3136 iCounters.append( t.result )
3137
3138 iCounterResults = True
3139 for i in addedIValues:
3140 tmpResult = i in iCounters
3141 iCounterResults = iCounterResults and tmpResult
3142 if not tmpResult:
3143 main.log.error( str( i ) + " is not in in-memory "
3144 "counter incremented results" )
3145 utilities.assert_equals( expect=True,
3146 actual=iCounterResults,
3147 onpass="In-memory counter incremented",
3148 onfail="Error incrementing in-memory" +
3149 " counter" )
3150
3151 main.step( "Counters we added have the correct values" )
3152 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3153 utilities.assert_equals( expect=main.TRUE,
3154 actual=incrementCheck,
3155 onpass="Added counters are correct",
3156 onfail="Added counters are incorrect" )
3157
3158 main.step( "Add -8 to then get a in-memory counter on each node" )
3159 iCounters = []
3160 threads = []
3161 addedIValues = []
3162 for i in range( main.numCtrls ):
3163 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3164 name="counterIncrement-" + str( i ),
3165 args=[ iCounterName ],
3166 kwargs={ "delta": -8, "inMemory": True } )
3167 iCounterValue += -8
3168 addedIValues.append( iCounterValue )
3169 threads.append( t )
3170 t.start()
3171
3172 for t in threads:
3173 t.join()
3174 iCounters.append( t.result )
3175
3176 iCounterResults = True
3177 for i in addedIValues:
3178 tmpResult = i in iCounters
3179 iCounterResults = iCounterResults and tmpResult
3180 if not tmpResult:
3181 main.log.error( str( i ) + " is not in in-memory "
3182 "counter incremented results" )
3183 utilities.assert_equals( expect=True,
3184 actual=pCounterResults,
3185 onpass="In-memory counter incremented",
3186 onfail="Error incrementing in-memory" +
3187 " counter" )
3188
3189 main.step( "Add 5 to then get a in-memory counter on each node" )
3190 iCounters = []
3191 threads = []
3192 addedIValues = []
3193 for i in range( main.numCtrls ):
3194 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3195 name="counterIncrement-" + str( i ),
3196 args=[ iCounterName ],
3197 kwargs={ "delta": 5, "inMemory": True } )
3198 iCounterValue += 5
3199 addedIValues.append( iCounterValue )
3200 threads.append( t )
3201 t.start()
3202
3203 for t in threads:
3204 t.join()
3205 iCounters.append( t.result )
3206
3207 iCounterResults = True
3208 for i in addedIValues:
3209 tmpResult = i in iCounters
3210 iCounterResults = iCounterResults and tmpResult
3211 if not tmpResult:
3212 main.log.error( str( i ) + " is not in in-memory "
3213 "counter incremented results" )
3214 utilities.assert_equals( expect=True,
3215 actual=pCounterResults,
3216 onpass="In-memory counter incremented",
3217 onfail="Error incrementing in-memory" +
3218 " counter" )
3219
3220 main.step( "Get then add 5 to a in-memory counter on each node" )
3221 iCounters = []
3222 threads = []
3223 addedIValues = []
3224 for i in range( main.numCtrls ):
3225 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3226 name="counterIncrement-" + str( i ),
3227 args=[ iCounterName ],
3228 kwargs={ "delta": 5, "inMemory": True } )
3229 addedIValues.append( iCounterValue )
3230 iCounterValue += 5
3231 threads.append( t )
3232 t.start()
3233
3234 for t in threads:
3235 t.join()
3236 iCounters.append( t.result )
3237
3238 iCounterResults = True
3239 for i in addedIValues:
3240 tmpResult = i in iCounters
3241 iCounterResults = iCounterResults and tmpResult
3242 if not tmpResult:
3243 main.log.error( str( i ) + " is not in in-memory "
3244 "counter incremented results" )
3245 utilities.assert_equals( expect=True,
3246 actual=iCounterResults,
3247 onpass="In-memory counter incremented",
3248 onfail="Error incrementing in-memory" +
3249 " counter" )
3250
3251 main.step( "Counters we added have the correct values" )
3252 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3253 utilities.assert_equals( expect=main.TRUE,
3254 actual=incrementCheck,
3255 onpass="Added counters are correct",
3256 onfail="Added counters are incorrect" )
3257
3258 main.step( "Check counters are consistant across nodes" )
3259 onosCounters = []
3260 threads = []
3261 for i in range( main.numCtrls ):
3262 t = main.Thread( target=main.CLIs[i].counters,
3263 name="counters-" + str( i ) )
3264 threads.append( t )
3265 t.start()
3266 for t in threads:
3267 t.join()
3268 onosCounters.append( t.result )
3269 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3270 if all( tmp ):
3271 main.log.info( "Counters are consistent across all nodes" )
3272 consistentCounterResults = main.TRUE
3273 else:
3274 main.log.error( "Counters are not consistent across all nodes" )
3275 consistentCounterResults = main.FALSE
3276 utilities.assert_equals( expect=main.TRUE,
3277 actual=consistentCounterResults,
3278 onpass="ONOS counters are consistent " +
3279 "across nodes",
3280 onfail="ONOS Counters are inconsistent " +
3281 "across nodes" )
3282
3283 main.step( "Counters we added have the correct values" )
3284 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3285 incrementCheck = incrementCheck and \
3286 main.Counters.counterCheck( iCounterName, iCounterValue )
3287 utilities.assert_equals( expect=main.TRUE,
3288 actual=incrementCheck,
3289 onpass="Added counters are correct",
3290 onfail="Added counters are incorrect" )
3291
3292
3293 main.step( "Distributed Set get" )
3294 size = len( onosSet )
3295 getResponses = []
3296 threads = []
3297 for i in range( main.numCtrls ):
3298 t = main.Thread( target=main.CLIs[i].setTestGet,
3299 name="setTestGet-" + str( i ),
3300 args=[ onosSetName ] )
3301 threads.append( t )
3302 t.start()
3303 for t in threads:
3304 t.join()
3305 getResponses.append( t.result )
3306
3307 getResults = main.TRUE
3308 for i in range( main.numCtrls ):
3309 if isinstance( getResponses[ i ], list):
3310 current = set( getResponses[ i ] )
3311 if len( current ) == len( getResponses[ i ] ):
3312
3313 if onosSet != current:
3314 main.log.error( "ONOS" + str( i + 1 ) +
3315 " has incorrect view" +
3316 " of set " + onosSetName + ":\n" +
3317 str( getResponses[ i ] ) )
3318 main.log.debug( "Expected: " + str( onosSet ) )
3319 main.log.debug( "Actual: " + str( current ) )
3320 getResults = main.FALSE
3321 else:
3322
3323 main.log.error( "ONOS" + str( i + 1 ) +
3324 " has repeat elements in" +
3325 " set " + onosSetName + ":\n" +
3326 str( getResponses[ i ] ) )
3327 getResults = main.FALSE
3328 elif getResponses[ i ] == main.ERROR:
3329 getResults = main.FALSE
3330 utilities.assert_equals( expect=main.TRUE,
3331 actual=getResults,
3332 onpass="Set elements are correct",
3333 onfail="Set elements are incorrect" )
3334
3335 main.step( "Distributed Set size" )
3336 sizeResponses = []
3337 threads = []
3338 for i in range( main.numCtrls ):
3339 t = main.Thread( target=main.CLIs[i].setTestSize,
3340 name="setTestSize-" + str( i ),
3341 args=[ onosSetName ] )
3342 threads.append( t )
3343 t.start()
3344 for t in threads:
3345 t.join()
3346 sizeResponses.append( t.result )
3347
3348 sizeResults = main.TRUE
3349 for i in range( main.numCtrls ):
3350 if size != sizeResponses[ i ]:
3351 sizeResults = main.FALSE
3352 main.log.error( "ONOS" + str( i + 1 ) +
3353 " expected a size of " + str( size ) +
3354 " for set " + onosSetName +
3355 " but got " + str( sizeResponses[ i ] ) )
3356 utilities.assert_equals( expect=main.TRUE,
3357 actual=sizeResults,
3358 onpass="Set sizes are correct",
3359 onfail="Set sizes are incorrect" )
3360
3361 main.step( "Distributed Set add()" )
3362 onosSet.add( addValue )
3363 addResponses = []
3364 threads = []
3365 for i in range( main.numCtrls ):
3366 t = main.Thread( target=main.CLIs[i].setTestAdd,
3367 name="setTestAdd-" + str( i ),
3368 args=[ onosSetName, addValue ] )
3369 threads.append( t )
3370 t.start()
3371 for t in threads:
3372 t.join()
3373 addResponses.append( t.result )
3374
3375
3376
3377
3378 addResults = main.TRUE
3379 for i in range( main.numCtrls ):
3380 if addResponses[ i ] == main.TRUE:
3381
3382 pass
3383 elif addResponses[ i ] == main.FALSE:
3384
3385 pass
3386 elif addResponses[ i ] == main.ERROR:
3387
3388 addResults = main.FALSE
3389 else:
3390
3391 addResults = main.FALSE
3392 if addResults != main.TRUE:
3393 main.log.error( "Error executing set add" )
3394
3395
3396 size = len( onosSet )
3397 getResponses = []
3398 threads = []
3399 for i in range( main.numCtrls ):
3400 t = main.Thread( target=main.CLIs[i].setTestGet,
3401 name="setTestGet-" + str( i ),
3402 args=[ onosSetName ] )
3403 threads.append( t )
3404 t.start()
3405 for t in threads:
3406 t.join()
3407 getResponses.append( t.result )
3408 getResults = main.TRUE
3409 for i in range( main.numCtrls ):
3410 if isinstance( getResponses[ i ], list):
3411 current = set( getResponses[ i ] )
3412 if len( current ) == len( getResponses[ i ] ):
3413
3414 if onosSet != current:
3415 main.log.error( "ONOS" + str( i + 1 ) +
3416 " has incorrect view" +
3417 " of set " + onosSetName + ":\n" +
3418 str( getResponses[ i ] ) )
3419 main.log.debug( "Expected: " + str( onosSet ) )
3420 main.log.debug( "Actual: " + str( current ) )
3421 getResults = main.FALSE
3422 else:
3423
3424 main.log.error( "ONOS" + str( i + 1 ) +
3425 " has repeat elements in" +
3426 " set " + onosSetName + ":\n" +
3427 str( getResponses[ i ] ) )
3428 getResults = main.FALSE
3429 elif getResponses[ i ] == main.ERROR:
3430 getResults = main.FALSE
3431 sizeResponses = []
3432 threads = []
3433 for i in range( main.numCtrls ):
3434 t = main.Thread( target=main.CLIs[i].setTestSize,
3435 name="setTestSize-" + str( i ),
3436 args=[ onosSetName ] )
3437 threads.append( t )
3438 t.start()
3439 for t in threads:
3440 t.join()
3441 sizeResponses.append( t.result )
3442 sizeResults = main.TRUE
3443 for i in range( main.numCtrls ):
3444 if size != sizeResponses[ i ]:
3445 sizeResults = main.FALSE
3446 main.log.error( "ONOS" + str( i + 1 ) +
3447 " expected a size of " + str( size ) +
3448 " for set " + onosSetName +
3449 " but got " + str( sizeResponses[ i ] ) )
3450 addResults = addResults and getResults and sizeResults
3451 utilities.assert_equals( expect=main.TRUE,
3452 actual=addResults,
3453 onpass="Set add correct",
3454 onfail="Set add was incorrect" )
3455
3456 main.step( "Distributed Set addAll()" )
3457 onosSet.update( addAllValue.split() )
3458 addResponses = []
3459 threads = []
3460 for i in range( main.numCtrls ):
3461 t = main.Thread( target=main.CLIs[i].setTestAdd,
3462 name="setTestAddAll-" + str( i ),
3463 args=[ onosSetName, addAllValue ] )
3464 threads.append( t )
3465 t.start()
3466 for t in threads:
3467 t.join()
3468 addResponses.append( t.result )
3469
3470
3471
3472
3473 addAllResults = main.TRUE
3474 for i in range( main.numCtrls ):
3475 if addResponses[ i ] == main.TRUE:
3476
3477 pass
3478 elif addResponses[ i ] == main.FALSE:
3479
3480 pass
3481 elif addResponses[ i ] == main.ERROR:
3482
3483 addAllResults = main.FALSE
3484 else:
3485
3486 addAllResults = main.FALSE
3487 if addAllResults != main.TRUE:
3488 main.log.error( "Error executing set addAll" )
3489
3490
3491 size = len( onosSet )
3492 getResponses = []
3493 threads = []
3494 for i in range( main.numCtrls ):
3495 t = main.Thread( target=main.CLIs[i].setTestGet,
3496 name="setTestGet-" + str( i ),
3497 args=[ onosSetName ] )
3498 threads.append( t )
3499 t.start()
3500 for t in threads:
3501 t.join()
3502 getResponses.append( t.result )
3503 getResults = main.TRUE
3504 for i in range( main.numCtrls ):
3505 if isinstance( getResponses[ i ], list):
3506 current = set( getResponses[ i ] )
3507 if len( current ) == len( getResponses[ i ] ):
3508
3509 if onosSet != current:
3510 main.log.error( "ONOS" + str( i + 1 ) +
3511 " has incorrect view" +
3512 " of set " + onosSetName + ":\n" +
3513 str( getResponses[ i ] ) )
3514 main.log.debug( "Expected: " + str( onosSet ) )
3515 main.log.debug( "Actual: " + str( current ) )
3516 getResults = main.FALSE
3517 else:
3518
3519 main.log.error( "ONOS" + str( i + 1 ) +
3520 " has repeat elements in" +
3521 " set " + onosSetName + ":\n" +
3522 str( getResponses[ i ] ) )
3523 getResults = main.FALSE
3524 elif getResponses[ i ] == main.ERROR:
3525 getResults = main.FALSE
3526 sizeResponses = []
3527 threads = []
3528 for i in range( main.numCtrls ):
3529 t = main.Thread( target=main.CLIs[i].setTestSize,
3530 name="setTestSize-" + str( i ),
3531 args=[ onosSetName ] )
3532 threads.append( t )
3533 t.start()
3534 for t in threads:
3535 t.join()
3536 sizeResponses.append( t.result )
3537 sizeResults = main.TRUE
3538 for i in range( main.numCtrls ):
3539 if size != sizeResponses[ i ]:
3540 sizeResults = main.FALSE
3541 main.log.error( "ONOS" + str( i + 1 ) +
3542 " expected a size of " + str( size ) +
3543 " for set " + onosSetName +
3544 " but got " + str( sizeResponses[ i ] ) )
3545 addAllResults = addAllResults and getResults and sizeResults
3546 utilities.assert_equals( expect=main.TRUE,
3547 actual=addAllResults,
3548 onpass="Set addAll correct",
3549 onfail="Set addAll was incorrect" )
3550
3551 main.step( "Distributed Set contains()" )
3552 containsResponses = []
3553 threads = []
3554 for i in range( main.numCtrls ):
3555 t = main.Thread( target=main.CLIs[i].setTestGet,
3556 name="setContains-" + str( i ),
3557 args=[ onosSetName ],
3558 kwargs={ "values": addValue } )
3559 threads.append( t )
3560 t.start()
3561 for t in threads:
3562 t.join()
3563
3564 containsResponses.append( t.result )
3565
3566 containsResults = main.TRUE
3567 for i in range( main.numCtrls ):
3568 if containsResponses[ i ] == main.ERROR:
3569 containsResults = main.FALSE
3570 else:
3571 containsResults = containsResults and\
3572 containsResponses[ i ][ 1 ]
3573 utilities.assert_equals( expect=main.TRUE,
3574 actual=containsResults,
3575 onpass="Set contains is functional",
3576 onfail="Set contains failed" )
3577
3578 main.step( "Distributed Set containsAll()" )
3579 containsAllResponses = []
3580 threads = []
3581 for i in range( main.numCtrls ):
3582 t = main.Thread( target=main.CLIs[i].setTestGet,
3583 name="setContainsAll-" + str( i ),
3584 args=[ onosSetName ],
3585 kwargs={ "values": addAllValue } )
3586 threads.append( t )
3587 t.start()
3588 for t in threads:
3589 t.join()
3590
3591 containsAllResponses.append( t.result )
3592
3593 containsAllResults = main.TRUE
3594 for i in range( main.numCtrls ):
3595 if containsResponses[ i ] == main.ERROR:
3596 containsResults = main.FALSE
3597 else:
3598 containsResults = containsResults and\
3599 containsResponses[ i ][ 1 ]
3600 utilities.assert_equals( expect=main.TRUE,
3601 actual=containsAllResults,
3602 onpass="Set containsAll is functional",
3603 onfail="Set containsAll failed" )
3604
3605 main.step( "Distributed Set remove()" )
3606 onosSet.remove( addValue )
3607 removeResponses = []
3608 threads = []
3609 for i in range( main.numCtrls ):
3610 t = main.Thread( target=main.CLIs[i].setTestRemove,
3611 name="setTestRemove-" + str( i ),
3612 args=[ onosSetName, addValue ] )
3613 threads.append( t )
3614 t.start()
3615 for t in threads:
3616 t.join()
3617 removeResponses.append( t.result )
3618
3619
3620
3621
3622 removeResults = main.TRUE
3623 for i in range( main.numCtrls ):
3624 if removeResponses[ i ] == main.TRUE:
3625
3626 pass
3627 elif removeResponses[ i ] == main.FALSE:
3628
3629 pass
3630 elif removeResponses[ i ] == main.ERROR:
3631
3632 removeResults = main.FALSE
3633 else:
3634
3635 removeResults = main.FALSE
3636 if removeResults != main.TRUE:
3637 main.log.error( "Error executing set remove" )
3638
3639
3640 size = len( onosSet )
3641 getResponses = []
3642 threads = []
3643 for i in range( main.numCtrls ):
3644 t = main.Thread( target=main.CLIs[i].setTestGet,
3645 name="setTestGet-" + str( i ),
3646 args=[ onosSetName ] )
3647 threads.append( t )
3648 t.start()
3649 for t in threads:
3650 t.join()
3651 getResponses.append( t.result )
3652 getResults = main.TRUE
3653 for i in range( main.numCtrls ):
3654 if isinstance( getResponses[ i ], list):
3655 current = set( getResponses[ i ] )
3656 if len( current ) == len( getResponses[ i ] ):
3657
3658 if onosSet != current:
3659 main.log.error( "ONOS" + str( i + 1 ) +
3660 " has incorrect view" +
3661 " of set " + onosSetName + ":\n" +
3662 str( getResponses[ i ] ) )
3663 main.log.debug( "Expected: " + str( onosSet ) )
3664 main.log.debug( "Actual: " + str( current ) )
3665 getResults = main.FALSE
3666 else:
3667
3668 main.log.error( "ONOS" + str( i + 1 ) +
3669 " has repeat elements in" +
3670 " set " + onosSetName + ":\n" +
3671 str( getResponses[ i ] ) )
3672 getResults = main.FALSE
3673 elif getResponses[ i ] == main.ERROR:
3674 getResults = main.FALSE
3675 sizeResponses = []
3676 threads = []
3677 for i in range( main.numCtrls ):
3678 t = main.Thread( target=main.CLIs[i].setTestSize,
3679 name="setTestSize-" + str( i ),
3680 args=[ onosSetName ] )
3681 threads.append( t )
3682 t.start()
3683 for t in threads:
3684 t.join()
3685 sizeResponses.append( t.result )
3686 sizeResults = main.TRUE
3687 for i in range( main.numCtrls ):
3688 if size != sizeResponses[ i ]:
3689 sizeResults = main.FALSE
3690 main.log.error( "ONOS" + str( i + 1 ) +
3691 " expected a size of " + str( size ) +
3692 " for set " + onosSetName +
3693 " but got " + str( sizeResponses[ i ] ) )
3694 removeResults = removeResults and getResults and sizeResults
3695 utilities.assert_equals( expect=main.TRUE,
3696 actual=removeResults,
3697 onpass="Set remove correct",
3698 onfail="Set remove was incorrect" )
3699
3700 main.step( "Distributed Set removeAll()" )
3701 onosSet.difference_update( addAllValue.split() )
3702 removeAllResponses = []
3703 threads = []
3704 try:
3705 for i in range( main.numCtrls ):
3706 t = main.Thread( target=main.CLIs[i].setTestRemove,
3707 name="setTestRemoveAll-" + str( i ),
3708 args=[ onosSetName, addAllValue ] )
3709 threads.append( t )
3710 t.start()
3711 for t in threads:
3712 t.join()
3713 removeAllResponses.append( t.result )
3714 except Exception, e:
3715 main.log.exception(e)
3716
3717
3718
3719
3720 removeAllResults = main.TRUE
3721 for i in range( main.numCtrls ):
3722 if removeAllResponses[ i ] == main.TRUE:
3723
3724 pass
3725 elif removeAllResponses[ i ] == main.FALSE:
3726
3727 pass
3728 elif removeAllResponses[ i ] == main.ERROR:
3729
3730 removeAllResults = main.FALSE
3731 else:
3732
3733 removeAllResults = main.FALSE
3734 if removeAllResults != main.TRUE:
3735 main.log.error( "Error executing set removeAll" )
3736
3737
3738 size = len( onosSet )
3739 getResponses = []
3740 threads = []
3741 for i in range( main.numCtrls ):
3742 t = main.Thread( target=main.CLIs[i].setTestGet,
3743 name="setTestGet-" + str( i ),
3744 args=[ onosSetName ] )
3745 threads.append( t )
3746 t.start()
3747 for t in threads:
3748 t.join()
3749 getResponses.append( t.result )
3750 getResults = main.TRUE
3751 for i in range( main.numCtrls ):
3752 if isinstance( getResponses[ i ], list):
3753 current = set( getResponses[ i ] )
3754 if len( current ) == len( getResponses[ i ] ):
3755
3756 if onosSet != current:
3757 main.log.error( "ONOS" + str( i + 1 ) +
3758 " has incorrect view" +
3759 " of set " + onosSetName + ":\n" +
3760 str( getResponses[ i ] ) )
3761 main.log.debug( "Expected: " + str( onosSet ) )
3762 main.log.debug( "Actual: " + str( current ) )
3763 getResults = main.FALSE
3764 else:
3765
3766 main.log.error( "ONOS" + str( i + 1 ) +
3767 " has repeat elements in" +
3768 " set " + onosSetName + ":\n" +
3769 str( getResponses[ i ] ) )
3770 getResults = main.FALSE
3771 elif getResponses[ i ] == main.ERROR:
3772 getResults = main.FALSE
3773 sizeResponses = []
3774 threads = []
3775 for i in range( main.numCtrls ):
3776 t = main.Thread( target=main.CLIs[i].setTestSize,
3777 name="setTestSize-" + str( i ),
3778 args=[ onosSetName ] )
3779 threads.append( t )
3780 t.start()
3781 for t in threads:
3782 t.join()
3783 sizeResponses.append( t.result )
3784 sizeResults = main.TRUE
3785 for i in range( main.numCtrls ):
3786 if size != sizeResponses[ i ]:
3787 sizeResults = main.FALSE
3788 main.log.error( "ONOS" + str( i + 1 ) +
3789 " expected a size of " + str( size ) +
3790 " for set " + onosSetName +
3791 " but got " + str( sizeResponses[ i ] ) )
3792 removeAllResults = removeAllResults and getResults and sizeResults
3793 utilities.assert_equals( expect=main.TRUE,
3794 actual=removeAllResults,
3795 onpass="Set removeAll correct",
3796 onfail="Set removeAll was incorrect" )
3797
3798 main.step( "Distributed Set addAll()" )
3799 onosSet.update( addAllValue.split() )
3800 addResponses = []
3801 threads = []
3802 for i in range( main.numCtrls ):
3803 t = main.Thread( target=main.CLIs[i].setTestAdd,
3804 name="setTestAddAll-" + str( i ),
3805 args=[ onosSetName, addAllValue ] )
3806 threads.append( t )
3807 t.start()
3808 for t in threads:
3809 t.join()
3810 addResponses.append( t.result )
3811
3812
3813
3814
3815 addAllResults = main.TRUE
3816 for i in range( main.numCtrls ):
3817 if addResponses[ i ] == main.TRUE:
3818
3819 pass
3820 elif addResponses[ i ] == main.FALSE:
3821
3822 pass
3823 elif addResponses[ i ] == main.ERROR:
3824
3825 addAllResults = main.FALSE
3826 else:
3827
3828 addAllResults = main.FALSE
3829 if addAllResults != main.TRUE:
3830 main.log.error( "Error executing set addAll" )
3831
3832
3833 size = len( onosSet )
3834 getResponses = []
3835 threads = []
3836 for i in range( main.numCtrls ):
3837 t = main.Thread( target=main.CLIs[i].setTestGet,
3838 name="setTestGet-" + str( i ),
3839 args=[ onosSetName ] )
3840 threads.append( t )
3841 t.start()
3842 for t in threads:
3843 t.join()
3844 getResponses.append( t.result )
3845 getResults = main.TRUE
3846 for i in range( main.numCtrls ):
3847 if isinstance( getResponses[ i ], list):
3848 current = set( getResponses[ i ] )
3849 if len( current ) == len( getResponses[ i ] ):
3850
3851 if onosSet != current:
3852 main.log.error( "ONOS" + str( i + 1 ) +
3853 " has incorrect view" +
3854 " of set " + onosSetName + ":\n" +
3855 str( getResponses[ i ] ) )
3856 main.log.debug( "Expected: " + str( onosSet ) )
3857 main.log.debug( "Actual: " + str( current ) )
3858 getResults = main.FALSE
3859 else:
3860
3861 main.log.error( "ONOS" + str( i + 1 ) +
3862 " has repeat elements in" +
3863 " set " + onosSetName + ":\n" +
3864 str( getResponses[ i ] ) )
3865 getResults = main.FALSE
3866 elif getResponses[ i ] == main.ERROR:
3867 getResults = main.FALSE
3868 sizeResponses = []
3869 threads = []
3870 for i in range( main.numCtrls ):
3871 t = main.Thread( target=main.CLIs[i].setTestSize,
3872 name="setTestSize-" + str( i ),
3873 args=[ onosSetName ] )
3874 threads.append( t )
3875 t.start()
3876 for t in threads:
3877 t.join()
3878 sizeResponses.append( t.result )
3879 sizeResults = main.TRUE
3880 for i in range( main.numCtrls ):
3881 if size != sizeResponses[ i ]:
3882 sizeResults = main.FALSE
3883 main.log.error( "ONOS" + str( i + 1 ) +
3884 " expected a size of " + str( size ) +
3885 " for set " + onosSetName +
3886 " but got " + str( sizeResponses[ i ] ) )
3887 addAllResults = addAllResults and getResults and sizeResults
3888 utilities.assert_equals( expect=main.TRUE,
3889 actual=addAllResults,
3890 onpass="Set addAll correct",
3891 onfail="Set addAll was incorrect" )
3892
3893 main.step( "Distributed Set clear()" )
3894 onosSet.clear()
3895 clearResponses = []
3896 threads = []
3897 for i in range( main.numCtrls ):
3898 t = main.Thread( target=main.CLIs[i].setTestRemove,
3899 name="setTestClear-" + str( i ),
3900 args=[ onosSetName, " "],
3901 kwargs={ "clear": True } )
3902 threads.append( t )
3903 t.start()
3904 for t in threads:
3905 t.join()
3906 clearResponses.append( t.result )
3907
3908
3909
3910
3911 clearResults = main.TRUE
3912 for i in range( main.numCtrls ):
3913 if clearResponses[ i ] == main.TRUE:
3914
3915 pass
3916 elif clearResponses[ i ] == main.FALSE:
3917
3918 pass
3919 elif clearResponses[ i ] == main.ERROR:
3920
3921 clearResults = main.FALSE
3922 else:
3923
3924 clearResults = main.FALSE
3925 if clearResults != main.TRUE:
3926 main.log.error( "Error executing set clear" )
3927
3928
3929 size = len( onosSet )
3930 getResponses = []
3931 threads = []
3932 for i in range( main.numCtrls ):
3933 t = main.Thread( target=main.CLIs[i].setTestGet,
3934 name="setTestGet-" + str( i ),
3935 args=[ onosSetName ] )
3936 threads.append( t )
3937 t.start()
3938 for t in threads:
3939 t.join()
3940 getResponses.append( t.result )
3941 getResults = main.TRUE
3942 for i in range( main.numCtrls ):
3943 if isinstance( getResponses[ i ], list):
3944 current = set( getResponses[ i ] )
3945 if len( current ) == len( getResponses[ i ] ):
3946
3947 if onosSet != current:
3948 main.log.error( "ONOS" + str( i + 1 ) +
3949 " has incorrect view" +
3950 " of set " + onosSetName + ":\n" +
3951 str( getResponses[ i ] ) )
3952 main.log.debug( "Expected: " + str( onosSet ) )
3953 main.log.debug( "Actual: " + str( current ) )
3954 getResults = main.FALSE
3955 else:
3956
3957 main.log.error( "ONOS" + str( i + 1 ) +
3958 " has repeat elements in" +
3959 " set " + onosSetName + ":\n" +
3960 str( getResponses[ i ] ) )
3961 getResults = main.FALSE
3962 elif getResponses[ i ] == main.ERROR:
3963 getResults = main.FALSE
3964 sizeResponses = []
3965 threads = []
3966 for i in range( main.numCtrls ):
3967 t = main.Thread( target=main.CLIs[i].setTestSize,
3968 name="setTestSize-" + str( i ),
3969 args=[ onosSetName ] )
3970 threads.append( t )
3971 t.start()
3972 for t in threads:
3973 t.join()
3974 sizeResponses.append( t.result )
3975 sizeResults = main.TRUE
3976 for i in range( main.numCtrls ):
3977 if size != sizeResponses[ i ]:
3978 sizeResults = main.FALSE
3979 main.log.error( "ONOS" + str( i + 1 ) +
3980 " expected a size of " + str( size ) +
3981 " for set " + onosSetName +
3982 " but got " + str( sizeResponses[ i ] ) )
3983 clearResults = clearResults and getResults and sizeResults
3984 utilities.assert_equals( expect=main.TRUE,
3985 actual=clearResults,
3986 onpass="Set clear correct",
3987 onfail="Set clear was incorrect" )
3988
3989 main.step( "Distributed Set addAll()" )
3990 onosSet.update( addAllValue.split() )
3991 addResponses = []
3992 threads = []
3993 for i in range( main.numCtrls ):
3994 t = main.Thread( target=main.CLIs[i].setTestAdd,
3995 name="setTestAddAll-" + str( i ),
3996 args=[ onosSetName, addAllValue ] )
3997 threads.append( t )
3998 t.start()
3999 for t in threads:
4000 t.join()
4001 addResponses.append( t.result )
4002
4003
4004
4005
4006 addAllResults = main.TRUE
4007 for i in range( main.numCtrls ):
4008 if addResponses[ i ] == main.TRUE:
4009
4010 pass
4011 elif addResponses[ i ] == main.FALSE:
4012
4013 pass
4014 elif addResponses[ i ] == main.ERROR:
4015
4016 addAllResults = main.FALSE
4017 else:
4018
4019 addAllResults = main.FALSE
4020 if addAllResults != main.TRUE:
4021 main.log.error( "Error executing set addAll" )
4022
4023
4024 size = len( onosSet )
4025 getResponses = []
4026 threads = []
4027 for i in range( main.numCtrls ):
4028 t = main.Thread( target=main.CLIs[i].setTestGet,
4029 name="setTestGet-" + str( i ),
4030 args=[ onosSetName ] )
4031 threads.append( t )
4032 t.start()
4033 for t in threads:
4034 t.join()
4035 getResponses.append( t.result )
4036 getResults = main.TRUE
4037 for i in range( main.numCtrls ):
4038 if isinstance( getResponses[ i ], list):
4039 current = set( getResponses[ i ] )
4040 if len( current ) == len( getResponses[ i ] ):
4041
4042 if onosSet != current:
4043 main.log.error( "ONOS" + str( i + 1 ) +
4044 " has incorrect view" +
4045 " of set " + onosSetName + ":\n" +
4046 str( getResponses[ i ] ) )
4047 main.log.debug( "Expected: " + str( onosSet ) )
4048 main.log.debug( "Actual: " + str( current ) )
4049 getResults = main.FALSE
4050 else:
4051
4052 main.log.error( "ONOS" + str( i + 1 ) +
4053 " has repeat elements in" +
4054 " set " + onosSetName + ":\n" +
4055 str( getResponses[ i ] ) )
4056 getResults = main.FALSE
4057 elif getResponses[ i ] == main.ERROR:
4058 getResults = main.FALSE
4059 sizeResponses = []
4060 threads = []
4061 for i in range( main.numCtrls ):
4062 t = main.Thread( target=main.CLIs[i].setTestSize,
4063 name="setTestSize-" + str( i ),
4064 args=[ onosSetName ] )
4065 threads.append( t )
4066 t.start()
4067 for t in threads:
4068 t.join()
4069 sizeResponses.append( t.result )
4070 sizeResults = main.TRUE
4071 for i in range( main.numCtrls ):
4072 if size != sizeResponses[ i ]:
4073 sizeResults = main.FALSE
4074 main.log.error( "ONOS" + str( i + 1 ) +
4075 " expected a size of " + str( size ) +
4076 " for set " + onosSetName +
4077 " but got " + str( sizeResponses[ i ] ) )
4078 addAllResults = addAllResults and getResults and sizeResults
4079 utilities.assert_equals( expect=main.TRUE,
4080 actual=addAllResults,
4081 onpass="Set addAll correct",
4082 onfail="Set addAll was incorrect" )
4083
4084 main.step( "Distributed Set retain()" )
4085 onosSet.intersection_update( retainValue.split() )
4086 retainResponses = []
4087 threads = []
4088 for i in range( main.numCtrls ):
4089 t = main.Thread( target=main.CLIs[i].setTestRemove,
4090 name="setTestRetain-" + str( i ),
4091 args=[ onosSetName, retainValue ],
4092 kwargs={ "retain": True } )
4093 threads.append( t )
4094 t.start()
4095 for t in threads:
4096 t.join()
4097 retainResponses.append( t.result )
4098
4099
4100
4101
4102 retainResults = main.TRUE
4103 for i in range( main.numCtrls ):
4104 if retainResponses[ i ] == main.TRUE:
4105
4106 pass
4107 elif retainResponses[ i ] == main.FALSE:
4108
4109 pass
4110 elif retainResponses[ i ] == main.ERROR:
4111
4112 retainResults = main.FALSE
4113 else:
4114
4115 retainResults = main.FALSE
4116 if retainResults != main.TRUE:
4117 main.log.error( "Error executing set retain" )
4118
4119
4120 size = len( onosSet )
4121 getResponses = []
4122 threads = []
4123 for i in range( main.numCtrls ):
4124 t = main.Thread( target=main.CLIs[i].setTestGet,
4125 name="setTestGet-" + str( i ),
4126 args=[ onosSetName ] )
4127 threads.append( t )
4128 t.start()
4129 for t in threads:
4130 t.join()
4131 getResponses.append( t.result )
4132 getResults = main.TRUE
4133 for i in range( main.numCtrls ):
4134 if isinstance( getResponses[ i ], list):
4135 current = set( getResponses[ i ] )
4136 if len( current ) == len( getResponses[ i ] ):
4137
4138 if onosSet != current:
4139 main.log.error( "ONOS" + str( i + 1 ) +
4140 " has incorrect view" +
4141 " of set " + onosSetName + ":\n" +
4142 str( getResponses[ i ] ) )
4143 main.log.debug( "Expected: " + str( onosSet ) )
4144 main.log.debug( "Actual: " + str( current ) )
4145 getResults = main.FALSE
4146 else:
4147
4148 main.log.error( "ONOS" + str( i + 1 ) +
4149 " has repeat elements in" +
4150 " set " + onosSetName + ":\n" +
4151 str( getResponses[ i ] ) )
4152 getResults = main.FALSE
4153 elif getResponses[ i ] == main.ERROR:
4154 getResults = main.FALSE
4155 sizeResponses = []
4156 threads = []
4157 for i in range( main.numCtrls ):
4158 t = main.Thread( target=main.CLIs[i].setTestSize,
4159 name="setTestSize-" + str( i ),
4160 args=[ onosSetName ] )
4161 threads.append( t )
4162 t.start()
4163 for t in threads:
4164 t.join()
4165 sizeResponses.append( t.result )
4166 sizeResults = main.TRUE
4167 for i in range( main.numCtrls ):
4168 if size != sizeResponses[ i ]:
4169 sizeResults = main.FALSE
4170 main.log.error( "ONOS" + str( i + 1 ) +
4171 " expected a size of " +
4172 str( size ) + " for set " + onosSetName +
4173 " but got " + str( sizeResponses[ i ] ) )
4174 retainResults = retainResults and getResults and sizeResults
4175 utilities.assert_equals( expect=main.TRUE,
4176 actual=retainResults,
4177 onpass="Set retain correct",
4178 onfail="Set retain was incorrect" )
4179