1 """
2 Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5 List of test cases:
6 CASE1: Compile ONOS and push it to the test machines
7 CASE2: Assign devices to controllers
8 CASE21: Assign mastership to controllers
9 CASE3: Assign intents
10 CASE4: Ping across added host intents
11 CASE5: Reading state of ONOS
12 CASE6: The Failure case.
13 CASE7: Check state after control plane failure
14 CASE8: Compare topo
15 CASE9: Link s3-s28 down
16 CASE10: Link s3-s28 up
17 CASE11: Switch down
18 CASE12: Switch up
19 CASE13: Clean up
20 CASE14: start election app on all onos nodes
21 CASE15: Check that Leadership Election is still functional
22 CASE16: Install Distributed Primitives app
23 CASE17: Check for basic functionality with distributed primitives
24 """
25
26
28
31
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
50 import imp
51 main.log.info( "ONOS HA test: Restart minority of ONOS nodes - " +
52 "initialization" )
53 main.case( "Setting up test environment" )
54 main.caseExplanation = "Setup the test environment including " +\
55 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57
58
59
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
66 main.numCtrls = int( main.params[ 'num_controllers' ] )
67 if main.ONOSbench.maxNodes:
68 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
70
71 global ONOS1Port
72 global ONOS2Port
73 global ONOS3Port
74 global ONOS4Port
75 global ONOS5Port
76 global ONOS6Port
77 global ONOS7Port
78
79
80
81 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
82 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
83 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
84 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
85 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
86 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
87 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
88
89 try:
90 fileName = "Counters"
91
92 path = main.params[ 'imports' ][ 'path' ]
93 main.Counters = imp.load_source( fileName,
94 path + fileName + ".py" )
95 except Exception as e:
96 main.log.exception( e )
97 main.cleanup()
98 main.exit()
99
100 main.CLIs = []
101 main.nodes = []
102 ipList = []
103 for i in range( 1, main.numCtrls + 1 ):
104 try:
105 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
106 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
107 ipList.append( main.nodes[ -1 ].ip_address )
108 except AttributeError:
109 break
110
111 main.step( "Create cell file" )
112 cellAppString = main.params[ 'ENV' ][ 'appString' ]
113 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
114 main.Mininet1.ip_address,
115 cellAppString, ipList )
116 main.step( "Applying cell variable to environment" )
117 cellResult = main.ONOSbench.setCell( cellName )
118 verifyResult = main.ONOSbench.verifyCell()
119
120
121 main.log.info( "Removing raft logs" )
122 main.ONOSbench.onosRemoveRaftLogs()
123
124 main.log.info( "Uninstalling ONOS" )
125 for node in main.nodes:
126 main.ONOSbench.onosUninstall( node.ip_address )
127
128
129 main.log.info( "Killing any ONOS processes" )
130 killResults = main.TRUE
131 for node in main.nodes:
132 killed = main.ONOSbench.onosKill( node.ip_address )
133 killResults = killResults and killed
134
135 cleanInstallResult = main.TRUE
136 gitPullResult = main.TRUE
137
138 main.step( "Starting Mininet" )
139
140
141 topoName = "obelisk.py"
142 filePath = main.ONOSbench.home + "/tools/test/topos/"
143 main.ONOSbench.copyMininetFile( topoName, filePath,
144 main.Mininet1.user_name,
145 main.Mininet1.ip_address )
146 mnResult = main.Mininet1.startNet( )
147 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
148 onpass="Mininet Started",
149 onfail="Error starting Mininet" )
150
151 main.step( "Git checkout and pull " + gitBranch )
152 if PULLCODE:
153 main.ONOSbench.gitCheckout( gitBranch )
154 gitPullResult = main.ONOSbench.gitPull()
155
156 utilities.assert_lesser( expect=0, actual=gitPullResult,
157 onpass="Git pull successful",
158 onfail="Git pull failed" )
159 main.ONOSbench.getVersion( report=True )
160
161 main.step( "Using mvn clean install" )
162 cleanInstallResult = main.TRUE
163 if PULLCODE and gitPullResult == main.TRUE:
164 cleanInstallResult = main.ONOSbench.cleanInstall()
165 else:
166 main.log.warn( "Did not pull new code so skipping mvn " +
167 "clean install" )
168 utilities.assert_equals( expect=main.TRUE,
169 actual=cleanInstallResult,
170 onpass="MCI successful",
171 onfail="MCI failed" )
172
173
174
175
176
177 job = "HAminorityRestart"
178 plotName = "Plot-HA"
179 graphs = '<ac:structured-macro ac:name="html">\n'
180 graphs += '<ac:plain-text-body><![CDATA[\n'
181 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
182 '/plot/' + plotName + '/getPlot?index=0' +\
183 '&width=500&height=300"' +\
184 'noborder="0" width="500" height="300" scrolling="yes" ' +\
185 'seamless="seamless"></iframe>\n'
186 graphs += ']]></ac:plain-text-body>\n'
187 graphs += '</ac:structured-macro>\n'
188 main.log.wiki(graphs)
189
190 main.step( "Creating ONOS package" )
191 packageResult = main.ONOSbench.onosPackage()
192 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
193 onpass="ONOS package successful",
194 onfail="ONOS package failed" )
195
196 main.step( "Installing ONOS package" )
197 onosInstallResult = main.TRUE
198 for node in main.nodes:
199 tmpResult = main.ONOSbench.onosInstall( options="-f",
200 node=node.ip_address )
201 onosInstallResult = onosInstallResult and tmpResult
202 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
203 onpass="ONOS install successful",
204 onfail="ONOS install failed" )
205
206 main.step( "Checking if ONOS is up yet" )
207 for i in range( 2 ):
208 onosIsupResult = main.TRUE
209 for node in main.nodes:
210 started = main.ONOSbench.isup( node.ip_address )
211 if not started:
212 main.log.error( node.name + " didn't start!" )
213 main.ONOSbench.onosStop( node.ip_address )
214 main.ONOSbench.onosStart( node.ip_address )
215 onosIsupResult = onosIsupResult and started
216 if onosIsupResult == main.TRUE:
217 break
218 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
219 onpass="ONOS startup successful",
220 onfail="ONOS startup failed" )
221
222 main.log.step( "Starting ONOS CLI sessions" )
223 cliResults = main.TRUE
224 threads = []
225 for i in range( main.numCtrls ):
226 t = main.Thread( target=main.CLIs[i].startOnosCli,
227 name="startOnosCli-" + str( i ),
228 args=[main.nodes[i].ip_address] )
229 threads.append( t )
230 t.start()
231
232 for t in threads:
233 t.join()
234 cliResults = cliResults and t.result
235 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
236 onpass="ONOS cli startup successful",
237 onfail="ONOS cli startup failed" )
238
239 if main.params[ 'tcpdump' ].lower() == "true":
240 main.step( "Start Packet Capture MN" )
241 main.Mininet2.startTcpdump(
242 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
243 + "-MN.pcap",
244 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
245 port=main.params[ 'MNtcpdump' ][ 'port' ] )
246
247 main.step( "App Ids check" )
248 appCheck = main.TRUE
249 threads = []
250 for i in range( main.numCtrls ):
251 t = main.Thread( target=main.CLIs[i].appToIDCheck,
252 name="appToIDCheck-" + str( i ),
253 args=[] )
254 threads.append( t )
255 t.start()
256
257 for t in threads:
258 t.join()
259 appCheck = appCheck and t.result
260 if appCheck != main.TRUE:
261 main.log.warn( main.CLIs[0].apps() )
262 main.log.warn( main.CLIs[0].appIDs() )
263 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
264 onpass="App Ids seem to be correct",
265 onfail="Something is wrong with app Ids" )
266
267 if cliResults == main.FALSE:
268 main.log.error( "Failed to start ONOS, stopping test" )
269 main.cleanup()
270 main.exit()
271
272 - def CASE2( self, main ):
273 """
274 Assign devices to controllers
275 """
276 import re
277 assert main.numCtrls, "main.numCtrls not defined"
278 assert main, "main not defined"
279 assert utilities.assert_equals, "utilities.assert_equals not defined"
280 assert main.CLIs, "main.CLIs not defined"
281 assert main.nodes, "main.nodes not defined"
282 assert ONOS1Port, "ONOS1Port not defined"
283 assert ONOS2Port, "ONOS2Port not defined"
284 assert ONOS3Port, "ONOS3Port not defined"
285 assert ONOS4Port, "ONOS4Port not defined"
286 assert ONOS5Port, "ONOS5Port not defined"
287 assert ONOS6Port, "ONOS6Port not defined"
288 assert ONOS7Port, "ONOS7Port not defined"
289
290 main.case( "Assigning devices to controllers" )
291 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
292 "and check that an ONOS node becomes the " +\
293 "master of the device."
294 main.step( "Assign switches to controllers" )
295
296 ipList = []
297 for i in range( main.numCtrls ):
298 ipList.append( main.nodes[ i ].ip_address )
299 swList = []
300 for i in range( 1, 29 ):
301 swList.append( "s" + str( i ) )
302 main.Mininet1.assignSwController( sw=swList, ip=ipList )
303
304 mastershipCheck = main.TRUE
305 for i in range( 1, 29 ):
306 response = main.Mininet1.getSwController( "s" + str( i ) )
307 try:
308 main.log.info( str( response ) )
309 except Exception:
310 main.log.info( repr( response ) )
311 for node in main.nodes:
312 if re.search( "tcp:" + node.ip_address, response ):
313 mastershipCheck = mastershipCheck and main.TRUE
314 else:
315 main.log.error( "Error, node " + node.ip_address + " is " +
316 "not in the list of controllers s" +
317 str( i ) + " is connecting to." )
318 mastershipCheck = main.FALSE
319 utilities.assert_equals(
320 expect=main.TRUE,
321 actual=mastershipCheck,
322 onpass="Switch mastership assigned correctly",
323 onfail="Switches not assigned correctly to controllers" )
324
326 """
327 Assign mastership to controllers
328 """
329 import time
330 assert main.numCtrls, "main.numCtrls not defined"
331 assert main, "main not defined"
332 assert utilities.assert_equals, "utilities.assert_equals not defined"
333 assert main.CLIs, "main.CLIs not defined"
334 assert main.nodes, "main.nodes not defined"
335 assert ONOS1Port, "ONOS1Port not defined"
336 assert ONOS2Port, "ONOS2Port not defined"
337 assert ONOS3Port, "ONOS3Port not defined"
338 assert ONOS4Port, "ONOS4Port not defined"
339 assert ONOS5Port, "ONOS5Port not defined"
340 assert ONOS6Port, "ONOS6Port not defined"
341 assert ONOS7Port, "ONOS7Port not defined"
342
343 main.case( "Assigning Controller roles for switches" )
344 main.caseExplanation = "Check that ONOS is connected to each " +\
345 "device. Then manually assign" +\
346 " mastership to specific ONOS nodes using" +\
347 " 'device-role'"
348 main.step( "Assign mastership of switches to specific controllers" )
349
350 roleCall = main.TRUE
351
352 ipList = [ ]
353 deviceList = []
354 try:
355
356
357
358 for i in range( 1, 29 ):
359
360 if i == 1:
361 c = 0
362 ip = main.nodes[ c ].ip_address
363 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
364 elif i == 2:
365 c = 1 % main.numCtrls
366 ip = main.nodes[ c ].ip_address
367 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
368 elif i == 3:
369 c = 1 % main.numCtrls
370 ip = main.nodes[ c ].ip_address
371 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
372 elif i == 4:
373 c = 3 % main.numCtrls
374 ip = main.nodes[ c ].ip_address
375 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
376 elif i == 5:
377 c = 2 % main.numCtrls
378 ip = main.nodes[ c ].ip_address
379 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
380 elif i == 6:
381 c = 2 % main.numCtrls
382 ip = main.nodes[ c ].ip_address
383 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
384 elif i == 7:
385 c = 5 % main.numCtrls
386 ip = main.nodes[ c ].ip_address
387 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
388 elif i >= 8 and i <= 17:
389 c = 4 % main.numCtrls
390 ip = main.nodes[ c ].ip_address
391 dpid = '3' + str( i ).zfill( 3 )
392 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
393 elif i >= 18 and i <= 27:
394 c = 6 % main.numCtrls
395 ip = main.nodes[ c ].ip_address
396 dpid = '6' + str( i ).zfill( 3 )
397 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
398 elif i == 28:
399 c = 0
400 ip = main.nodes[ c ].ip_address
401 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
402 else:
403 main.log.error( "You didn't write an else statement for " +
404 "switch s" + str( i ) )
405 roleCall = main.FALSE
406
407 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
408
409 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
410 ip )
411 ipList.append( ip )
412 deviceList.append( deviceId )
413 except ( AttributeError, AssertionError ):
414 main.log.exception( "Something is wrong with ONOS device view" )
415 main.log.info( main.ONOScli1.devices() )
416 utilities.assert_equals(
417 expect=main.TRUE,
418 actual=roleCall,
419 onpass="Re-assigned switch mastership to designated controller",
420 onfail="Something wrong with deviceRole calls" )
421
422 main.step( "Check mastership was correctly assigned" )
423 roleCheck = main.TRUE
424
425
426 time.sleep( 5 )
427 for i in range( len( ipList ) ):
428 ip = ipList[i]
429 deviceId = deviceList[i]
430
431 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
432 if ip in master:
433 roleCheck = roleCheck and main.TRUE
434 else:
435 roleCheck = roleCheck and main.FALSE
436 main.log.error( "Error, controller " + ip + " is not" +
437 " master " + "of device " +
438 str( deviceId ) + ". Master is " +
439 repr( master ) + "." )
440 utilities.assert_equals(
441 expect=main.TRUE,
442 actual=roleCheck,
443 onpass="Switches were successfully reassigned to designated " +
444 "controller",
445 onfail="Switches were not successfully reassigned" )
446
447 - def CASE3( self, main ):
448 """
449 Assign intents
450 """
451 import time
452 import json
453 assert main.numCtrls, "main.numCtrls not defined"
454 assert main, "main not defined"
455 assert utilities.assert_equals, "utilities.assert_equals not defined"
456 assert main.CLIs, "main.CLIs not defined"
457 assert main.nodes, "main.nodes not defined"
458 main.case( "Adding host Intents" )
459 main.caseExplanation = "Discover hosts by using pingall then " +\
460 "assign predetermined host-to-host intents." +\
461 " After installation, check that the intent" +\
462 " is distributed to all nodes and the state" +\
463 " is INSTALLED"
464
465
466 main.step( "Install reactive forwarding app" )
467 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
468 utilities.assert_equals( expect=main.TRUE, actual=installResults,
469 onpass="Install fwd successful",
470 onfail="Install fwd failed" )
471
472 main.step( "Check app ids" )
473 appCheck = main.TRUE
474 threads = []
475 for i in range( main.numCtrls ):
476 t = main.Thread( target=main.CLIs[i].appToIDCheck,
477 name="appToIDCheck-" + str( i ),
478 args=[] )
479 threads.append( t )
480 t.start()
481
482 for t in threads:
483 t.join()
484 appCheck = appCheck and t.result
485 if appCheck != main.TRUE:
486 main.log.warn( main.CLIs[0].apps() )
487 main.log.warn( main.CLIs[0].appIDs() )
488 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
489 onpass="App Ids seem to be correct",
490 onfail="Something is wrong with app Ids" )
491
492 main.step( "Discovering Hosts( Via pingall for now )" )
493
494
495 pingResult = main.FALSE
496 for i in range(2):
497 time1 = time.time()
498 pingResult = main.Mininet1.pingall()
499 if i == 0:
500 utilities.assert_equals(
501 expect=main.TRUE,
502 actual=pingResult,
503 onpass="Reactive Pingall test passed",
504 onfail="Reactive Pingall failed, " +
505 "one or more ping pairs failed" )
506 time2 = time.time()
507 main.log.info( "Time for pingall: %2f seconds" %
508 ( time2 - time1 ) )
509
510 time.sleep( 11 )
511
512 main.step( "Uninstall reactive forwarding app" )
513 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
514 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
515 onpass="Uninstall fwd successful",
516 onfail="Uninstall fwd failed" )
517
518 main.step( "Check app ids" )
519 threads = []
520 appCheck2 = main.TRUE
521 for i in range( main.numCtrls ):
522 t = main.Thread( target=main.CLIs[i].appToIDCheck,
523 name="appToIDCheck-" + str( i ),
524 args=[] )
525 threads.append( t )
526 t.start()
527
528 for t in threads:
529 t.join()
530 appCheck2 = appCheck2 and t.result
531 if appCheck2 != main.TRUE:
532 main.log.warn( main.CLIs[0].apps() )
533 main.log.warn( main.CLIs[0].appIDs() )
534 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
535 onpass="App Ids seem to be correct",
536 onfail="Something is wrong with app Ids" )
537
538 main.step( "Add host intents via cli" )
539 intentIds = []
540
541
542 intentAddResult = True
543 hostResult = main.TRUE
544 for i in range( 8, 18 ):
545 main.log.info( "Adding host intent between h" + str( i ) +
546 " and h" + str( i + 10 ) )
547 host1 = "00:00:00:00:00:" + \
548 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
549 host2 = "00:00:00:00:00:" + \
550 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
551
552 host1Dict = main.ONOScli1.getHost( host1 )
553 host2Dict = main.ONOScli1.getHost( host2 )
554 host1Id = None
555 host2Id = None
556 if host1Dict and host2Dict:
557 host1Id = host1Dict.get( 'id', None )
558 host2Id = host2Dict.get( 'id', None )
559 if host1Id and host2Id:
560 nodeNum = ( i % main.numCtrls )
561 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
562 if tmpId:
563 main.log.info( "Added intent with id: " + tmpId )
564 intentIds.append( tmpId )
565 else:
566 main.log.error( "addHostIntent returned: " +
567 repr( tmpId ) )
568 else:
569 main.log.error( "Error, getHost() failed for h" + str( i ) +
570 " and/or h" + str( i + 10 ) )
571 hosts = main.CLIs[ 0 ].hosts()
572 main.log.warn( "Hosts output: " )
573 try:
574 main.log.warn( json.dumps( json.loads( hosts ),
575 sort_keys=True,
576 indent=4,
577 separators=( ',', ': ' ) ) )
578 except ( ValueError, TypeError ):
579 main.log.warn( repr( hosts ) )
580 hostResult = main.FALSE
581 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
582 onpass="Found a host id for each host",
583 onfail="Error looking up host ids" )
584
585 intentStart = time.time()
586 onosIds = main.ONOScli1.getAllIntentsId()
587 main.log.info( "Submitted intents: " + str( intentIds ) )
588 main.log.info( "Intents in ONOS: " + str( onosIds ) )
589 for intent in intentIds:
590 if intent in onosIds:
591 pass
592 else:
593 intentAddResult = False
594 if intentAddResult:
595 intentStop = time.time()
596 else:
597 intentStop = None
598
599 intents = main.ONOScli1.intents()
600 intentStates = []
601 installedCheck = True
602 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
603 count = 0
604 try:
605 for intent in json.loads( intents ):
606 state = intent.get( 'state', None )
607 if "INSTALLED" not in state:
608 installedCheck = False
609 intentId = intent.get( 'id', None )
610 intentStates.append( ( intentId, state ) )
611 except ( ValueError, TypeError ):
612 main.log.exception( "Error parsing intents" )
613
614 tmplist = [ i for i, s in intentStates ]
615 missingIntents = False
616 for i in intentIds:
617 if i not in tmplist:
618 intentStates.append( ( i, " - " ) )
619 missingIntents = True
620 intentStates.sort()
621 for i, s in intentStates:
622 count += 1
623 main.log.info( "%-6s%-15s%-15s" %
624 ( str( count ), str( i ), str( s ) ) )
625 leaders = main.ONOScli1.leaders()
626 try:
627 missing = False
628 if leaders:
629 parsedLeaders = json.loads( leaders )
630 main.log.warn( json.dumps( parsedLeaders,
631 sort_keys=True,
632 indent=4,
633 separators=( ',', ': ' ) ) )
634
635 topics = []
636 for i in range( 14 ):
637 topics.append( "intent-partition-" + str( i ) )
638 main.log.debug( topics )
639 ONOStopics = [ j['topic'] for j in parsedLeaders ]
640 for topic in topics:
641 if topic not in ONOStopics:
642 main.log.error( "Error: " + topic +
643 " not in leaders" )
644 missing = True
645 else:
646 main.log.error( "leaders() returned None" )
647 except ( ValueError, TypeError ):
648 main.log.exception( "Error parsing leaders" )
649 main.log.error( repr( leaders ) )
650
651 if missing:
652 for node in main.CLIs:
653 response = node.leaders( jsonFormat=False)
654 main.log.warn( str( node.name ) + " leaders output: \n" +
655 str( response ) )
656
657 partitions = main.ONOScli1.partitions()
658 try:
659 if partitions :
660 parsedPartitions = json.loads( partitions )
661 main.log.warn( json.dumps( parsedPartitions,
662 sort_keys=True,
663 indent=4,
664 separators=( ',', ': ' ) ) )
665
666
667 else:
668 main.log.error( "partitions() returned None" )
669 except ( ValueError, TypeError ):
670 main.log.exception( "Error parsing partitions" )
671 main.log.error( repr( partitions ) )
672 pendingMap = main.ONOScli1.pendingMap()
673 try:
674 if pendingMap :
675 parsedPending = json.loads( pendingMap )
676 main.log.warn( json.dumps( parsedPending,
677 sort_keys=True,
678 indent=4,
679 separators=( ',', ': ' ) ) )
680
681 else:
682 main.log.error( "pendingMap() returned None" )
683 except ( ValueError, TypeError ):
684 main.log.exception( "Error parsing pending map" )
685 main.log.error( repr( pendingMap ) )
686
687 intentAddResult = bool( intentAddResult and not missingIntents and
688 installedCheck )
689 if not intentAddResult:
690 main.log.error( "Error in pushing host intents to ONOS" )
691
692 main.step( "Intent Anti-Entropy dispersion" )
693 for i in range(100):
694 correct = True
695 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
696 for cli in main.CLIs:
697 onosIds = []
698 ids = cli.getAllIntentsId()
699 onosIds.append( ids )
700 main.log.debug( "Intents in " + cli.name + ": " +
701 str( sorted( onosIds ) ) )
702 if sorted( ids ) != sorted( intentIds ):
703 main.log.warn( "Set of intent IDs doesn't match" )
704 correct = False
705 break
706 else:
707 intents = json.loads( cli.intents() )
708 for intent in intents:
709 if intent[ 'state' ] != "INSTALLED":
710 main.log.warn( "Intent " + intent[ 'id' ] +
711 " is " + intent[ 'state' ] )
712 correct = False
713 break
714 if correct:
715 break
716 else:
717 time.sleep(1)
718 if not intentStop:
719 intentStop = time.time()
720 global gossipTime
721 gossipTime = intentStop - intentStart
722 main.log.info( "It took about " + str( gossipTime ) +
723 " seconds for all intents to appear in each node" )
724
725
726 utilities.assert_greater_equals(
727 expect=40, actual=gossipTime,
728 onpass="ECM anti-entropy for intents worked within " +
729 "expected time",
730 onfail="Intent ECM anti-entropy took too long" )
731 if gossipTime <= 40:
732 intentAddResult = True
733
734 if not intentAddResult or "key" in pendingMap:
735 import time
736 installedCheck = True
737 main.log.info( "Sleeping 60 seconds to see if intents are found" )
738 time.sleep( 60 )
739 onosIds = main.ONOScli1.getAllIntentsId()
740 main.log.info( "Submitted intents: " + str( intentIds ) )
741 main.log.info( "Intents in ONOS: " + str( onosIds ) )
742
743 intents = main.ONOScli1.intents()
744 intentStates = []
745 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
746 count = 0
747 try:
748 for intent in json.loads( intents ):
749
750 state = intent.get( 'state', None )
751 if "INSTALLED" not in state:
752 installedCheck = False
753 intentId = intent.get( 'id', None )
754 intentStates.append( ( intentId, state ) )
755 except ( ValueError, TypeError ):
756 main.log.exception( "Error parsing intents" )
757
758 tmplist = [ i for i, s in intentStates ]
759 for i in intentIds:
760 if i not in tmplist:
761 intentStates.append( ( i, " - " ) )
762 intentStates.sort()
763 for i, s in intentStates:
764 count += 1
765 main.log.info( "%-6s%-15s%-15s" %
766 ( str( count ), str( i ), str( s ) ) )
767 leaders = main.ONOScli1.leaders()
768 try:
769 missing = False
770 if leaders:
771 parsedLeaders = json.loads( leaders )
772 main.log.warn( json.dumps( parsedLeaders,
773 sort_keys=True,
774 indent=4,
775 separators=( ',', ': ' ) ) )
776
777
778 topics = []
779 for i in range( 14 ):
780 topics.append( "intent-partition-" + str( i ) )
781
782 topics.append( "org.onosproject.election" )
783 main.log.debug( topics )
784 ONOStopics = [ j['topic'] for j in parsedLeaders ]
785 for topic in topics:
786 if topic not in ONOStopics:
787 main.log.error( "Error: " + topic +
788 " not in leaders" )
789 missing = True
790 else:
791 main.log.error( "leaders() returned None" )
792 except ( ValueError, TypeError ):
793 main.log.exception( "Error parsing leaders" )
794 main.log.error( repr( leaders ) )
795
796 if missing:
797 for node in main.CLIs:
798 response = node.leaders( jsonFormat=False)
799 main.log.warn( str( node.name ) + " leaders output: \n" +
800 str( response ) )
801
802 partitions = main.ONOScli1.partitions()
803 try:
804 if partitions :
805 parsedPartitions = json.loads( partitions )
806 main.log.warn( json.dumps( parsedPartitions,
807 sort_keys=True,
808 indent=4,
809 separators=( ',', ': ' ) ) )
810
811
812 else:
813 main.log.error( "partitions() returned None" )
814 except ( ValueError, TypeError ):
815 main.log.exception( "Error parsing partitions" )
816 main.log.error( repr( partitions ) )
817 pendingMap = main.ONOScli1.pendingMap()
818 try:
819 if pendingMap :
820 parsedPending = json.loads( pendingMap )
821 main.log.warn( json.dumps( parsedPending,
822 sort_keys=True,
823 indent=4,
824 separators=( ',', ': ' ) ) )
825
826 else:
827 main.log.error( "pendingMap() returned None" )
828 except ( ValueError, TypeError ):
829 main.log.exception( "Error parsing pending map" )
830 main.log.error( repr( pendingMap ) )
831
832 - def CASE4( self, main ):
833 """
834 Ping across added host intents
835 """
836 import json
837 import time
838 assert main.numCtrls, "main.numCtrls not defined"
839 assert main, "main not defined"
840 assert utilities.assert_equals, "utilities.assert_equals not defined"
841 assert main.CLIs, "main.CLIs not defined"
842 assert main.nodes, "main.nodes not defined"
843 main.case( "Verify connectivity by sendind traffic across Intents" )
844 main.caseExplanation = "Ping across added host intents to check " +\
845 "functionality and check the state of " +\
846 "the intent"
847 main.step( "Ping across added host intents" )
848 PingResult = main.TRUE
849 for i in range( 8, 18 ):
850 ping = main.Mininet1.pingHost( src="h" + str( i ),
851 target="h" + str( i + 10 ) )
852 PingResult = PingResult and ping
853 if ping == main.FALSE:
854 main.log.warn( "Ping failed between h" + str( i ) +
855 " and h" + str( i + 10 ) )
856 elif ping == main.TRUE:
857 main.log.info( "Ping test passed!" )
858
859 if PingResult == main.FALSE:
860 main.log.error(
861 "Intents have not been installed correctly, pings failed." )
862
863 main.log.warn( "ONOS1 intents: " )
864 try:
865 tmpIntents = main.ONOScli1.intents()
866 main.log.warn( json.dumps( json.loads( tmpIntents ),
867 sort_keys=True,
868 indent=4,
869 separators=( ',', ': ' ) ) )
870 except ( ValueError, TypeError ):
871 main.log.warn( repr( tmpIntents ) )
872 utilities.assert_equals(
873 expect=main.TRUE,
874 actual=PingResult,
875 onpass="Intents have been installed correctly and pings work",
876 onfail="Intents have not been installed correctly, pings failed." )
877
878 main.step( "Check Intent state" )
879 installedCheck = False
880 loopCount = 0
881 while not installedCheck and loopCount < 40:
882 installedCheck = True
883
884 intents = main.ONOScli1.intents()
885 intentStates = []
886 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
887 count = 0
888
889 try:
890 for intent in json.loads( intents ):
891 state = intent.get( 'state', None )
892 if "INSTALLED" not in state:
893 installedCheck = False
894 intentId = intent.get( 'id', None )
895 intentStates.append( ( intentId, state ) )
896 except ( ValueError, TypeError ):
897 main.log.exception( "Error parsing intents." )
898
899 intentStates.sort()
900 for i, s in intentStates:
901 count += 1
902 main.log.info( "%-6s%-15s%-15s" %
903 ( str( count ), str( i ), str( s ) ) )
904 if not installedCheck:
905 time.sleep( 1 )
906 loopCount += 1
907 utilities.assert_equals( expect=True, actual=installedCheck,
908 onpass="Intents are all INSTALLED",
909 onfail="Intents are not all in " +
910 "INSTALLED state" )
911
912 main.step( "Check leadership of topics" )
913 leaders = main.ONOScli1.leaders()
914 topicCheck = main.TRUE
915 try:
916 if leaders:
917 parsedLeaders = json.loads( leaders )
918 main.log.warn( json.dumps( parsedLeaders,
919 sort_keys=True,
920 indent=4,
921 separators=( ',', ': ' ) ) )
922
923
924
925 topics = []
926 for i in range( 14 ):
927 topics.append( "intent-partition-" + str( i ) )
928
929
930
931 main.log.debug( topics )
932 ONOStopics = [ j['topic'] for j in parsedLeaders ]
933 for topic in topics:
934 if topic not in ONOStopics:
935 main.log.error( "Error: " + topic +
936 " not in leaders" )
937 topicCheck = main.FALSE
938 else:
939 main.log.error( "leaders() returned None" )
940 topicCheck = main.FALSE
941 except ( ValueError, TypeError ):
942 topicCheck = main.FALSE
943 main.log.exception( "Error parsing leaders" )
944 main.log.error( repr( leaders ) )
945
946
947 if topicCheck:
948 for node in main.CLIs:
949 response = node.leaders( jsonFormat=False)
950 main.log.warn( str( node.name ) + " leaders output: \n" +
951 str( response ) )
952
953 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
954 onpass="intent Partitions is in leaders",
955 onfail="Some topics were lost " )
956
957 partitions = main.ONOScli1.partitions()
958 try:
959 if partitions :
960 parsedPartitions = json.loads( partitions )
961 main.log.warn( json.dumps( parsedPartitions,
962 sort_keys=True,
963 indent=4,
964 separators=( ',', ': ' ) ) )
965
966
967 else:
968 main.log.error( "partitions() returned None" )
969 except ( ValueError, TypeError ):
970 main.log.exception( "Error parsing partitions" )
971 main.log.error( repr( partitions ) )
972
973 pendingMap = main.ONOScli1.pendingMap()
974 try:
975 if pendingMap :
976 parsedPending = json.loads( pendingMap )
977 main.log.warn( json.dumps( parsedPending,
978 sort_keys=True,
979 indent=4,
980 separators=( ',', ': ' ) ) )
981
982 else:
983 main.log.error( "pendingMap() returned None" )
984 except ( ValueError, TypeError ):
985 main.log.exception( "Error parsing pending map" )
986 main.log.error( repr( pendingMap ) )
987
988 if not installedCheck:
989 main.log.info( "Waiting 60 seconds to see if the state of " +
990 "intents change" )
991 time.sleep( 60 )
992
993 intents = main.ONOScli1.intents()
994 intentStates = []
995 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
996 count = 0
997
998 try:
999 for intent in json.loads( intents ):
1000 state = intent.get( 'state', None )
1001 if "INSTALLED" not in state:
1002 installedCheck = False
1003 intentId = intent.get( 'id', None )
1004 intentStates.append( ( intentId, state ) )
1005 except ( ValueError, TypeError ):
1006 main.log.exception( "Error parsing intents." )
1007 intentStates.sort()
1008 for i, s in intentStates:
1009 count += 1
1010 main.log.info( "%-6s%-15s%-15s" %
1011 ( str( count ), str( i ), str( s ) ) )
1012 leaders = main.ONOScli1.leaders()
1013 try:
1014 missing = False
1015 if leaders:
1016 parsedLeaders = json.loads( leaders )
1017 main.log.warn( json.dumps( parsedLeaders,
1018 sort_keys=True,
1019 indent=4,
1020 separators=( ',', ': ' ) ) )
1021
1022
1023 topics = []
1024 for i in range( 14 ):
1025 topics.append( "intent-partition-" + str( i ) )
1026
1027 topics.append( "org.onosproject.election" )
1028 main.log.debug( topics )
1029 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1030 for topic in topics:
1031 if topic not in ONOStopics:
1032 main.log.error( "Error: " + topic +
1033 " not in leaders" )
1034 missing = True
1035 else:
1036 main.log.error( "leaders() returned None" )
1037 except ( ValueError, TypeError ):
1038 main.log.exception( "Error parsing leaders" )
1039 main.log.error( repr( leaders ) )
1040 if missing:
1041 for node in main.CLIs:
1042 response = node.leaders( jsonFormat=False)
1043 main.log.warn( str( node.name ) + " leaders output: \n" +
1044 str( response ) )
1045
1046 partitions = main.ONOScli1.partitions()
1047 try:
1048 if partitions :
1049 parsedPartitions = json.loads( partitions )
1050 main.log.warn( json.dumps( parsedPartitions,
1051 sort_keys=True,
1052 indent=4,
1053 separators=( ',', ': ' ) ) )
1054
1055
1056 else:
1057 main.log.error( "partitions() returned None" )
1058 except ( ValueError, TypeError ):
1059 main.log.exception( "Error parsing partitions" )
1060 main.log.error( repr( partitions ) )
1061 pendingMap = main.ONOScli1.pendingMap()
1062 try:
1063 if pendingMap :
1064 parsedPending = json.loads( pendingMap )
1065 main.log.warn( json.dumps( parsedPending,
1066 sort_keys=True,
1067 indent=4,
1068 separators=( ',', ': ' ) ) )
1069
1070 else:
1071 main.log.error( "pendingMap() returned None" )
1072 except ( ValueError, TypeError ):
1073 main.log.exception( "Error parsing pending map" )
1074 main.log.error( repr( pendingMap ) )
1075
1076 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
1077 main.step( "Wait a minute then ping again" )
1078
1079 PingResult = main.TRUE
1080 for i in range( 8, 18 ):
1081 ping = main.Mininet1.pingHost( src="h" + str( i ),
1082 target="h" + str( i + 10 ) )
1083 PingResult = PingResult and ping
1084 if ping == main.FALSE:
1085 main.log.warn( "Ping failed between h" + str( i ) +
1086 " and h" + str( i + 10 ) )
1087 elif ping == main.TRUE:
1088 main.log.info( "Ping test passed!" )
1089
1090 if PingResult == main.FALSE:
1091 main.log.error(
1092 "Intents have not been installed correctly, pings failed." )
1093
1094 main.log.warn( "ONOS1 intents: " )
1095 try:
1096 tmpIntents = main.ONOScli1.intents()
1097 main.log.warn( json.dumps( json.loads( tmpIntents ),
1098 sort_keys=True,
1099 indent=4,
1100 separators=( ',', ': ' ) ) )
1101 except ( ValueError, TypeError ):
1102 main.log.warn( repr( tmpIntents ) )
1103 utilities.assert_equals(
1104 expect=main.TRUE,
1105 actual=PingResult,
1106 onpass="Intents have been installed correctly and pings work",
1107 onfail="Intents have not been installed correctly, pings failed." )
1108
1109 - def CASE5( self, main ):
1110 """
1111 Reading state of ONOS
1112 """
1113 import json
1114 import time
1115 assert main.numCtrls, "main.numCtrls not defined"
1116 assert main, "main not defined"
1117 assert utilities.assert_equals, "utilities.assert_equals not defined"
1118 assert main.CLIs, "main.CLIs not defined"
1119 assert main.nodes, "main.nodes not defined"
1120
1121 main.case( "Setting up and gathering data for current state" )
1122
1123
1124
1125
1126 main.step( "Check that each switch has a master" )
1127 global mastershipState
1128 mastershipState = '[]'
1129
1130
1131 rolesNotNull = main.TRUE
1132 threads = []
1133 for i in range( main.numCtrls ):
1134 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1135 name="rolesNotNull-" + str( i ),
1136 args=[] )
1137 threads.append( t )
1138 t.start()
1139
1140 for t in threads:
1141 t.join()
1142 rolesNotNull = rolesNotNull and t.result
1143 utilities.assert_equals(
1144 expect=main.TRUE,
1145 actual=rolesNotNull,
1146 onpass="Each device has a master",
1147 onfail="Some devices don't have a master assigned" )
1148
1149 main.step( "Get the Mastership of each switch from each controller" )
1150 ONOSMastership = []
1151 mastershipCheck = main.FALSE
1152 consistentMastership = True
1153 rolesResults = True
1154 threads = []
1155 for i in range( main.numCtrls ):
1156 t = main.Thread( target=main.CLIs[i].roles,
1157 name="roles-" + str( i ),
1158 args=[] )
1159 threads.append( t )
1160 t.start()
1161
1162 for t in threads:
1163 t.join()
1164 ONOSMastership.append( t.result )
1165
1166 for i in range( main.numCtrls ):
1167 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1168 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1169 " roles" )
1170 main.log.warn(
1171 "ONOS" + str( i + 1 ) + " mastership response: " +
1172 repr( ONOSMastership[i] ) )
1173 rolesResults = False
1174 utilities.assert_equals(
1175 expect=True,
1176 actual=rolesResults,
1177 onpass="No error in reading roles output",
1178 onfail="Error in reading roles from ONOS" )
1179
1180 main.step( "Check for consistency in roles from each controller" )
1181 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1182 main.log.info(
1183 "Switch roles are consistent across all ONOS nodes" )
1184 else:
1185 consistentMastership = False
1186 utilities.assert_equals(
1187 expect=True,
1188 actual=consistentMastership,
1189 onpass="Switch roles are consistent across all ONOS nodes",
1190 onfail="ONOS nodes have different views of switch roles" )
1191
1192 if rolesResults and not consistentMastership:
1193 for i in range( main.numCtrls ):
1194 try:
1195 main.log.warn(
1196 "ONOS" + str( i + 1 ) + " roles: ",
1197 json.dumps(
1198 json.loads( ONOSMastership[ i ] ),
1199 sort_keys=True,
1200 indent=4,
1201 separators=( ',', ': ' ) ) )
1202 except ( ValueError, TypeError ):
1203 main.log.warn( repr( ONOSMastership[ i ] ) )
1204 elif rolesResults and consistentMastership:
1205 mastershipCheck = main.TRUE
1206 mastershipState = ONOSMastership[ 0 ]
1207
1208 main.step( "Get the intents from each controller" )
1209 global intentState
1210 intentState = []
1211 ONOSIntents = []
1212 intentCheck = main.FALSE
1213 consistentIntents = True
1214 intentsResults = True
1215 threads = []
1216 for i in range( main.numCtrls ):
1217 t = main.Thread( target=main.CLIs[i].intents,
1218 name="intents-" + str( i ),
1219 args=[],
1220 kwargs={ 'jsonFormat': True } )
1221 threads.append( t )
1222 t.start()
1223
1224 for t in threads:
1225 t.join()
1226 ONOSIntents.append( t.result )
1227
1228 for i in range( main.numCtrls ):
1229 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1230 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1231 " intents" )
1232 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1233 repr( ONOSIntents[ i ] ) )
1234 intentsResults = False
1235 utilities.assert_equals(
1236 expect=True,
1237 actual=intentsResults,
1238 onpass="No error in reading intents output",
1239 onfail="Error in reading intents from ONOS" )
1240
1241 main.step( "Check for consistency in Intents from each controller" )
1242 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1243 main.log.info( "Intents are consistent across all ONOS " +
1244 "nodes" )
1245 else:
1246 consistentIntents = False
1247 main.log.error( "Intents not consistent" )
1248 utilities.assert_equals(
1249 expect=True,
1250 actual=consistentIntents,
1251 onpass="Intents are consistent across all ONOS nodes",
1252 onfail="ONOS nodes have different views of intents" )
1253
1254 if intentsResults:
1255
1256
1257
1258
1259
1260
1261 title = " Id"
1262 for n in range( main.numCtrls ):
1263 title += " " * 10 + "ONOS" + str( n + 1 )
1264 main.log.warn( title )
1265
1266 keys = []
1267 for nodeStr in ONOSIntents:
1268 node = json.loads( nodeStr )
1269 for intent in node:
1270 keys.append( intent.get( 'id' ) )
1271 keys = set( keys )
1272 for key in keys:
1273 row = "%-13s" % key
1274 for nodeStr in ONOSIntents:
1275 node = json.loads( nodeStr )
1276 for intent in node:
1277 if intent.get( 'id', "Error" ) == key:
1278 row += "%-15s" % intent.get( 'state' )
1279 main.log.warn( row )
1280
1281
1282 if intentsResults and not consistentIntents:
1283
1284 n = len(ONOSIntents)
1285 main.log.debug( "ONOS" + str( n ) + " intents: " )
1286 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1287 sort_keys=True,
1288 indent=4,
1289 separators=( ',', ': ' ) ) )
1290 for i in range( main.numCtrls ):
1291 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1292 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1293 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1294 sort_keys=True,
1295 indent=4,
1296 separators=( ',', ': ' ) ) )
1297 else:
1298 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
1299 str( n ) + " intents" )
1300 elif intentsResults and consistentIntents:
1301 intentCheck = main.TRUE
1302 intentState = ONOSIntents[ 0 ]
1303
1304 main.step( "Get the flows from each controller" )
1305 global flowState
1306 flowState = []
1307 ONOSFlows = []
1308 ONOSFlowsJson = []
1309 flowCheck = main.FALSE
1310 consistentFlows = True
1311 flowsResults = True
1312 threads = []
1313 for i in range( main.numCtrls ):
1314 t = main.Thread( target=main.CLIs[i].flows,
1315 name="flows-" + str( i ),
1316 args=[],
1317 kwargs={ 'jsonFormat': True } )
1318 threads.append( t )
1319 t.start()
1320
1321
1322 time.sleep(30)
1323 for t in threads:
1324 t.join()
1325 result = t.result
1326 ONOSFlows.append( result )
1327
1328 for i in range( main.numCtrls ):
1329 num = str( i + 1 )
1330 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1331 main.log.error( "Error in getting ONOS" + num + " flows" )
1332 main.log.warn( "ONOS" + num + " flows response: " +
1333 repr( ONOSFlows[ i ] ) )
1334 flowsResults = False
1335 ONOSFlowsJson.append( None )
1336 else:
1337 try:
1338 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1339 except ( ValueError, TypeError ):
1340
1341 main.log.exception( "Error in parsing ONOS" + num +
1342 " response as json." )
1343 main.log.error( repr( ONOSFlows[ i ] ) )
1344 ONOSFlowsJson.append( None )
1345 flowsResults = False
1346 utilities.assert_equals(
1347 expect=True,
1348 actual=flowsResults,
1349 onpass="No error in reading flows output",
1350 onfail="Error in reading flows from ONOS" )
1351
1352 main.step( "Check for consistency in Flows from each controller" )
1353 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1354 if all( tmp ):
1355 main.log.info( "Flow count is consistent across all ONOS nodes" )
1356 else:
1357 consistentFlows = False
1358 utilities.assert_equals(
1359 expect=True,
1360 actual=consistentFlows,
1361 onpass="The flow count is consistent across all ONOS nodes",
1362 onfail="ONOS nodes have different flow counts" )
1363
1364 if flowsResults and not consistentFlows:
1365 for i in range( main.numCtrls ):
1366 try:
1367 main.log.warn(
1368 "ONOS" + str( i + 1 ) + " flows: " +
1369 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1370 indent=4, separators=( ',', ': ' ) ) )
1371 except ( ValueError, TypeError ):
1372 main.log.warn(
1373 "ONOS" + str( i + 1 ) + " flows: " +
1374 repr( ONOSFlows[ i ] ) )
1375 elif flowsResults and consistentFlows:
1376 flowCheck = main.TRUE
1377 flowState = ONOSFlows[ 0 ]
1378
1379 main.step( "Get the OF Table entries" )
1380 global flows
1381 flows = []
1382 for i in range( 1, 29 ):
1383 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
1384 if flowCheck == main.FALSE:
1385 for table in flows:
1386 main.log.warn( table )
1387
1388
1389 main.step( "Start continuous pings" )
1390 main.Mininet2.pingLong(
1391 src=main.params[ 'PING' ][ 'source1' ],
1392 target=main.params[ 'PING' ][ 'target1' ],
1393 pingTime=500 )
1394 main.Mininet2.pingLong(
1395 src=main.params[ 'PING' ][ 'source2' ],
1396 target=main.params[ 'PING' ][ 'target2' ],
1397 pingTime=500 )
1398 main.Mininet2.pingLong(
1399 src=main.params[ 'PING' ][ 'source3' ],
1400 target=main.params[ 'PING' ][ 'target3' ],
1401 pingTime=500 )
1402 main.Mininet2.pingLong(
1403 src=main.params[ 'PING' ][ 'source4' ],
1404 target=main.params[ 'PING' ][ 'target4' ],
1405 pingTime=500 )
1406 main.Mininet2.pingLong(
1407 src=main.params[ 'PING' ][ 'source5' ],
1408 target=main.params[ 'PING' ][ 'target5' ],
1409 pingTime=500 )
1410 main.Mininet2.pingLong(
1411 src=main.params[ 'PING' ][ 'source6' ],
1412 target=main.params[ 'PING' ][ 'target6' ],
1413 pingTime=500 )
1414 main.Mininet2.pingLong(
1415 src=main.params[ 'PING' ][ 'source7' ],
1416 target=main.params[ 'PING' ][ 'target7' ],
1417 pingTime=500 )
1418 main.Mininet2.pingLong(
1419 src=main.params[ 'PING' ][ 'source8' ],
1420 target=main.params[ 'PING' ][ 'target8' ],
1421 pingTime=500 )
1422 main.Mininet2.pingLong(
1423 src=main.params[ 'PING' ][ 'source9' ],
1424 target=main.params[ 'PING' ][ 'target9' ],
1425 pingTime=500 )
1426 main.Mininet2.pingLong(
1427 src=main.params[ 'PING' ][ 'source10' ],
1428 target=main.params[ 'PING' ][ 'target10' ],
1429 pingTime=500 )
1430
1431 main.step( "Collecting topology information from ONOS" )
1432 devices = []
1433 threads = []
1434 for i in range( main.numCtrls ):
1435 t = main.Thread( target=main.CLIs[i].devices,
1436 name="devices-" + str( i ),
1437 args=[ ] )
1438 threads.append( t )
1439 t.start()
1440
1441 for t in threads:
1442 t.join()
1443 devices.append( t.result )
1444 hosts = []
1445 threads = []
1446 for i in range( main.numCtrls ):
1447 t = main.Thread( target=main.CLIs[i].hosts,
1448 name="hosts-" + str( i ),
1449 args=[ ] )
1450 threads.append( t )
1451 t.start()
1452
1453 for t in threads:
1454 t.join()
1455 try:
1456 hosts.append( json.loads( t.result ) )
1457 except ( ValueError, TypeError ):
1458
1459
1460 main.log.exception( "Error parsing json output of hosts" )
1461
1462 hosts.append( None )
1463
1464 ports = []
1465 threads = []
1466 for i in range( main.numCtrls ):
1467 t = main.Thread( target=main.CLIs[i].ports,
1468 name="ports-" + str( i ),
1469 args=[ ] )
1470 threads.append( t )
1471 t.start()
1472
1473 for t in threads:
1474 t.join()
1475 ports.append( t.result )
1476 links = []
1477 threads = []
1478 for i in range( main.numCtrls ):
1479 t = main.Thread( target=main.CLIs[i].links,
1480 name="links-" + str( i ),
1481 args=[ ] )
1482 threads.append( t )
1483 t.start()
1484
1485 for t in threads:
1486 t.join()
1487 links.append( t.result )
1488 clusters = []
1489 threads = []
1490 for i in range( main.numCtrls ):
1491 t = main.Thread( target=main.CLIs[i].clusters,
1492 name="clusters-" + str( i ),
1493 args=[ ] )
1494 threads.append( t )
1495 t.start()
1496
1497 for t in threads:
1498 t.join()
1499 clusters.append( t.result )
1500
1501
1502
1503 main.step( "Host view is consistent across ONOS nodes" )
1504 consistentHostsResult = main.TRUE
1505 for controller in range( len( hosts ) ):
1506 controllerStr = str( controller + 1 )
1507 if "Error" not in hosts[ controller ]:
1508 if hosts[ controller ] == hosts[ 0 ]:
1509 continue
1510 else:
1511 main.log.error( "hosts from ONOS" +
1512 controllerStr +
1513 " is inconsistent with ONOS1" )
1514 main.log.warn( repr( hosts[ controller ] ) )
1515 consistentHostsResult = main.FALSE
1516
1517 else:
1518 main.log.error( "Error in getting ONOS hosts from ONOS" +
1519 controllerStr )
1520 consistentHostsResult = main.FALSE
1521 main.log.warn( "ONOS" + controllerStr +
1522 " hosts response: " +
1523 repr( hosts[ controller ] ) )
1524 utilities.assert_equals(
1525 expect=main.TRUE,
1526 actual=consistentHostsResult,
1527 onpass="Hosts view is consistent across all ONOS nodes",
1528 onfail="ONOS nodes have different views of hosts" )
1529
1530 main.step( "Each host has an IP address" )
1531 ipResult = main.TRUE
1532 for controller in range( 0, len( hosts ) ):
1533 controllerStr = str( controller + 1 )
1534 for host in hosts[ controller ]:
1535 if not host.get( 'ipAddresses', [ ] ):
1536 main.log.error( "DEBUG:Error with host ips on controller" +
1537 controllerStr + ": " + str( host ) )
1538 ipResult = main.FALSE
1539 utilities.assert_equals(
1540 expect=main.TRUE,
1541 actual=ipResult,
1542 onpass="The ips of the hosts aren't empty",
1543 onfail="The ip of at least one host is missing" )
1544
1545
1546 main.step( "Cluster view is consistent across ONOS nodes" )
1547 consistentClustersResult = main.TRUE
1548 for controller in range( len( clusters ) ):
1549 controllerStr = str( controller + 1 )
1550 if "Error" not in clusters[ controller ]:
1551 if clusters[ controller ] == clusters[ 0 ]:
1552 continue
1553 else:
1554 main.log.error( "clusters from ONOS" + controllerStr +
1555 " is inconsistent with ONOS1" )
1556 consistentClustersResult = main.FALSE
1557
1558 else:
1559 main.log.error( "Error in getting dataplane clusters " +
1560 "from ONOS" + controllerStr )
1561 consistentClustersResult = main.FALSE
1562 main.log.warn( "ONOS" + controllerStr +
1563 " clusters response: " +
1564 repr( clusters[ controller ] ) )
1565 utilities.assert_equals(
1566 expect=main.TRUE,
1567 actual=consistentClustersResult,
1568 onpass="Clusters view is consistent across all ONOS nodes",
1569 onfail="ONOS nodes have different views of clusters" )
1570
1571 main.step( "Cluster view correct across ONOS nodes" )
1572 try:
1573 numClusters = len( json.loads( clusters[ 0 ] ) )
1574 except ( ValueError, TypeError ):
1575 main.log.exception( "Error parsing clusters[0]: " +
1576 repr( clusters[ 0 ] ) )
1577 clusterResults = main.FALSE
1578 if numClusters == 1:
1579 clusterResults = main.TRUE
1580 utilities.assert_equals(
1581 expect=1,
1582 actual=numClusters,
1583 onpass="ONOS shows 1 SCC",
1584 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1585
1586 main.step( "Comparing ONOS topology to MN" )
1587 devicesResults = main.TRUE
1588 linksResults = main.TRUE
1589 hostsResults = main.TRUE
1590 mnSwitches = main.Mininet1.getSwitches()
1591 mnLinks = main.Mininet1.getLinks()
1592 mnHosts = main.Mininet1.getHosts()
1593 for controller in range( main.numCtrls ):
1594 controllerStr = str( controller + 1 )
1595 if devices[ controller ] and ports[ controller ] and\
1596 "Error" not in devices[ controller ] and\
1597 "Error" not in ports[ controller ]:
1598
1599 currentDevicesResult = main.Mininet1.compareSwitches(
1600 mnSwitches,
1601 json.loads( devices[ controller ] ),
1602 json.loads( ports[ controller ] ) )
1603 else:
1604 currentDevicesResult = main.FALSE
1605 utilities.assert_equals( expect=main.TRUE,
1606 actual=currentDevicesResult,
1607 onpass="ONOS" + controllerStr +
1608 " Switches view is correct",
1609 onfail="ONOS" + controllerStr +
1610 " Switches view is incorrect" )
1611 if links[ controller ] and "Error" not in links[ controller ]:
1612 currentLinksResult = main.Mininet1.compareLinks(
1613 mnSwitches, mnLinks,
1614 json.loads( links[ controller ] ) )
1615 else:
1616 currentLinksResult = main.FALSE
1617 utilities.assert_equals( expect=main.TRUE,
1618 actual=currentLinksResult,
1619 onpass="ONOS" + controllerStr +
1620 " links view is correct",
1621 onfail="ONOS" + controllerStr +
1622 " links view is incorrect" )
1623
1624 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1625 currentHostsResult = main.Mininet1.compareHosts(
1626 mnHosts,
1627 hosts[ controller ] )
1628 else:
1629 currentHostsResult = main.FALSE
1630 utilities.assert_equals( expect=main.TRUE,
1631 actual=currentHostsResult,
1632 onpass="ONOS" + controllerStr +
1633 " hosts exist in Mininet",
1634 onfail="ONOS" + controllerStr +
1635 " hosts don't match Mininet" )
1636
1637 devicesResults = devicesResults and currentDevicesResult
1638 linksResults = linksResults and currentLinksResult
1639 hostsResults = hostsResults and currentHostsResult
1640
1641 main.step( "Device information is correct" )
1642 utilities.assert_equals(
1643 expect=main.TRUE,
1644 actual=devicesResults,
1645 onpass="Device information is correct",
1646 onfail="Device information is incorrect" )
1647
1648 main.step( "Links are correct" )
1649 utilities.assert_equals(
1650 expect=main.TRUE,
1651 actual=linksResults,
1652 onpass="Link are correct",
1653 onfail="Links are incorrect" )
1654
1655 main.step( "Hosts are correct" )
1656 utilities.assert_equals(
1657 expect=main.TRUE,
1658 actual=hostsResults,
1659 onpass="Hosts are correct",
1660 onfail="Hosts are incorrect" )
1661
1662 - def CASE6( self, main ):
1663 """
1664 The Failure case.
1665 """
1666 import time
1667 assert main.numCtrls, "main.numCtrls not defined"
1668 assert main, "main not defined"
1669 assert utilities.assert_equals, "utilities.assert_equals not defined"
1670 assert main.CLIs, "main.CLIs not defined"
1671 assert main.nodes, "main.nodes not defined"
1672 main.case( "Restart minority of ONOS nodes" )
1673 main.step( "Killing 3 ONOS nodes" )
1674 killTime = time.time()
1675
1676
1677 killResults = main.ONOSbench.onosKill( main.nodes[0].ip_address )
1678 time.sleep( 10 )
1679 killResults = killResults and\
1680 main.ONOSbench.onosKill( main.nodes[1].ip_address )
1681 time.sleep( 10 )
1682 killResults = killResults and\
1683 main.ONOSbench.onosKill( main.nodes[2].ip_address )
1684 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1685 onpass="ONOS Killed successfully",
1686 onfail="ONOS kill NOT successful" )
1687
1688 main.step( "Checking if ONOS is up yet" )
1689 count = 0
1690 onosIsupResult = main.FALSE
1691 while onosIsupResult == main.FALSE and count < 10:
1692 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
1693 onos2Isup = main.ONOSbench.isup( main.nodes[1].ip_address )
1694 onos3Isup = main.ONOSbench.isup( main.nodes[2].ip_address )
1695 onosIsupResult = onos1Isup and onos2Isup and onos3Isup
1696 count = count + 1
1697
1698 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1699 onpass="ONOS restarted successfully",
1700 onfail="ONOS restart NOT successful" )
1701
1702 main.step( "Restarting ONOS main.CLIs" )
1703 cliResult1 = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
1704 cliResult2 = main.ONOScli2.startOnosCli( main.nodes[1].ip_address )
1705 cliResult3 = main.ONOScli3.startOnosCli( main.nodes[2].ip_address )
1706 cliResults = cliResult1 and cliResult2 and cliResult3
1707 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1708 onpass="ONOS cli restarted",
1709 onfail="ONOS cli did not restart" )
1710
1711
1712
1713 main.restartTime = time.time() - killTime
1714 main.log.debug( "Restart time: " + str( main.restartTime ) )
1715 '''
1716 # FIXME: revisit test plan for election with madan
1717 # Rerun for election on restarted nodes
1718 run1 = main.CLIs[0].electionTestRun()
1719 run2 = main.CLIs[1].electionTestRun()
1720 run3 = main.CLIs[2].electionTestRun()
1721 runResults = run1 and run2 and run3
1722 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1723 onpass="Reran for election",
1724 onfail="Failed to rerun for election" )
1725 '''
1726
1727 time.sleep( 60 )
1728 main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) )
1729 main.log.debug( main.CLIs[0].leaders( jsonFormat=False ) )
1730 main.log.debug( main.CLIs[0].partitions( jsonFormat=False ) )
1731
1732 - def CASE7( self, main ):
1733 """
1734 Check state after ONOS failure
1735 """
1736 import json
1737 assert main.numCtrls, "main.numCtrls not defined"
1738 assert main, "main not defined"
1739 assert utilities.assert_equals, "utilities.assert_equals not defined"
1740 assert main.CLIs, "main.CLIs not defined"
1741 assert main.nodes, "main.nodes not defined"
1742 main.case( "Running ONOS Constant State Tests" )
1743
1744 main.step( "Check that each switch has a master" )
1745
1746 rolesNotNull = main.TRUE
1747 threads = []
1748 for i in range( main.numCtrls ):
1749 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1750 name="rolesNotNull-" + str( i ),
1751 args=[ ] )
1752 threads.append( t )
1753 t.start()
1754
1755 for t in threads:
1756 t.join()
1757 rolesNotNull = rolesNotNull and t.result
1758 utilities.assert_equals(
1759 expect=main.TRUE,
1760 actual=rolesNotNull,
1761 onpass="Each device has a master",
1762 onfail="Some devices don't have a master assigned" )
1763
1764 main.step( "Read device roles from ONOS" )
1765 ONOSMastership = []
1766 consistentMastership = True
1767 rolesResults = True
1768 threads = []
1769 for i in range( main.numCtrls ):
1770 t = main.Thread( target=main.CLIs[i].roles,
1771 name="roles-" + str( i ),
1772 args=[] )
1773 threads.append( t )
1774 t.start()
1775
1776 for t in threads:
1777 t.join()
1778 ONOSMastership.append( t.result )
1779
1780 for i in range( main.numCtrls ):
1781 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1782 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1783 " roles" )
1784 main.log.warn(
1785 "ONOS" + str( i + 1 ) + " mastership response: " +
1786 repr( ONOSMastership[i] ) )
1787 rolesResults = False
1788 utilities.assert_equals(
1789 expect=True,
1790 actual=rolesResults,
1791 onpass="No error in reading roles output",
1792 onfail="Error in reading roles from ONOS" )
1793
1794 main.step( "Check for consistency in roles from each controller" )
1795 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1796 main.log.info(
1797 "Switch roles are consistent across all ONOS nodes" )
1798 else:
1799 consistentMastership = False
1800 utilities.assert_equals(
1801 expect=True,
1802 actual=consistentMastership,
1803 onpass="Switch roles are consistent across all ONOS nodes",
1804 onfail="ONOS nodes have different views of switch roles" )
1805
1806 if rolesResults and not consistentMastership:
1807 for i in range( main.numCtrls ):
1808 main.log.warn(
1809 "ONOS" + str( i + 1 ) + " roles: ",
1810 json.dumps(
1811 json.loads( ONOSMastership[ i ] ),
1812 sort_keys=True,
1813 indent=4,
1814 separators=( ',', ': ' ) ) )
1815
1816
1817 '''
1818 description2 = "Compare switch roles from before failure"
1819 main.step( description2 )
1820 try:
1821 currentJson = json.loads( ONOSMastership[0] )
1822 oldJson = json.loads( mastershipState )
1823 except ( ValueError, TypeError ):
1824 main.log.exception( "Something is wrong with parsing " +
1825 "ONOSMastership[0] or mastershipState" )
1826 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1827 main.log.error( "mastershipState" + repr( mastershipState ) )
1828 main.cleanup()
1829 main.exit()
1830 mastershipCheck = main.TRUE
1831 for i in range( 1, 29 ):
1832 switchDPID = str(
1833 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1834 current = [ switch[ 'master' ] for switch in currentJson
1835 if switchDPID in switch[ 'id' ] ]
1836 old = [ switch[ 'master' ] for switch in oldJson
1837 if switchDPID in switch[ 'id' ] ]
1838 if current == old:
1839 mastershipCheck = mastershipCheck and main.TRUE
1840 else:
1841 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1842 mastershipCheck = main.FALSE
1843 utilities.assert_equals(
1844 expect=main.TRUE,
1845 actual=mastershipCheck,
1846 onpass="Mastership of Switches was not changed",
1847 onfail="Mastership of some switches changed" )
1848 '''
1849
1850 main.step( "Get the intents and compare across all nodes" )
1851 ONOSIntents = []
1852 intentCheck = main.FALSE
1853 consistentIntents = True
1854 intentsResults = True
1855 threads = []
1856 for i in range( main.numCtrls ):
1857 t = main.Thread( target=main.CLIs[i].intents,
1858 name="intents-" + str( i ),
1859 args=[],
1860 kwargs={ 'jsonFormat': True } )
1861 threads.append( t )
1862 t.start()
1863
1864 for t in threads:
1865 t.join()
1866 ONOSIntents.append( t.result )
1867
1868 for i in range( main.numCtrls ):
1869 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1870 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1871 " intents" )
1872 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1873 repr( ONOSIntents[ i ] ) )
1874 intentsResults = False
1875 utilities.assert_equals(
1876 expect=True,
1877 actual=intentsResults,
1878 onpass="No error in reading intents output",
1879 onfail="Error in reading intents from ONOS" )
1880
1881 main.step( "Check for consistency in Intents from each controller" )
1882 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1883 main.log.info( "Intents are consistent across all ONOS " +
1884 "nodes" )
1885 else:
1886 consistentIntents = False
1887
1888
1889
1890
1891
1892
1893
1894 title = " ID"
1895 for n in range( main.numCtrls ):
1896 title += " " * 10 + "ONOS" + str( n + 1 )
1897 main.log.warn( title )
1898
1899 keys = []
1900 for nodeStr in ONOSIntents:
1901 node = json.loads( nodeStr )
1902 for intent in node:
1903 keys.append( intent.get( 'id' ) )
1904 keys = set( keys )
1905 for key in keys:
1906 row = "%-13s" % key
1907 for nodeStr in ONOSIntents:
1908 node = json.loads( nodeStr )
1909 for intent in node:
1910 if intent.get( 'id' ) == key:
1911 row += "%-15s" % intent.get( 'state' )
1912 main.log.warn( row )
1913
1914
1915 utilities.assert_equals(
1916 expect=True,
1917 actual=consistentIntents,
1918 onpass="Intents are consistent across all ONOS nodes",
1919 onfail="ONOS nodes have different views of intents" )
1920 intentStates = []
1921 for node in ONOSIntents:
1922 nodeStates = []
1923
1924 try:
1925 for intent in json.loads( node ):
1926 nodeStates.append( intent[ 'state' ] )
1927 except ( ValueError, TypeError ):
1928 main.log.exception( "Error in parsing intents" )
1929 main.log.error( repr( node ) )
1930 intentStates.append( nodeStates )
1931 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1932 main.log.info( dict( out ) )
1933
1934 if intentsResults and not consistentIntents:
1935 for i in range( main.numCtrls ):
1936 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1937 main.log.warn( json.dumps(
1938 json.loads( ONOSIntents[ i ] ),
1939 sort_keys=True,
1940 indent=4,
1941 separators=( ',', ': ' ) ) )
1942 elif intentsResults and consistentIntents:
1943 intentCheck = main.TRUE
1944
1945
1946
1947 main.step( "Compare current intents with intents before the failure" )
1948
1949
1950 sameIntents = main.FALSE
1951 if intentState and intentState == ONOSIntents[ 0 ]:
1952 sameIntents = main.TRUE
1953 main.log.info( "Intents are consistent with before failure" )
1954
1955
1956 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1957 sameIntents = main.TRUE
1958 try:
1959 before = json.loads( intentState )
1960 after = json.loads( ONOSIntents[ 0 ] )
1961 for intent in before:
1962 if intent not in after:
1963 sameIntents = main.FALSE
1964 main.log.debug( "Intent is not currently in ONOS " +
1965 "(at least in the same form):" )
1966 main.log.debug( json.dumps( intent ) )
1967 except ( ValueError, TypeError ):
1968 main.log.exception( "Exception printing intents" )
1969 main.log.debug( repr( ONOSIntents[0] ) )
1970 main.log.debug( repr( intentState ) )
1971 if sameIntents == main.FALSE:
1972 try:
1973 main.log.debug( "ONOS intents before: " )
1974 main.log.debug( json.dumps( json.loads( intentState ),
1975 sort_keys=True, indent=4,
1976 separators=( ',', ': ' ) ) )
1977 main.log.debug( "Current ONOS intents: " )
1978 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1979 sort_keys=True, indent=4,
1980 separators=( ',', ': ' ) ) )
1981 except ( ValueError, TypeError ):
1982 main.log.exception( "Exception printing intents" )
1983 main.log.debug( repr( ONOSIntents[0] ) )
1984 main.log.debug( repr( intentState ) )
1985 utilities.assert_equals(
1986 expect=main.TRUE,
1987 actual=sameIntents,
1988 onpass="Intents are consistent with before failure",
1989 onfail="The Intents changed during failure" )
1990 intentCheck = intentCheck and sameIntents
1991
1992 main.step( "Get the OF Table entries and compare to before " +
1993 "component failure" )
1994 FlowTables = main.TRUE
1995 flows2 = []
1996 for i in range( 28 ):
1997 main.log.info( "Checking flow table on s" + str( i + 1 ) )
1998 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
1999 flows2.append( tmpFlows )
2000 tempResult = main.Mininet1.flowComp(
2001 flow1=flows[ i ],
2002 flow2=tmpFlows )
2003 FlowTables = FlowTables and tempResult
2004 if FlowTables == main.FALSE:
2005 main.log.info( "Differences in flow table for switch: s" +
2006 str( i + 1 ) )
2007 utilities.assert_equals(
2008 expect=main.TRUE,
2009 actual=FlowTables,
2010 onpass="No changes were found in the flow tables",
2011 onfail="Changes were found in the flow tables" )
2012
2013 main.Mininet2.pingLongKill()
2014 '''
2015 main.step( "Check the continuous pings to ensure that no packets " +
2016 "were dropped during component failure" )
2017 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2018 main.params[ 'TESTONIP' ] )
2019 LossInPings = main.FALSE
2020 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2021 for i in range( 8, 18 ):
2022 main.log.info(
2023 "Checking for a loss in pings along flow from s" +
2024 str( i ) )
2025 LossInPings = main.Mininet2.checkForLoss(
2026 "/tmp/ping.h" +
2027 str( i ) ) or LossInPings
2028 if LossInPings == main.TRUE:
2029 main.log.info( "Loss in ping detected" )
2030 elif LossInPings == main.ERROR:
2031 main.log.info( "There are multiple mininet process running" )
2032 elif LossInPings == main.FALSE:
2033 main.log.info( "No Loss in the pings" )
2034 main.log.info( "No loss of dataplane connectivity" )
2035 utilities.assert_equals(
2036 expect=main.FALSE,
2037 actual=LossInPings,
2038 onpass="No Loss of connectivity",
2039 onfail="Loss of dataplane connectivity detected" )
2040 '''
2041
2042 main.step( "Leadership Election is still functional" )
2043
2044 leaderList = []
2045
2046 restarted = [ main.nodes[0].ip_address, main.nodes[1].ip_address,
2047 main.nodes[2].ip_address ]
2048
2049 leaderResult = main.TRUE
2050 for cli in main.CLIs:
2051 leaderN = cli.electionTestLeader()
2052 leaderList.append( leaderN )
2053 if leaderN == main.FALSE:
2054
2055 main.log.error( "Something is wrong with " +
2056 "electionTestLeader function, check the" +
2057 " error logs" )
2058 leaderResult = main.FALSE
2059 elif leaderN is None:
2060 main.log.error( cli.name +
2061 " shows no leader for the election-app was" +
2062 " elected after the old one died" )
2063 leaderResult = main.FALSE
2064 elif leaderN in restarted:
2065 main.log.error( cli.name + " shows " + str( leaderN ) +
2066 " as leader for the election-app, but it " +
2067 "was restarted" )
2068 leaderResult = main.FALSE
2069 if len( set( leaderList ) ) != 1:
2070 leaderResult = main.FALSE
2071 main.log.error(
2072 "Inconsistent view of leader for the election test app" )
2073
2074 utilities.assert_equals(
2075 expect=main.TRUE,
2076 actual=leaderResult,
2077 onpass="Leadership election passed",
2078 onfail="Something went wrong with Leadership election" )
2079
2080 - def CASE8( self, main ):
2081 """
2082 Compare topo
2083 """
2084 import json
2085 import time
2086 assert main.numCtrls, "main.numCtrls not defined"
2087 assert main, "main not defined"
2088 assert utilities.assert_equals, "utilities.assert_equals not defined"
2089 assert main.CLIs, "main.CLIs not defined"
2090 assert main.nodes, "main.nodes not defined"
2091
2092 main.case( "Compare ONOS Topology view to Mininet topology" )
2093 main.caseExplanation = "Compare topology objects between Mininet" +\
2094 " and ONOS"
2095
2096 main.step( "Comparing ONOS topology to MN" )
2097 devicesResults = main.TRUE
2098 linksResults = main.TRUE
2099 hostsResults = main.TRUE
2100 hostAttachmentResults = True
2101 topoResult = main.FALSE
2102 elapsed = 0
2103 count = 0
2104 main.step( "Collecting topology information from ONOS" )
2105 startTime = time.time()
2106
2107 while topoResult == main.FALSE and elapsed < 60:
2108 count += 1
2109 cliStart = time.time()
2110 devices = []
2111 threads = []
2112 for i in range( main.numCtrls ):
2113 t = main.Thread( target=main.CLIs[i].devices,
2114 name="devices-" + str( i ),
2115 args=[ ] )
2116 threads.append( t )
2117 t.start()
2118
2119 for t in threads:
2120 t.join()
2121 devices.append( t.result )
2122 hosts = []
2123 ipResult = main.TRUE
2124 threads = []
2125 for i in range( main.numCtrls ):
2126 t = main.Thread( target=main.CLIs[i].hosts,
2127 name="hosts-" + str( i ),
2128 args=[ ] )
2129 threads.append( t )
2130 t.start()
2131
2132 for t in threads:
2133 t.join()
2134 try:
2135 hosts.append( json.loads( t.result ) )
2136 except ( ValueError, TypeError ):
2137 main.log.exception( "Error parsing hosts results" )
2138 main.log.error( repr( t.result ) )
2139 for controller in range( 0, len( hosts ) ):
2140 controllerStr = str( controller + 1 )
2141 for host in hosts[ controller ]:
2142 if host is None or host.get( 'ipAddresses', [] ) == []:
2143 main.log.error(
2144 "DEBUG:Error with host ipAddresses on controller" +
2145 controllerStr + ": " + str( host ) )
2146 ipResult = main.FALSE
2147 ports = []
2148 threads = []
2149 for i in range( main.numCtrls ):
2150 t = main.Thread( target=main.CLIs[i].ports,
2151 name="ports-" + str( i ),
2152 args=[ ] )
2153 threads.append( t )
2154 t.start()
2155
2156 for t in threads:
2157 t.join()
2158 ports.append( t.result )
2159 links = []
2160 threads = []
2161 for i in range( main.numCtrls ):
2162 t = main.Thread( target=main.CLIs[i].links,
2163 name="links-" + str( i ),
2164 args=[ ] )
2165 threads.append( t )
2166 t.start()
2167
2168 for t in threads:
2169 t.join()
2170 links.append( t.result )
2171 clusters = []
2172 threads = []
2173 for i in range( main.numCtrls ):
2174 t = main.Thread( target=main.CLIs[i].clusters,
2175 name="clusters-" + str( i ),
2176 args=[ ] )
2177 threads.append( t )
2178 t.start()
2179
2180 for t in threads:
2181 t.join()
2182 clusters.append( t.result )
2183
2184 elapsed = time.time() - startTime
2185 cliTime = time.time() - cliStart
2186 print "Elapsed time: " + str( elapsed )
2187 print "CLI time: " + str( cliTime )
2188
2189 mnSwitches = main.Mininet1.getSwitches()
2190 mnLinks = main.Mininet1.getLinks()
2191 mnHosts = main.Mininet1.getHosts()
2192 for controller in range( main.numCtrls ):
2193 controllerStr = str( controller + 1 )
2194 if devices[ controller ] and ports[ controller ] and\
2195 "Error" not in devices[ controller ] and\
2196 "Error" not in ports[ controller ]:
2197
2198 currentDevicesResult = main.Mininet1.compareSwitches(
2199 mnSwitches,
2200 json.loads( devices[ controller ] ),
2201 json.loads( ports[ controller ] ) )
2202 else:
2203 currentDevicesResult = main.FALSE
2204 utilities.assert_equals( expect=main.TRUE,
2205 actual=currentDevicesResult,
2206 onpass="ONOS" + controllerStr +
2207 " Switches view is correct",
2208 onfail="ONOS" + controllerStr +
2209 " Switches view is incorrect" )
2210
2211 if links[ controller ] and "Error" not in links[ controller ]:
2212 currentLinksResult = main.Mininet1.compareLinks(
2213 mnSwitches, mnLinks,
2214 json.loads( links[ controller ] ) )
2215 else:
2216 currentLinksResult = main.FALSE
2217 utilities.assert_equals( expect=main.TRUE,
2218 actual=currentLinksResult,
2219 onpass="ONOS" + controllerStr +
2220 " links view is correct",
2221 onfail="ONOS" + controllerStr +
2222 " links view is incorrect" )
2223
2224 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2225 currentHostsResult = main.Mininet1.compareHosts(
2226 mnHosts,
2227 hosts[ controller ] )
2228 else:
2229 currentHostsResult = main.FALSE
2230 utilities.assert_equals( expect=main.TRUE,
2231 actual=currentHostsResult,
2232 onpass="ONOS" + controllerStr +
2233 " hosts exist in Mininet",
2234 onfail="ONOS" + controllerStr +
2235 " hosts don't match Mininet" )
2236
2237 hostAttachment = True
2238 zeroHosts = False
2239
2240
2241 mappings = {}
2242 for i in range( 1, 29 ):
2243
2244 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2245 if i == 1:
2246 deviceId = "1000".zfill(16)
2247 elif i == 2:
2248 deviceId = "2000".zfill(16)
2249 elif i == 3:
2250 deviceId = "3000".zfill(16)
2251 elif i == 4:
2252 deviceId = "3004".zfill(16)
2253 elif i == 5:
2254 deviceId = "5000".zfill(16)
2255 elif i == 6:
2256 deviceId = "6000".zfill(16)
2257 elif i == 7:
2258 deviceId = "6007".zfill(16)
2259 elif i >= 8 and i <= 17:
2260 dpid = '3' + str( i ).zfill( 3 )
2261 deviceId = dpid.zfill(16)
2262 elif i >= 18 and i <= 27:
2263 dpid = '6' + str( i ).zfill( 3 )
2264 deviceId = dpid.zfill(16)
2265 elif i == 28:
2266 deviceId = "2800".zfill(16)
2267 mappings[ macId ] = deviceId
2268 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2269 if hosts[ controller ] == []:
2270 main.log.warn( "There are no hosts discovered" )
2271 zeroHosts = True
2272 else:
2273 for host in hosts[ controller ]:
2274 mac = None
2275 location = None
2276 device = None
2277 port = None
2278 try:
2279 mac = host.get( 'mac' )
2280 assert mac, "mac field could not be found for this host object"
2281
2282 location = host.get( 'location' )
2283 assert location, "location field could not be found for this host object"
2284
2285
2286 device = str( location.get( 'elementId' ) ).split(':')[1]
2287 assert device, "elementId field could not be found for this host location object"
2288
2289 port = location.get( 'port' )
2290 assert port, "port field could not be found for this host location object"
2291
2292
2293 if mac and device and port:
2294 if str( port ) != "1":
2295 main.log.error( "The attachment port is incorrect for " +
2296 "host " + str( mac ) +
2297 ". Expected: 1 Actual: " + str( port) )
2298 hostAttachment = False
2299 if device != mappings[ str( mac ) ]:
2300 main.log.error( "The attachment device is incorrect for " +
2301 "host " + str( mac ) +
2302 ". Expected: " + mappings[ str( mac ) ] +
2303 " Actual: " + device )
2304 hostAttachment = False
2305 else:
2306 hostAttachment = False
2307 except AssertionError:
2308 main.log.exception( "Json object not as expected" )
2309 main.log.error( repr( host ) )
2310 hostAttachment = False
2311 else:
2312 main.log.error( "No hosts json output or \"Error\"" +
2313 " in output. hosts = " +
2314 repr( hosts[ controller ] ) )
2315 if zeroHosts is False:
2316 hostAttachment = True
2317
2318
2319 devicesResults = devicesResults and currentDevicesResult
2320 linksResults = linksResults and currentLinksResult
2321 hostsResults = hostsResults and currentHostsResult
2322 hostAttachmentResults = hostAttachmentResults and\
2323 hostAttachment
2324
2325
2326
2327
2328 main.step( "Hosts view is consistent across all ONOS nodes" )
2329 consistentHostsResult = main.TRUE
2330 for controller in range( len( hosts ) ):
2331 controllerStr = str( controller + 1 )
2332 if "Error" not in hosts[ controller ]:
2333 if hosts[ controller ] == hosts[ 0 ]:
2334 continue
2335 else:
2336 main.log.error( "hosts from ONOS" + controllerStr +
2337 " is inconsistent with ONOS1" )
2338 main.log.warn( repr( hosts[ controller ] ) )
2339 consistentHostsResult = main.FALSE
2340
2341 else:
2342 main.log.error( "Error in getting ONOS hosts from ONOS" +
2343 controllerStr )
2344 consistentHostsResult = main.FALSE
2345 main.log.warn( "ONOS" + controllerStr +
2346 " hosts response: " +
2347 repr( hosts[ controller ] ) )
2348 utilities.assert_equals(
2349 expect=main.TRUE,
2350 actual=consistentHostsResult,
2351 onpass="Hosts view is consistent across all ONOS nodes",
2352 onfail="ONOS nodes have different views of hosts" )
2353
2354 main.step( "Hosts information is correct" )
2355 hostsResults = hostsResults and ipResult
2356 utilities.assert_equals(
2357 expect=main.TRUE,
2358 actual=hostsResults,
2359 onpass="Host information is correct",
2360 onfail="Host information is incorrect" )
2361
2362 main.step( "Host attachment points to the network" )
2363 utilities.assert_equals(
2364 expect=True,
2365 actual=hostAttachmentResults,
2366 onpass="Hosts are correctly attached to the network",
2367 onfail="ONOS did not correctly attach hosts to the network" )
2368
2369
2370 main.step( "Clusters view is consistent across all ONOS nodes" )
2371 consistentClustersResult = main.TRUE
2372 for controller in range( len( clusters ) ):
2373 controllerStr = str( controller + 1 )
2374 if "Error" not in clusters[ controller ]:
2375 if clusters[ controller ] == clusters[ 0 ]:
2376 continue
2377 else:
2378 main.log.error( "clusters from ONOS" +
2379 controllerStr +
2380 " is inconsistent with ONOS1" )
2381 consistentClustersResult = main.FALSE
2382
2383 else:
2384 main.log.error( "Error in getting dataplane clusters " +
2385 "from ONOS" + controllerStr )
2386 consistentClustersResult = main.FALSE
2387 main.log.warn( "ONOS" + controllerStr +
2388 " clusters response: " +
2389 repr( clusters[ controller ] ) )
2390 utilities.assert_equals(
2391 expect=main.TRUE,
2392 actual=consistentClustersResult,
2393 onpass="Clusters view is consistent across all ONOS nodes",
2394 onfail="ONOS nodes have different views of clusters" )
2395
2396 main.step( "There is only one SCC" )
2397
2398 try:
2399 numClusters = len( json.loads( clusters[ 0 ] ) )
2400 except ( ValueError, TypeError ):
2401 main.log.exception( "Error parsing clusters[0]: " +
2402 repr( clusters[0] ) )
2403 clusterResults = main.FALSE
2404 if numClusters == 1:
2405 clusterResults = main.TRUE
2406 utilities.assert_equals(
2407 expect=1,
2408 actual=numClusters,
2409 onpass="ONOS shows 1 SCC",
2410 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2411
2412 topoResult = ( devicesResults and linksResults
2413 and hostsResults and consistentHostsResult
2414 and consistentClustersResult and clusterResults
2415 and ipResult and hostAttachmentResults )
2416
2417 topoResult = topoResult and int( count <= 2 )
2418 note = "note it takes about " + str( int( cliTime ) ) + \
2419 " seconds for the test to make all the cli calls to fetch " +\
2420 "the topology from each ONOS instance"
2421 main.log.info(
2422 "Very crass estimate for topology discovery/convergence( " +
2423 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2424 str( count ) + " tries" )
2425
2426 main.step( "Device information is correct" )
2427 utilities.assert_equals(
2428 expect=main.TRUE,
2429 actual=devicesResults,
2430 onpass="Device information is correct",
2431 onfail="Device information is incorrect" )
2432
2433 main.step( "Links are correct" )
2434 utilities.assert_equals(
2435 expect=main.TRUE,
2436 actual=linksResults,
2437 onpass="Link are correct",
2438 onfail="Links are incorrect" )
2439
2440
2441 main.step( "Checking ONOS nodes" )
2442 nodesOutput = []
2443 nodeResults = main.TRUE
2444 threads = []
2445 for i in range( main.numCtrls ):
2446 t = main.Thread( target=main.CLIs[i].nodes,
2447 name="nodes-" + str( i ),
2448 args=[ ] )
2449 threads.append( t )
2450 t.start()
2451
2452 for t in threads:
2453 t.join()
2454 nodesOutput.append( t.result )
2455 ips = [ node.ip_address for node in main.nodes ]
2456 for i in nodesOutput:
2457 try:
2458 current = json.loads( i )
2459 for node in current:
2460 currentResult = main.FALSE
2461 if node['ip'] in ips:
2462 if node['state'] == 'ACTIVE':
2463 currentResult = main.TRUE
2464 else:
2465 main.log.error( "Error in ONOS node availability" )
2466 main.log.error(
2467 json.dumps( current,
2468 sort_keys=True,
2469 indent=4,
2470 separators=( ',', ': ' ) ) )
2471 break
2472 nodeResults = nodeResults and currentResult
2473 except ( ValueError, TypeError ):
2474 main.log.error( "Error parsing nodes output" )
2475 main.log.warn( repr( i ) )
2476 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2477 onpass="Nodes check successful",
2478 onfail="Nodes check NOT successful" )
2479
2480 - def CASE9( self, main ):
2481 """
2482 Link s3-s28 down
2483 """
2484 import time
2485 assert main.numCtrls, "main.numCtrls not defined"
2486 assert main, "main not defined"
2487 assert utilities.assert_equals, "utilities.assert_equals not defined"
2488 assert main.CLIs, "main.CLIs not defined"
2489 assert main.nodes, "main.nodes not defined"
2490
2491
2492 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2493
2494 description = "Turn off a link to ensure that Link Discovery " +\
2495 "is working properly"
2496 main.case( description )
2497
2498 main.step( "Kill Link between s3 and s28" )
2499 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2500 main.log.info( "Waiting " + str( linkSleep ) +
2501 " seconds for link down to be discovered" )
2502 time.sleep( linkSleep )
2503 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2504 onpass="Link down successful",
2505 onfail="Failed to bring link down" )
2506
2507
2509 """
2510 Link s3-s28 up
2511 """
2512 import time
2513 assert main.numCtrls, "main.numCtrls not defined"
2514 assert main, "main not defined"
2515 assert utilities.assert_equals, "utilities.assert_equals not defined"
2516 assert main.CLIs, "main.CLIs not defined"
2517 assert main.nodes, "main.nodes not defined"
2518
2519
2520 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2521
2522 description = "Restore a link to ensure that Link Discovery is " + \
2523 "working properly"
2524 main.case( description )
2525
2526 main.step( "Bring link between s3 and s28 back up" )
2527 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2528 main.log.info( "Waiting " + str( linkSleep ) +
2529 " seconds for link up to be discovered" )
2530 time.sleep( linkSleep )
2531 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2532 onpass="Link up successful",
2533 onfail="Failed to bring link up" )
2534
2535
2537 """
2538 Switch Down
2539 """
2540
2541 import time
2542 assert main.numCtrls, "main.numCtrls not defined"
2543 assert main, "main not defined"
2544 assert utilities.assert_equals, "utilities.assert_equals not defined"
2545 assert main.CLIs, "main.CLIs not defined"
2546 assert main.nodes, "main.nodes not defined"
2547
2548 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2549
2550 description = "Killing a switch to ensure it is discovered correctly"
2551 main.case( description )
2552 switch = main.params[ 'kill' ][ 'switch' ]
2553 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2554
2555
2556 main.step( "Kill " + switch )
2557 main.log.info( "Deleting " + switch )
2558 main.Mininet1.delSwitch( switch )
2559 main.log.info( "Waiting " + str( switchSleep ) +
2560 " seconds for switch down to be discovered" )
2561 time.sleep( switchSleep )
2562 device = main.ONOScli1.getDevice( dpid=switchDPID )
2563
2564 main.log.warn( str( device ) )
2565 result = main.FALSE
2566 if device and device[ 'available' ] is False:
2567 result = main.TRUE
2568 utilities.assert_equals( expect=main.TRUE, actual=result,
2569 onpass="Kill switch successful",
2570 onfail="Failed to kill switch?" )
2571
2573 """
2574 Switch Up
2575 """
2576
2577 import time
2578 assert main.numCtrls, "main.numCtrls not defined"
2579 assert main, "main not defined"
2580 assert utilities.assert_equals, "utilities.assert_equals not defined"
2581 assert main.CLIs, "main.CLIs not defined"
2582 assert main.nodes, "main.nodes not defined"
2583 assert ONOS1Port, "ONOS1Port not defined"
2584 assert ONOS2Port, "ONOS2Port not defined"
2585 assert ONOS3Port, "ONOS3Port not defined"
2586 assert ONOS4Port, "ONOS4Port not defined"
2587 assert ONOS5Port, "ONOS5Port not defined"
2588 assert ONOS6Port, "ONOS6Port not defined"
2589 assert ONOS7Port, "ONOS7Port not defined"
2590
2591 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2592 switch = main.params[ 'kill' ][ 'switch' ]
2593 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2594 links = main.params[ 'kill' ][ 'links' ].split()
2595 description = "Adding a switch to ensure it is discovered correctly"
2596 main.case( description )
2597
2598 main.step( "Add back " + switch )
2599 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2600 for peer in links:
2601 main.Mininet1.addLink( switch, peer )
2602 ipList = []
2603 for i in range( main.numCtrls ):
2604 ipList.append( main.nodes[ i ].ip_address )
2605 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2606 main.log.info( "Waiting " + str( switchSleep ) +
2607 " seconds for switch up to be discovered" )
2608 time.sleep( switchSleep )
2609 device = main.ONOScli1.getDevice( dpid=switchDPID )
2610
2611 main.log.warn( str( device ) )
2612 result = main.FALSE
2613 if device and device[ 'available' ]:
2614 result = main.TRUE
2615 utilities.assert_equals( expect=main.TRUE, actual=result,
2616 onpass="add switch successful",
2617 onfail="Failed to add switch?" )
2618
2620 """
2621 Clean up
2622 """
2623 import os
2624 import time
2625 assert main.numCtrls, "main.numCtrls not defined"
2626 assert main, "main not defined"
2627 assert utilities.assert_equals, "utilities.assert_equals not defined"
2628 assert main.CLIs, "main.CLIs not defined"
2629 assert main.nodes, "main.nodes not defined"
2630
2631
2632 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2633 'blue': '\033[94m', 'green': '\033[92m',
2634 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2635 main.case( "Test Cleanup" )
2636 main.step( "Killing tcpdumps" )
2637 main.Mininet2.stopTcpdump()
2638
2639 testname = main.TEST
2640 if main.params[ 'BACKUP' ] == "True":
2641 main.step( "Copying MN pcap and ONOS log files to test station" )
2642 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2643 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
2644
2645
2646
2647
2648
2649
2650 logFolder = "/opt/onos/log/"
2651 logFiles = [ "karaf.log", "karaf.log.1" ]
2652
2653 dstDir = "~/packet_captures/"
2654 for f in logFiles:
2655 for node in main.nodes:
2656 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2657 ":" + logFolder + f + " " +
2658 teststationUser + "@" +
2659 teststationIP + ":" +
2660 dstDir + str( testname ) +
2661 "-" + node.name + "-" + f )
2662 main.ONOSbench.handle.expect( "\$" )
2663
2664
2665
2666 logFolder = "/opt/onos/var/"
2667 logFiles = [ "stderr.log", "stdout.log" ]
2668
2669 dstDir = "~/packet_captures/"
2670 for f in logFiles:
2671 for node in main.nodes:
2672 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2673 ":" + logFolder + f + " " +
2674 teststationUser + "@" +
2675 teststationIP + ":" +
2676 dstDir + str( testname ) +
2677 "-" + node.name + "-" + f )
2678 main.ONOSbench.handle.expect( "\$" )
2679
2680 time.sleep( 10 )
2681 main.step( "Packing and rotating pcap archives" )
2682 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
2683
2684 main.step( "Stopping Mininet" )
2685 mnResult = main.Mininet1.stopNet()
2686 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2687 onpass="Mininet stopped",
2688 onfail="MN cleanup NOT successful" )
2689
2690 main.step( "Checking ONOS Logs for errors" )
2691 for node in main.nodes:
2692 print colors[ 'purple' ] + "Checking logs for errors on " + \
2693 node.name + ":" + colors[ 'end' ]
2694 print main.ONOSbench.checkLogs( node.ip_address, restart=True )
2695
2696 try:
2697 timerLog = open( main.logdir + "/Timers.csv", 'w')
2698
2699 labels = "Gossip Intents, Restart"
2700 data = str( gossipTime ) + ", " + str( main.restartTime )
2701 timerLog.write( labels + "\n" + data )
2702 timerLog.close()
2703 except NameError, e:
2704 main.log.exception(e)
2705
2707 """
2708 start election app on all onos nodes
2709 """
2710 assert main.numCtrls, "main.numCtrls not defined"
2711 assert main, "main not defined"
2712 assert utilities.assert_equals, "utilities.assert_equals not defined"
2713 assert main.CLIs, "main.CLIs not defined"
2714 assert main.nodes, "main.nodes not defined"
2715
2716 main.case("Start Leadership Election app")
2717 main.step( "Install leadership election app" )
2718 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2719 utilities.assert_equals(
2720 expect=main.TRUE,
2721 actual=appResult,
2722 onpass="Election app installed",
2723 onfail="Something went wrong with installing Leadership election" )
2724
2725 main.step( "Run for election on each node" )
2726 leaderResult = main.TRUE
2727 leaders = []
2728 for cli in main.CLIs:
2729 cli.electionTestRun()
2730 for cli in main.CLIs:
2731 leader = cli.electionTestLeader()
2732 if leader is None or leader == main.FALSE:
2733 main.log.error( cli.name + ": Leader for the election app " +
2734 "should be an ONOS node, instead got '" +
2735 str( leader ) + "'" )
2736 leaderResult = main.FALSE
2737 leaders.append( leader )
2738 utilities.assert_equals(
2739 expect=main.TRUE,
2740 actual=leaderResult,
2741 onpass="Successfully ran for leadership",
2742 onfail="Failed to run for leadership" )
2743
2744 main.step( "Check that each node shows the same leader" )
2745 sameLeader = main.TRUE
2746 if len( set( leaders ) ) != 1:
2747 sameLeader = main.FALSE
2748 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
2749 str( leaders ) )
2750 utilities.assert_equals(
2751 expect=main.TRUE,
2752 actual=sameLeader,
2753 onpass="Leadership is consistent for the election topic",
2754 onfail="Nodes have different leaders" )
2755
2757 """
2758 Check that Leadership Election is still functional
2759 """
2760 import time
2761 assert main.numCtrls, "main.numCtrls not defined"
2762 assert main, "main not defined"
2763 assert utilities.assert_equals, "utilities.assert_equals not defined"
2764 assert main.CLIs, "main.CLIs not defined"
2765 assert main.nodes, "main.nodes not defined"
2766
2767 leaderResult = main.TRUE
2768 description = "Check that Leadership Election is still functional"
2769 main.case( description )
2770
2771 main.step( "Check that each node shows the same leader" )
2772 sameLeader = main.TRUE
2773 leaders = []
2774 for cli in main.CLIs:
2775 leader = cli.electionTestLeader()
2776 leaders.append( leader )
2777 if len( set( leaders ) ) != 1:
2778 sameLeader = main.FALSE
2779 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
2780 str( leaders ) )
2781 utilities.assert_equals(
2782 expect=main.TRUE,
2783 actual=sameLeader,
2784 onpass="Leadership is consistent for the election topic",
2785 onfail="Nodes have different leaders" )
2786
2787 main.step( "Find current leader and withdraw" )
2788 leader = main.ONOScli1.electionTestLeader()
2789
2790 withdrawResult = main.FALSE
2791 if leader is None or leader == main.FALSE:
2792 main.log.error(
2793 "Leader for the election app should be an ONOS node," +
2794 "instead got '" + str( leader ) + "'" )
2795 leaderResult = main.FALSE
2796 oldLeader = None
2797 for i in range( len( main.CLIs ) ):
2798 if leader == main.nodes[ i ].ip_address:
2799 oldLeader = main.CLIs[ i ]
2800 break
2801 else:
2802 main.log.error( "Leader election, could not find current leader" )
2803 if oldLeader:
2804 withdrawResult = oldLeader.electionTestWithdraw()
2805 utilities.assert_equals(
2806 expect=main.TRUE,
2807 actual=withdrawResult,
2808 onpass="Node was withdrawn from election",
2809 onfail="Node was not withdrawn from election" )
2810
2811 main.step( "Make sure new leader is elected" )
2812
2813 leaderList = []
2814 for cli in main.CLIs:
2815 leaderN = cli.electionTestLeader()
2816 leaderList.append( leaderN )
2817 if leaderN == leader:
2818 main.log.error( cli.name + " still sees " + str( leader ) +
2819 " as leader after they withdrew" )
2820 leaderResult = main.FALSE
2821 elif leaderN == main.FALSE:
2822
2823
2824
2825 main.log.error( "Something is wrong with " +
2826 "electionTestLeader function, " +
2827 "check the error logs" )
2828 leaderResult = main.FALSE
2829 elif leaderN is None:
2830
2831 time.sleep(7)
2832 leaderN = cli.electionTestLeader()
2833 leaderList.pop()
2834 leaderList.append( leaderN )
2835 consistentLeader = main.FALSE
2836 if len( set( leaderList ) ) == 1:
2837 main.log.info( "Each Election-app sees '" +
2838 str( leaderList[ 0 ] ) +
2839 "' as the leader" )
2840 consistentLeader = main.TRUE
2841 else:
2842 main.log.error(
2843 "Inconsistent responses for leader of Election-app:" )
2844 for n in range( len( leaderList ) ):
2845 main.log.error( "ONOS" + str( n + 1 ) + " response: " +
2846 str( leaderList[ n ] ) )
2847 leaderResult = leaderResult and consistentLeader
2848 utilities.assert_equals(
2849 expect=main.TRUE,
2850 actual=leaderResult,
2851 onpass="Leadership election passed",
2852 onfail="Something went wrong with Leadership election" )
2853
2854 main.step( "Run for election on old leader( just so everyone " +
2855 "is in the hat )" )
2856 if oldLeader:
2857 runResult = oldLeader.electionTestRun()
2858 else:
2859 runResult = main.FALSE
2860 utilities.assert_equals(
2861 expect=main.TRUE,
2862 actual=runResult,
2863 onpass="App re-ran for election",
2864 onfail="App failed to run for election" )
2865
2866 main.step( "Leader did not change when old leader re-ran" )
2867 afterRun = main.ONOScli1.electionTestLeader()
2868
2869 if afterRun == leaderList[ 0 ]:
2870 afterResult = main.TRUE
2871 else:
2872 afterResult = main.FALSE
2873
2874 utilities.assert_equals(
2875 expect=main.TRUE,
2876 actual=afterResult,
2877 onpass="Old leader successfully re-ran for election",
2878 onfail="Something went wrong with Leadership election after " +
2879 "the old leader re-ran for election" )
2880
2882 """
2883 Install Distributed Primitives app
2884 """
2885 import time
2886 assert main.numCtrls, "main.numCtrls not defined"
2887 assert main, "main not defined"
2888 assert utilities.assert_equals, "utilities.assert_equals not defined"
2889 assert main.CLIs, "main.CLIs not defined"
2890 assert main.nodes, "main.nodes not defined"
2891
2892
2893 global pCounterName
2894 global iCounterName
2895 global pCounterValue
2896 global iCounterValue
2897 global onosSet
2898 global onosSetName
2899 pCounterName = "TestON-Partitions"
2900 iCounterName = "TestON-inMemory"
2901 pCounterValue = 0
2902 iCounterValue = 0
2903 onosSet = set([])
2904 onosSetName = "TestON-set"
2905
2906 description = "Install Primitives app"
2907 main.case( description )
2908 main.step( "Install Primitives app" )
2909 appName = "org.onosproject.distributedprimitives"
2910 appResults = main.CLIs[0].activateApp( appName )
2911 utilities.assert_equals( expect=main.TRUE,
2912 actual=appResults,
2913 onpass="Primitives app activated",
2914 onfail="Primitives app not activated" )
2915 time.sleep( 5 )
2916
2918 """
2919 Check for basic functionality with distributed primitives
2920 """
2921
2922 assert main.numCtrls, "main.numCtrls not defined"
2923 assert main, "main not defined"
2924 assert utilities.assert_equals, "utilities.assert_equals not defined"
2925 assert main.CLIs, "main.CLIs not defined"
2926 assert main.nodes, "main.nodes not defined"
2927 assert pCounterName, "pCounterName not defined"
2928 assert iCounterName, "iCounterName not defined"
2929 assert onosSetName, "onosSetName not defined"
2930
2931 try:
2932 pCounterValue
2933 except NameError:
2934 main.log.error( "pCounterValue not defined, setting to 0" )
2935 pCounterValue = 0
2936 try:
2937 iCounterValue
2938 except NameError:
2939 main.log.error( "iCounterValue not defined, setting to 0" )
2940 iCounterValue = 0
2941 try:
2942 onosSet
2943 except NameError:
2944 main.log.error( "onosSet not defined, setting to empty Set" )
2945 onosSet = set([])
2946
2947 addValue = "a"
2948 addAllValue = "a b c d e f"
2949 retainValue = "c d e f"
2950
2951 description = "Check for basic functionality with distributed " +\
2952 "primitives"
2953 main.case( description )
2954 main.caseExplanation = "Test the methods of the distributed " +\
2955 "primitives (counters and sets) throught the cli"
2956
2957
2958 main.step( "Increment then get a default counter on each node" )
2959 pCounters = []
2960 threads = []
2961 addedPValues = []
2962 for i in range( main.numCtrls ):
2963 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2964 name="counterAddAndGet-" + str( i ),
2965 args=[ pCounterName ] )
2966 pCounterValue += 1
2967 addedPValues.append( pCounterValue )
2968 threads.append( t )
2969 t.start()
2970
2971 for t in threads:
2972 t.join()
2973 pCounters.append( t.result )
2974
2975 pCounterResults = True
2976 for i in addedPValues:
2977 tmpResult = i in pCounters
2978 pCounterResults = pCounterResults and tmpResult
2979 if not tmpResult:
2980 main.log.error( str( i ) + " is not in partitioned "
2981 "counter incremented results" )
2982 utilities.assert_equals( expect=True,
2983 actual=pCounterResults,
2984 onpass="Default counter incremented",
2985 onfail="Error incrementing default" +
2986 " counter" )
2987
2988 main.step( "Get then Increment a default counter on each node" )
2989 pCounters = []
2990 threads = []
2991 addedPValues = []
2992 for i in range( main.numCtrls ):
2993 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2994 name="counterGetAndAdd-" + str( i ),
2995 args=[ pCounterName ] )
2996 addedPValues.append( pCounterValue )
2997 pCounterValue += 1
2998 threads.append( t )
2999 t.start()
3000
3001 for t in threads:
3002 t.join()
3003 pCounters.append( t.result )
3004
3005 pCounterResults = True
3006 for i in addedPValues:
3007 tmpResult = i in pCounters
3008 pCounterResults = pCounterResults and tmpResult
3009 if not tmpResult:
3010 main.log.error( str( i ) + " is not in partitioned "
3011 "counter incremented results" )
3012 utilities.assert_equals( expect=True,
3013 actual=pCounterResults,
3014 onpass="Default counter incremented",
3015 onfail="Error incrementing default" +
3016 " counter" )
3017
3018 main.step( "Counters we added have the correct values" )
3019 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3020 utilities.assert_equals( expect=main.TRUE,
3021 actual=incrementCheck,
3022 onpass="Added counters are correct",
3023 onfail="Added counters are incorrect" )
3024
3025 main.step( "Add -8 to then get a default counter on each node" )
3026 pCounters = []
3027 threads = []
3028 addedPValues = []
3029 for i in range( main.numCtrls ):
3030 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3031 name="counterIncrement-" + str( i ),
3032 args=[ pCounterName ],
3033 kwargs={ "delta": -8 } )
3034 pCounterValue += -8
3035 addedPValues.append( pCounterValue )
3036 threads.append( t )
3037 t.start()
3038
3039 for t in threads:
3040 t.join()
3041 pCounters.append( t.result )
3042
3043 pCounterResults = True
3044 for i in addedPValues:
3045 tmpResult = i in pCounters
3046 pCounterResults = pCounterResults and tmpResult
3047 if not tmpResult:
3048 main.log.error( str( i ) + " is not in partitioned "
3049 "counter incremented results" )
3050 utilities.assert_equals( expect=True,
3051 actual=pCounterResults,
3052 onpass="Default counter incremented",
3053 onfail="Error incrementing default" +
3054 " counter" )
3055
3056 main.step( "Add 5 to then get a default counter on each node" )
3057 pCounters = []
3058 threads = []
3059 addedPValues = []
3060 for i in range( main.numCtrls ):
3061 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3062 name="counterIncrement-" + str( i ),
3063 args=[ pCounterName ],
3064 kwargs={ "delta": 5 } )
3065 pCounterValue += 5
3066 addedPValues.append( pCounterValue )
3067 threads.append( t )
3068 t.start()
3069
3070 for t in threads:
3071 t.join()
3072 pCounters.append( t.result )
3073
3074 pCounterResults = True
3075 for i in addedPValues:
3076 tmpResult = i in pCounters
3077 pCounterResults = pCounterResults and tmpResult
3078 if not tmpResult:
3079 main.log.error( str( i ) + " is not in partitioned "
3080 "counter incremented results" )
3081 utilities.assert_equals( expect=True,
3082 actual=pCounterResults,
3083 onpass="Default counter incremented",
3084 onfail="Error incrementing default" +
3085 " counter" )
3086
3087 main.step( "Get then add 5 to a default counter on each node" )
3088 pCounters = []
3089 threads = []
3090 addedPValues = []
3091 for i in range( main.numCtrls ):
3092 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3093 name="counterIncrement-" + str( i ),
3094 args=[ pCounterName ],
3095 kwargs={ "delta": 5 } )
3096 addedPValues.append( pCounterValue )
3097 pCounterValue += 5
3098 threads.append( t )
3099 t.start()
3100
3101 for t in threads:
3102 t.join()
3103 pCounters.append( t.result )
3104
3105 pCounterResults = True
3106 for i in addedPValues:
3107 tmpResult = i in pCounters
3108 pCounterResults = pCounterResults and tmpResult
3109 if not tmpResult:
3110 main.log.error( str( i ) + " is not in partitioned "
3111 "counter incremented results" )
3112 utilities.assert_equals( expect=True,
3113 actual=pCounterResults,
3114 onpass="Default counter incremented",
3115 onfail="Error incrementing default" +
3116 " counter" )
3117
3118 main.step( "Counters we added have the correct values" )
3119 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3120 utilities.assert_equals( expect=main.TRUE,
3121 actual=incrementCheck,
3122 onpass="Added counters are correct",
3123 onfail="Added counters are incorrect" )
3124
3125
3126 main.step( "Increment and get an in-memory counter on each node" )
3127 iCounters = []
3128 addedIValues = []
3129 threads = []
3130 for i in range( main.numCtrls ):
3131 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3132 name="icounterIncrement-" + str( i ),
3133 args=[ iCounterName ],
3134 kwargs={ "inMemory": True } )
3135 iCounterValue += 1
3136 addedIValues.append( iCounterValue )
3137 threads.append( t )
3138 t.start()
3139
3140 for t in threads:
3141 t.join()
3142 iCounters.append( t.result )
3143
3144 iCounterResults = True
3145 for i in addedIValues:
3146 tmpResult = i in iCounters
3147 iCounterResults = iCounterResults and tmpResult
3148 if not tmpResult:
3149 main.log.error( str( i ) + " is not in the in-memory "
3150 "counter incremented results" )
3151 utilities.assert_equals( expect=True,
3152 actual=iCounterResults,
3153 onpass="In-memory counter incremented",
3154 onfail="Error incrementing in-memory" +
3155 " counter" )
3156
3157 main.step( "Get then Increment a in-memory counter on each node" )
3158 iCounters = []
3159 threads = []
3160 addedIValues = []
3161 for i in range( main.numCtrls ):
3162 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3163 name="counterGetAndAdd-" + str( i ),
3164 args=[ iCounterName ],
3165 kwargs={ "inMemory": True } )
3166 addedIValues.append( iCounterValue )
3167 iCounterValue += 1
3168 threads.append( t )
3169 t.start()
3170
3171 for t in threads:
3172 t.join()
3173 iCounters.append( t.result )
3174
3175 iCounterResults = True
3176 for i in addedIValues:
3177 tmpResult = i in iCounters
3178 iCounterResults = iCounterResults and tmpResult
3179 if not tmpResult:
3180 main.log.error( str( i ) + " is not in in-memory "
3181 "counter incremented results" )
3182 utilities.assert_equals( expect=True,
3183 actual=iCounterResults,
3184 onpass="In-memory counter incremented",
3185 onfail="Error incrementing in-memory" +
3186 " counter" )
3187
3188 main.step( "Counters we added have the correct values" )
3189 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3190 utilities.assert_equals( expect=main.TRUE,
3191 actual=incrementCheck,
3192 onpass="Added counters are correct",
3193 onfail="Added counters are incorrect" )
3194
3195 main.step( "Add -8 to then get a in-memory counter on each node" )
3196 iCounters = []
3197 threads = []
3198 addedIValues = []
3199 for i in range( main.numCtrls ):
3200 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3201 name="counterIncrement-" + str( i ),
3202 args=[ iCounterName ],
3203 kwargs={ "delta": -8, "inMemory": True } )
3204 iCounterValue += -8
3205 addedIValues.append( iCounterValue )
3206 threads.append( t )
3207 t.start()
3208
3209 for t in threads:
3210 t.join()
3211 iCounters.append( t.result )
3212
3213 iCounterResults = True
3214 for i in addedIValues:
3215 tmpResult = i in iCounters
3216 iCounterResults = iCounterResults and tmpResult
3217 if not tmpResult:
3218 main.log.error( str( i ) + " is not in in-memory "
3219 "counter incremented results" )
3220 utilities.assert_equals( expect=True,
3221 actual=pCounterResults,
3222 onpass="In-memory counter incremented",
3223 onfail="Error incrementing in-memory" +
3224 " counter" )
3225
3226 main.step( "Add 5 to then get a in-memory counter on each node" )
3227 iCounters = []
3228 threads = []
3229 addedIValues = []
3230 for i in range( main.numCtrls ):
3231 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3232 name="counterIncrement-" + str( i ),
3233 args=[ iCounterName ],
3234 kwargs={ "delta": 5, "inMemory": True } )
3235 iCounterValue += 5
3236 addedIValues.append( iCounterValue )
3237 threads.append( t )
3238 t.start()
3239
3240 for t in threads:
3241 t.join()
3242 iCounters.append( t.result )
3243
3244 iCounterResults = True
3245 for i in addedIValues:
3246 tmpResult = i in iCounters
3247 iCounterResults = iCounterResults and tmpResult
3248 if not tmpResult:
3249 main.log.error( str( i ) + " is not in in-memory "
3250 "counter incremented results" )
3251 utilities.assert_equals( expect=True,
3252 actual=pCounterResults,
3253 onpass="In-memory counter incremented",
3254 onfail="Error incrementing in-memory" +
3255 " counter" )
3256
3257 main.step( "Get then add 5 to a in-memory counter on each node" )
3258 iCounters = []
3259 threads = []
3260 addedIValues = []
3261 for i in range( main.numCtrls ):
3262 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3263 name="counterIncrement-" + str( i ),
3264 args=[ iCounterName ],
3265 kwargs={ "delta": 5, "inMemory": True } )
3266 addedIValues.append( iCounterValue )
3267 iCounterValue += 5
3268 threads.append( t )
3269 t.start()
3270
3271 for t in threads:
3272 t.join()
3273 iCounters.append( t.result )
3274
3275 iCounterResults = True
3276 for i in addedIValues:
3277 tmpResult = i in iCounters
3278 iCounterResults = iCounterResults and tmpResult
3279 if not tmpResult:
3280 main.log.error( str( i ) + " is not in in-memory "
3281 "counter incremented results" )
3282 utilities.assert_equals( expect=True,
3283 actual=iCounterResults,
3284 onpass="In-memory counter incremented",
3285 onfail="Error incrementing in-memory" +
3286 " counter" )
3287
3288 main.step( "Counters we added have the correct values" )
3289 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3290 utilities.assert_equals( expect=main.TRUE,
3291 actual=incrementCheck,
3292 onpass="Added counters are correct",
3293 onfail="Added counters are incorrect" )
3294
3295 main.step( "Check counters are consistant across nodes" )
3296 onosCounters = []
3297 threads = []
3298 for i in range( main.numCtrls ):
3299 t = main.Thread( target=main.CLIs[i].counters,
3300 name="counters-" + str( i ) )
3301 threads.append( t )
3302 t.start()
3303 for t in threads:
3304 t.join()
3305 onosCounters.append( t.result )
3306 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3307 if all( tmp ):
3308 main.log.info( "Counters are consistent across all nodes" )
3309 consistentCounterResults = main.TRUE
3310 else:
3311 main.log.error( "Counters are not consistent across all nodes" )
3312 consistentCounterResults = main.FALSE
3313 utilities.assert_equals( expect=main.TRUE,
3314 actual=consistentCounterResults,
3315 onpass="ONOS counters are consistent " +
3316 "across nodes",
3317 onfail="ONOS Counters are inconsistent " +
3318 "across nodes" )
3319
3320 main.step( "Counters we added have the correct values" )
3321 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3322 incrementCheck = incrementCheck and \
3323 main.Counters.counterCheck( iCounterName, iCounterValue )
3324 utilities.assert_equals( expect=main.TRUE,
3325 actual=incrementCheck,
3326 onpass="Added counters are correct",
3327 onfail="Added counters are incorrect" )
3328
3329
3330 main.step( "Distributed Set get" )
3331 size = len( onosSet )
3332 getResponses = []
3333 threads = []
3334 for i in range( main.numCtrls ):
3335 t = main.Thread( target=main.CLIs[i].setTestGet,
3336 name="setTestGet-" + str( i ),
3337 args=[ onosSetName ] )
3338 threads.append( t )
3339 t.start()
3340 for t in threads:
3341 t.join()
3342 getResponses.append( t.result )
3343
3344 getResults = main.TRUE
3345 for i in range( main.numCtrls ):
3346 if isinstance( getResponses[ i ], list):
3347 current = set( getResponses[ i ] )
3348 if len( current ) == len( getResponses[ i ] ):
3349
3350 if onosSet != current:
3351 main.log.error( "ONOS" + str( i + 1 ) +
3352 " has incorrect view" +
3353 " of set " + onosSetName + ":\n" +
3354 str( getResponses[ i ] ) )
3355 main.log.debug( "Expected: " + str( onosSet ) )
3356 main.log.debug( "Actual: " + str( current ) )
3357 getResults = main.FALSE
3358 else:
3359
3360 main.log.error( "ONOS" + str( i + 1 ) +
3361 " has repeat elements in" +
3362 " set " + onosSetName + ":\n" +
3363 str( getResponses[ i ] ) )
3364 getResults = main.FALSE
3365 elif getResponses[ i ] == main.ERROR:
3366 getResults = main.FALSE
3367 utilities.assert_equals( expect=main.TRUE,
3368 actual=getResults,
3369 onpass="Set elements are correct",
3370 onfail="Set elements are incorrect" )
3371
3372 main.step( "Distributed Set size" )
3373 sizeResponses = []
3374 threads = []
3375 for i in range( main.numCtrls ):
3376 t = main.Thread( target=main.CLIs[i].setTestSize,
3377 name="setTestSize-" + str( i ),
3378 args=[ onosSetName ] )
3379 threads.append( t )
3380 t.start()
3381 for t in threads:
3382 t.join()
3383 sizeResponses.append( t.result )
3384
3385 sizeResults = main.TRUE
3386 for i in range( main.numCtrls ):
3387 if size != sizeResponses[ i ]:
3388 sizeResults = main.FALSE
3389 main.log.error( "ONOS" + str( i + 1 ) +
3390 " expected a size of " + str( size ) +
3391 " for set " + onosSetName +
3392 " but got " + str( sizeResponses[ i ] ) )
3393 utilities.assert_equals( expect=main.TRUE,
3394 actual=sizeResults,
3395 onpass="Set sizes are correct",
3396 onfail="Set sizes are incorrect" )
3397
3398 main.step( "Distributed Set add()" )
3399 onosSet.add( addValue )
3400 addResponses = []
3401 threads = []
3402 for i in range( main.numCtrls ):
3403 t = main.Thread( target=main.CLIs[i].setTestAdd,
3404 name="setTestAdd-" + str( i ),
3405 args=[ onosSetName, addValue ] )
3406 threads.append( t )
3407 t.start()
3408 for t in threads:
3409 t.join()
3410 addResponses.append( t.result )
3411
3412
3413
3414
3415 addResults = main.TRUE
3416 for i in range( main.numCtrls ):
3417 if addResponses[ i ] == main.TRUE:
3418
3419 pass
3420 elif addResponses[ i ] == main.FALSE:
3421
3422 pass
3423 elif addResponses[ i ] == main.ERROR:
3424
3425 addResults = main.FALSE
3426 else:
3427
3428 addResults = main.FALSE
3429 if addResults != main.TRUE:
3430 main.log.error( "Error executing set add" )
3431
3432
3433 size = len( onosSet )
3434 getResponses = []
3435 threads = []
3436 for i in range( main.numCtrls ):
3437 t = main.Thread( target=main.CLIs[i].setTestGet,
3438 name="setTestGet-" + str( i ),
3439 args=[ onosSetName ] )
3440 threads.append( t )
3441 t.start()
3442 for t in threads:
3443 t.join()
3444 getResponses.append( t.result )
3445 getResults = main.TRUE
3446 for i in range( main.numCtrls ):
3447 if isinstance( getResponses[ i ], list):
3448 current = set( getResponses[ i ] )
3449 if len( current ) == len( getResponses[ i ] ):
3450
3451 if onosSet != current:
3452 main.log.error( "ONOS" + str( i + 1 ) +
3453 " has incorrect view" +
3454 " of set " + onosSetName + ":\n" +
3455 str( getResponses[ i ] ) )
3456 main.log.debug( "Expected: " + str( onosSet ) )
3457 main.log.debug( "Actual: " + str( current ) )
3458 getResults = main.FALSE
3459 else:
3460
3461 main.log.error( "ONOS" + str( i + 1 ) +
3462 " has repeat elements in" +
3463 " set " + onosSetName + ":\n" +
3464 str( getResponses[ i ] ) )
3465 getResults = main.FALSE
3466 elif getResponses[ i ] == main.ERROR:
3467 getResults = main.FALSE
3468 sizeResponses = []
3469 threads = []
3470 for i in range( main.numCtrls ):
3471 t = main.Thread( target=main.CLIs[i].setTestSize,
3472 name="setTestSize-" + str( i ),
3473 args=[ onosSetName ] )
3474 threads.append( t )
3475 t.start()
3476 for t in threads:
3477 t.join()
3478 sizeResponses.append( t.result )
3479 sizeResults = main.TRUE
3480 for i in range( main.numCtrls ):
3481 if size != sizeResponses[ i ]:
3482 sizeResults = main.FALSE
3483 main.log.error( "ONOS" + str( i + 1 ) +
3484 " expected a size of " + str( size ) +
3485 " for set " + onosSetName +
3486 " but got " + str( sizeResponses[ i ] ) )
3487 addResults = addResults and getResults and sizeResults
3488 utilities.assert_equals( expect=main.TRUE,
3489 actual=addResults,
3490 onpass="Set add correct",
3491 onfail="Set add was incorrect" )
3492
3493 main.step( "Distributed Set addAll()" )
3494 onosSet.update( addAllValue.split() )
3495 addResponses = []
3496 threads = []
3497 for i in range( main.numCtrls ):
3498 t = main.Thread( target=main.CLIs[i].setTestAdd,
3499 name="setTestAddAll-" + str( i ),
3500 args=[ onosSetName, addAllValue ] )
3501 threads.append( t )
3502 t.start()
3503 for t in threads:
3504 t.join()
3505 addResponses.append( t.result )
3506
3507
3508
3509
3510 addAllResults = main.TRUE
3511 for i in range( main.numCtrls ):
3512 if addResponses[ i ] == main.TRUE:
3513
3514 pass
3515 elif addResponses[ i ] == main.FALSE:
3516
3517 pass
3518 elif addResponses[ i ] == main.ERROR:
3519
3520 addAllResults = main.FALSE
3521 else:
3522
3523 addAllResults = main.FALSE
3524 if addAllResults != main.TRUE:
3525 main.log.error( "Error executing set addAll" )
3526
3527
3528 size = len( onosSet )
3529 getResponses = []
3530 threads = []
3531 for i in range( main.numCtrls ):
3532 t = main.Thread( target=main.CLIs[i].setTestGet,
3533 name="setTestGet-" + str( i ),
3534 args=[ onosSetName ] )
3535 threads.append( t )
3536 t.start()
3537 for t in threads:
3538 t.join()
3539 getResponses.append( t.result )
3540 getResults = main.TRUE
3541 for i in range( main.numCtrls ):
3542 if isinstance( getResponses[ i ], list):
3543 current = set( getResponses[ i ] )
3544 if len( current ) == len( getResponses[ i ] ):
3545
3546 if onosSet != current:
3547 main.log.error( "ONOS" + str( i + 1 ) +
3548 " has incorrect view" +
3549 " of set " + onosSetName + ":\n" +
3550 str( getResponses[ i ] ) )
3551 main.log.debug( "Expected: " + str( onosSet ) )
3552 main.log.debug( "Actual: " + str( current ) )
3553 getResults = main.FALSE
3554 else:
3555
3556 main.log.error( "ONOS" + str( i + 1 ) +
3557 " has repeat elements in" +
3558 " set " + onosSetName + ":\n" +
3559 str( getResponses[ i ] ) )
3560 getResults = main.FALSE
3561 elif getResponses[ i ] == main.ERROR:
3562 getResults = main.FALSE
3563 sizeResponses = []
3564 threads = []
3565 for i in range( main.numCtrls ):
3566 t = main.Thread( target=main.CLIs[i].setTestSize,
3567 name="setTestSize-" + str( i ),
3568 args=[ onosSetName ] )
3569 threads.append( t )
3570 t.start()
3571 for t in threads:
3572 t.join()
3573 sizeResponses.append( t.result )
3574 sizeResults = main.TRUE
3575 for i in range( main.numCtrls ):
3576 if size != sizeResponses[ i ]:
3577 sizeResults = main.FALSE
3578 main.log.error( "ONOS" + str( i + 1 ) +
3579 " expected a size of " + str( size ) +
3580 " for set " + onosSetName +
3581 " but got " + str( sizeResponses[ i ] ) )
3582 addAllResults = addAllResults and getResults and sizeResults
3583 utilities.assert_equals( expect=main.TRUE,
3584 actual=addAllResults,
3585 onpass="Set addAll correct",
3586 onfail="Set addAll was incorrect" )
3587
3588 main.step( "Distributed Set contains()" )
3589 containsResponses = []
3590 threads = []
3591 for i in range( main.numCtrls ):
3592 t = main.Thread( target=main.CLIs[i].setTestGet,
3593 name="setContains-" + str( i ),
3594 args=[ onosSetName ],
3595 kwargs={ "values": addValue } )
3596 threads.append( t )
3597 t.start()
3598 for t in threads:
3599 t.join()
3600
3601 containsResponses.append( t.result )
3602
3603 containsResults = main.TRUE
3604 for i in range( main.numCtrls ):
3605 if containsResponses[ i ] == main.ERROR:
3606 containsResults = main.FALSE
3607 else:
3608 containsResults = containsResults and\
3609 containsResponses[ i ][ 1 ]
3610 utilities.assert_equals( expect=main.TRUE,
3611 actual=containsResults,
3612 onpass="Set contains is functional",
3613 onfail="Set contains failed" )
3614
3615 main.step( "Distributed Set containsAll()" )
3616 containsAllResponses = []
3617 threads = []
3618 for i in range( main.numCtrls ):
3619 t = main.Thread( target=main.CLIs[i].setTestGet,
3620 name="setContainsAll-" + str( i ),
3621 args=[ onosSetName ],
3622 kwargs={ "values": addAllValue } )
3623 threads.append( t )
3624 t.start()
3625 for t in threads:
3626 t.join()
3627
3628 containsAllResponses.append( t.result )
3629
3630 containsAllResults = main.TRUE
3631 for i in range( main.numCtrls ):
3632 if containsResponses[ i ] == main.ERROR:
3633 containsResults = main.FALSE
3634 else:
3635 containsResults = containsResults and\
3636 containsResponses[ i ][ 1 ]
3637 utilities.assert_equals( expect=main.TRUE,
3638 actual=containsAllResults,
3639 onpass="Set containsAll is functional",
3640 onfail="Set containsAll failed" )
3641
3642 main.step( "Distributed Set remove()" )
3643 onosSet.remove( addValue )
3644 removeResponses = []
3645 threads = []
3646 for i in range( main.numCtrls ):
3647 t = main.Thread( target=main.CLIs[i].setTestRemove,
3648 name="setTestRemove-" + str( i ),
3649 args=[ onosSetName, addValue ] )
3650 threads.append( t )
3651 t.start()
3652 for t in threads:
3653 t.join()
3654 removeResponses.append( t.result )
3655
3656
3657
3658
3659 removeResults = main.TRUE
3660 for i in range( main.numCtrls ):
3661 if removeResponses[ i ] == main.TRUE:
3662
3663 pass
3664 elif removeResponses[ i ] == main.FALSE:
3665
3666 pass
3667 elif removeResponses[ i ] == main.ERROR:
3668
3669 removeResults = main.FALSE
3670 else:
3671
3672 removeResults = main.FALSE
3673 if removeResults != main.TRUE:
3674 main.log.error( "Error executing set remove" )
3675
3676
3677 size = len( onosSet )
3678 getResponses = []
3679 threads = []
3680 for i in range( main.numCtrls ):
3681 t = main.Thread( target=main.CLIs[i].setTestGet,
3682 name="setTestGet-" + str( i ),
3683 args=[ onosSetName ] )
3684 threads.append( t )
3685 t.start()
3686 for t in threads:
3687 t.join()
3688 getResponses.append( t.result )
3689 getResults = main.TRUE
3690 for i in range( main.numCtrls ):
3691 if isinstance( getResponses[ i ], list):
3692 current = set( getResponses[ i ] )
3693 if len( current ) == len( getResponses[ i ] ):
3694
3695 if onosSet != current:
3696 main.log.error( "ONOS" + str( i + 1 ) +
3697 " has incorrect view" +
3698 " of set " + onosSetName + ":\n" +
3699 str( getResponses[ i ] ) )
3700 main.log.debug( "Expected: " + str( onosSet ) )
3701 main.log.debug( "Actual: " + str( current ) )
3702 getResults = main.FALSE
3703 else:
3704
3705 main.log.error( "ONOS" + str( i + 1 ) +
3706 " has repeat elements in" +
3707 " set " + onosSetName + ":\n" +
3708 str( getResponses[ i ] ) )
3709 getResults = main.FALSE
3710 elif getResponses[ i ] == main.ERROR:
3711 getResults = main.FALSE
3712 sizeResponses = []
3713 threads = []
3714 for i in range( main.numCtrls ):
3715 t = main.Thread( target=main.CLIs[i].setTestSize,
3716 name="setTestSize-" + str( i ),
3717 args=[ onosSetName ] )
3718 threads.append( t )
3719 t.start()
3720 for t in threads:
3721 t.join()
3722 sizeResponses.append( t.result )
3723 sizeResults = main.TRUE
3724 for i in range( main.numCtrls ):
3725 if size != sizeResponses[ i ]:
3726 sizeResults = main.FALSE
3727 main.log.error( "ONOS" + str( i + 1 ) +
3728 " expected a size of " + str( size ) +
3729 " for set " + onosSetName +
3730 " but got " + str( sizeResponses[ i ] ) )
3731 removeResults = removeResults and getResults and sizeResults
3732 utilities.assert_equals( expect=main.TRUE,
3733 actual=removeResults,
3734 onpass="Set remove correct",
3735 onfail="Set remove was incorrect" )
3736
3737 main.step( "Distributed Set removeAll()" )
3738 onosSet.difference_update( addAllValue.split() )
3739 removeAllResponses = []
3740 threads = []
3741 try:
3742 for i in range( main.numCtrls ):
3743 t = main.Thread( target=main.CLIs[i].setTestRemove,
3744 name="setTestRemoveAll-" + str( i ),
3745 args=[ onosSetName, addAllValue ] )
3746 threads.append( t )
3747 t.start()
3748 for t in threads:
3749 t.join()
3750 removeAllResponses.append( t.result )
3751 except Exception, e:
3752 main.log.exception(e)
3753
3754
3755
3756
3757 removeAllResults = main.TRUE
3758 for i in range( main.numCtrls ):
3759 if removeAllResponses[ i ] == main.TRUE:
3760
3761 pass
3762 elif removeAllResponses[ i ] == main.FALSE:
3763
3764 pass
3765 elif removeAllResponses[ i ] == main.ERROR:
3766
3767 removeAllResults = main.FALSE
3768 else:
3769
3770 removeAllResults = main.FALSE
3771 if removeAllResults != main.TRUE:
3772 main.log.error( "Error executing set removeAll" )
3773
3774
3775 size = len( onosSet )
3776 getResponses = []
3777 threads = []
3778 for i in range( main.numCtrls ):
3779 t = main.Thread( target=main.CLIs[i].setTestGet,
3780 name="setTestGet-" + str( i ),
3781 args=[ onosSetName ] )
3782 threads.append( t )
3783 t.start()
3784 for t in threads:
3785 t.join()
3786 getResponses.append( t.result )
3787 getResults = main.TRUE
3788 for i in range( main.numCtrls ):
3789 if isinstance( getResponses[ i ], list):
3790 current = set( getResponses[ i ] )
3791 if len( current ) == len( getResponses[ i ] ):
3792
3793 if onosSet != current:
3794 main.log.error( "ONOS" + str( i + 1 ) +
3795 " has incorrect view" +
3796 " of set " + onosSetName + ":\n" +
3797 str( getResponses[ i ] ) )
3798 main.log.debug( "Expected: " + str( onosSet ) )
3799 main.log.debug( "Actual: " + str( current ) )
3800 getResults = main.FALSE
3801 else:
3802
3803 main.log.error( "ONOS" + str( i + 1 ) +
3804 " has repeat elements in" +
3805 " set " + onosSetName + ":\n" +
3806 str( getResponses[ i ] ) )
3807 getResults = main.FALSE
3808 elif getResponses[ i ] == main.ERROR:
3809 getResults = main.FALSE
3810 sizeResponses = []
3811 threads = []
3812 for i in range( main.numCtrls ):
3813 t = main.Thread( target=main.CLIs[i].setTestSize,
3814 name="setTestSize-" + str( i ),
3815 args=[ onosSetName ] )
3816 threads.append( t )
3817 t.start()
3818 for t in threads:
3819 t.join()
3820 sizeResponses.append( t.result )
3821 sizeResults = main.TRUE
3822 for i in range( main.numCtrls ):
3823 if size != sizeResponses[ i ]:
3824 sizeResults = main.FALSE
3825 main.log.error( "ONOS" + str( i + 1 ) +
3826 " expected a size of " + str( size ) +
3827 " for set " + onosSetName +
3828 " but got " + str( sizeResponses[ i ] ) )
3829 removeAllResults = removeAllResults and getResults and sizeResults
3830 utilities.assert_equals( expect=main.TRUE,
3831 actual=removeAllResults,
3832 onpass="Set removeAll correct",
3833 onfail="Set removeAll was incorrect" )
3834
3835 main.step( "Distributed Set addAll()" )
3836 onosSet.update( addAllValue.split() )
3837 addResponses = []
3838 threads = []
3839 for i in range( main.numCtrls ):
3840 t = main.Thread( target=main.CLIs[i].setTestAdd,
3841 name="setTestAddAll-" + str( i ),
3842 args=[ onosSetName, addAllValue ] )
3843 threads.append( t )
3844 t.start()
3845 for t in threads:
3846 t.join()
3847 addResponses.append( t.result )
3848
3849
3850
3851
3852 addAllResults = main.TRUE
3853 for i in range( main.numCtrls ):
3854 if addResponses[ i ] == main.TRUE:
3855
3856 pass
3857 elif addResponses[ i ] == main.FALSE:
3858
3859 pass
3860 elif addResponses[ i ] == main.ERROR:
3861
3862 addAllResults = main.FALSE
3863 else:
3864
3865 addAllResults = main.FALSE
3866 if addAllResults != main.TRUE:
3867 main.log.error( "Error executing set addAll" )
3868
3869
3870 size = len( onosSet )
3871 getResponses = []
3872 threads = []
3873 for i in range( main.numCtrls ):
3874 t = main.Thread( target=main.CLIs[i].setTestGet,
3875 name="setTestGet-" + str( i ),
3876 args=[ onosSetName ] )
3877 threads.append( t )
3878 t.start()
3879 for t in threads:
3880 t.join()
3881 getResponses.append( t.result )
3882 getResults = main.TRUE
3883 for i in range( main.numCtrls ):
3884 if isinstance( getResponses[ i ], list):
3885 current = set( getResponses[ i ] )
3886 if len( current ) == len( getResponses[ i ] ):
3887
3888 if onosSet != current:
3889 main.log.error( "ONOS" + str( i + 1 ) +
3890 " has incorrect view" +
3891 " of set " + onosSetName + ":\n" +
3892 str( getResponses[ i ] ) )
3893 main.log.debug( "Expected: " + str( onosSet ) )
3894 main.log.debug( "Actual: " + str( current ) )
3895 getResults = main.FALSE
3896 else:
3897
3898 main.log.error( "ONOS" + str( i + 1 ) +
3899 " has repeat elements in" +
3900 " set " + onosSetName + ":\n" +
3901 str( getResponses[ i ] ) )
3902 getResults = main.FALSE
3903 elif getResponses[ i ] == main.ERROR:
3904 getResults = main.FALSE
3905 sizeResponses = []
3906 threads = []
3907 for i in range( main.numCtrls ):
3908 t = main.Thread( target=main.CLIs[i].setTestSize,
3909 name="setTestSize-" + str( i ),
3910 args=[ onosSetName ] )
3911 threads.append( t )
3912 t.start()
3913 for t in threads:
3914 t.join()
3915 sizeResponses.append( t.result )
3916 sizeResults = main.TRUE
3917 for i in range( main.numCtrls ):
3918 if size != sizeResponses[ i ]:
3919 sizeResults = main.FALSE
3920 main.log.error( "ONOS" + str( i + 1 ) +
3921 " expected a size of " + str( size ) +
3922 " for set " + onosSetName +
3923 " but got " + str( sizeResponses[ i ] ) )
3924 addAllResults = addAllResults and getResults and sizeResults
3925 utilities.assert_equals( expect=main.TRUE,
3926 actual=addAllResults,
3927 onpass="Set addAll correct",
3928 onfail="Set addAll was incorrect" )
3929
3930 main.step( "Distributed Set clear()" )
3931 onosSet.clear()
3932 clearResponses = []
3933 threads = []
3934 for i in range( main.numCtrls ):
3935 t = main.Thread( target=main.CLIs[i].setTestRemove,
3936 name="setTestClear-" + str( i ),
3937 args=[ onosSetName, " "],
3938 kwargs={ "clear": True } )
3939 threads.append( t )
3940 t.start()
3941 for t in threads:
3942 t.join()
3943 clearResponses.append( t.result )
3944
3945
3946
3947
3948 clearResults = main.TRUE
3949 for i in range( main.numCtrls ):
3950 if clearResponses[ i ] == main.TRUE:
3951
3952 pass
3953 elif clearResponses[ i ] == main.FALSE:
3954
3955 pass
3956 elif clearResponses[ i ] == main.ERROR:
3957
3958 clearResults = main.FALSE
3959 else:
3960
3961 clearResults = main.FALSE
3962 if clearResults != main.TRUE:
3963 main.log.error( "Error executing set clear" )
3964
3965
3966 size = len( onosSet )
3967 getResponses = []
3968 threads = []
3969 for i in range( main.numCtrls ):
3970 t = main.Thread( target=main.CLIs[i].setTestGet,
3971 name="setTestGet-" + str( i ),
3972 args=[ onosSetName ] )
3973 threads.append( t )
3974 t.start()
3975 for t in threads:
3976 t.join()
3977 getResponses.append( t.result )
3978 getResults = main.TRUE
3979 for i in range( main.numCtrls ):
3980 if isinstance( getResponses[ i ], list):
3981 current = set( getResponses[ i ] )
3982 if len( current ) == len( getResponses[ i ] ):
3983
3984 if onosSet != current:
3985 main.log.error( "ONOS" + str( i + 1 ) +
3986 " has incorrect view" +
3987 " of set " + onosSetName + ":\n" +
3988 str( getResponses[ i ] ) )
3989 main.log.debug( "Expected: " + str( onosSet ) )
3990 main.log.debug( "Actual: " + str( current ) )
3991 getResults = main.FALSE
3992 else:
3993
3994 main.log.error( "ONOS" + str( i + 1 ) +
3995 " has repeat elements in" +
3996 " set " + onosSetName + ":\n" +
3997 str( getResponses[ i ] ) )
3998 getResults = main.FALSE
3999 elif getResponses[ i ] == main.ERROR:
4000 getResults = main.FALSE
4001 sizeResponses = []
4002 threads = []
4003 for i in range( main.numCtrls ):
4004 t = main.Thread( target=main.CLIs[i].setTestSize,
4005 name="setTestSize-" + str( i ),
4006 args=[ onosSetName ] )
4007 threads.append( t )
4008 t.start()
4009 for t in threads:
4010 t.join()
4011 sizeResponses.append( t.result )
4012 sizeResults = main.TRUE
4013 for i in range( main.numCtrls ):
4014 if size != sizeResponses[ i ]:
4015 sizeResults = main.FALSE
4016 main.log.error( "ONOS" + str( i + 1 ) +
4017 " expected a size of " + str( size ) +
4018 " for set " + onosSetName +
4019 " but got " + str( sizeResponses[ i ] ) )
4020 clearResults = clearResults and getResults and sizeResults
4021 utilities.assert_equals( expect=main.TRUE,
4022 actual=clearResults,
4023 onpass="Set clear correct",
4024 onfail="Set clear was incorrect" )
4025
4026 main.step( "Distributed Set addAll()" )
4027 onosSet.update( addAllValue.split() )
4028 addResponses = []
4029 threads = []
4030 for i in range( main.numCtrls ):
4031 t = main.Thread( target=main.CLIs[i].setTestAdd,
4032 name="setTestAddAll-" + str( i ),
4033 args=[ onosSetName, addAllValue ] )
4034 threads.append( t )
4035 t.start()
4036 for t in threads:
4037 t.join()
4038 addResponses.append( t.result )
4039
4040
4041
4042
4043 addAllResults = main.TRUE
4044 for i in range( main.numCtrls ):
4045 if addResponses[ i ] == main.TRUE:
4046
4047 pass
4048 elif addResponses[ i ] == main.FALSE:
4049
4050 pass
4051 elif addResponses[ i ] == main.ERROR:
4052
4053 addAllResults = main.FALSE
4054 else:
4055
4056 addAllResults = main.FALSE
4057 if addAllResults != main.TRUE:
4058 main.log.error( "Error executing set addAll" )
4059
4060
4061 size = len( onosSet )
4062 getResponses = []
4063 threads = []
4064 for i in range( main.numCtrls ):
4065 t = main.Thread( target=main.CLIs[i].setTestGet,
4066 name="setTestGet-" + str( i ),
4067 args=[ onosSetName ] )
4068 threads.append( t )
4069 t.start()
4070 for t in threads:
4071 t.join()
4072 getResponses.append( t.result )
4073 getResults = main.TRUE
4074 for i in range( main.numCtrls ):
4075 if isinstance( getResponses[ i ], list):
4076 current = set( getResponses[ i ] )
4077 if len( current ) == len( getResponses[ i ] ):
4078
4079 if onosSet != current:
4080 main.log.error( "ONOS" + str( i + 1 ) +
4081 " has incorrect view" +
4082 " of set " + onosSetName + ":\n" +
4083 str( getResponses[ i ] ) )
4084 main.log.debug( "Expected: " + str( onosSet ) )
4085 main.log.debug( "Actual: " + str( current ) )
4086 getResults = main.FALSE
4087 else:
4088
4089 main.log.error( "ONOS" + str( i + 1 ) +
4090 " has repeat elements in" +
4091 " set " + onosSetName + ":\n" +
4092 str( getResponses[ i ] ) )
4093 getResults = main.FALSE
4094 elif getResponses[ i ] == main.ERROR:
4095 getResults = main.FALSE
4096 sizeResponses = []
4097 threads = []
4098 for i in range( main.numCtrls ):
4099 t = main.Thread( target=main.CLIs[i].setTestSize,
4100 name="setTestSize-" + str( i ),
4101 args=[ onosSetName ] )
4102 threads.append( t )
4103 t.start()
4104 for t in threads:
4105 t.join()
4106 sizeResponses.append( t.result )
4107 sizeResults = main.TRUE
4108 for i in range( main.numCtrls ):
4109 if size != sizeResponses[ i ]:
4110 sizeResults = main.FALSE
4111 main.log.error( "ONOS" + str( i + 1 ) +
4112 " expected a size of " + str( size ) +
4113 " for set " + onosSetName +
4114 " but got " + str( sizeResponses[ i ] ) )
4115 addAllResults = addAllResults and getResults and sizeResults
4116 utilities.assert_equals( expect=main.TRUE,
4117 actual=addAllResults,
4118 onpass="Set addAll correct",
4119 onfail="Set addAll was incorrect" )
4120
4121 main.step( "Distributed Set retain()" )
4122 onosSet.intersection_update( retainValue.split() )
4123 retainResponses = []
4124 threads = []
4125 for i in range( main.numCtrls ):
4126 t = main.Thread( target=main.CLIs[i].setTestRemove,
4127 name="setTestRetain-" + str( i ),
4128 args=[ onosSetName, retainValue ],
4129 kwargs={ "retain": True } )
4130 threads.append( t )
4131 t.start()
4132 for t in threads:
4133 t.join()
4134 retainResponses.append( t.result )
4135
4136
4137
4138
4139 retainResults = main.TRUE
4140 for i in range( main.numCtrls ):
4141 if retainResponses[ i ] == main.TRUE:
4142
4143 pass
4144 elif retainResponses[ i ] == main.FALSE:
4145
4146 pass
4147 elif retainResponses[ i ] == main.ERROR:
4148
4149 retainResults = main.FALSE
4150 else:
4151
4152 retainResults = main.FALSE
4153 if retainResults != main.TRUE:
4154 main.log.error( "Error executing set retain" )
4155
4156
4157 size = len( onosSet )
4158 getResponses = []
4159 threads = []
4160 for i in range( main.numCtrls ):
4161 t = main.Thread( target=main.CLIs[i].setTestGet,
4162 name="setTestGet-" + str( i ),
4163 args=[ onosSetName ] )
4164 threads.append( t )
4165 t.start()
4166 for t in threads:
4167 t.join()
4168 getResponses.append( t.result )
4169 getResults = main.TRUE
4170 for i in range( main.numCtrls ):
4171 if isinstance( getResponses[ i ], list):
4172 current = set( getResponses[ i ] )
4173 if len( current ) == len( getResponses[ i ] ):
4174
4175 if onosSet != current:
4176 main.log.error( "ONOS" + str( i + 1 ) +
4177 " has incorrect view" +
4178 " of set " + onosSetName + ":\n" +
4179 str( getResponses[ i ] ) )
4180 main.log.debug( "Expected: " + str( onosSet ) )
4181 main.log.debug( "Actual: " + str( current ) )
4182 getResults = main.FALSE
4183 else:
4184
4185 main.log.error( "ONOS" + str( i + 1 ) +
4186 " has repeat elements in" +
4187 " set " + onosSetName + ":\n" +
4188 str( getResponses[ i ] ) )
4189 getResults = main.FALSE
4190 elif getResponses[ i ] == main.ERROR:
4191 getResults = main.FALSE
4192 sizeResponses = []
4193 threads = []
4194 for i in range( main.numCtrls ):
4195 t = main.Thread( target=main.CLIs[i].setTestSize,
4196 name="setTestSize-" + str( i ),
4197 args=[ onosSetName ] )
4198 threads.append( t )
4199 t.start()
4200 for t in threads:
4201 t.join()
4202 sizeResponses.append( t.result )
4203 sizeResults = main.TRUE
4204 for i in range( main.numCtrls ):
4205 if size != sizeResponses[ i ]:
4206 sizeResults = main.FALSE
4207 main.log.error( "ONOS" + str( i + 1 ) +
4208 " expected a size of " +
4209 str( size ) + " for set " + onosSetName +
4210 " but got " + str( sizeResponses[ i ] ) )
4211 retainResults = retainResults and getResults and sizeResults
4212 utilities.assert_equals( expect=main.TRUE,
4213 actual=retainResults,
4214 onpass="Set retain correct",
4215 onfail="Set retain was incorrect" )
4216