1 """
2 Description: This test is to determine if a single
3 instance ONOS 'cluster' can handle a restart
4
5 List of test cases:
6 CASE1: Compile ONOS and push it to the test machines
7 CASE2: Assign devices to controllers
8 CASE21: Assign mastership to controllers
9 CASE3: Assign intents
10 CASE4: Ping across added host intents
11 CASE5: Reading state of ONOS
12 CASE6: The Failure case.
13 CASE7: Check state after control plane failure
14 CASE8: Compare topo
15 CASE9: Link s3-s28 down
16 CASE10: Link s3-s28 up
17 CASE11: Switch down
18 CASE12: Switch up
19 CASE13: Clean up
20 CASE14: start election app on all onos nodes
21 CASE15: Check that Leadership Election is still functional
22 CASE16: Install Distributed Primitives app
23 CASE17: Check for basic functionality with distributed primitives
24 """
25
26
28
31
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
50 import imp
51 main.log.info( "ONOS Single node cluster restart " +
52 "HA test - initialization" )
53 main.case( "Setting up test environment" )
54 main.caseExplanation = "Setup the test environment including " +\
55 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57
58
59
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
66 main.numCtrls = int( main.params[ 'num_controllers' ] )
67 if main.ONOSbench.maxNodes:
68 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
70
71 try:
72 fileName = "Counters"
73 path = main.params[ 'imports' ][ 'path' ]
74 main.Counters = imp.load_source( fileName,
75 path + fileName + ".py" )
76 except Exception as e:
77 main.log.exception( e )
78 main.cleanup()
79 main.exit()
80
81 main.CLIs = []
82 main.nodes = []
83 ipList = []
84 for i in range( 1, int( main.ONOSbench.maxNodes ) + 1 ):
85 try:
86 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
87 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
88 ipList.append( main.nodes[ -1 ].ip_address )
89 except AttributeError:
90 break
91
92 main.step( "Create cell file" )
93 cellAppString = main.params[ 'ENV' ][ 'appString' ]
94 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
95 main.Mininet1.ip_address,
96 cellAppString, ipList )
97 main.step( "Applying cell variable to environment" )
98 cellResult = main.ONOSbench.setCell( cellName )
99 verifyResult = main.ONOSbench.verifyCell()
100
101
102 main.log.info( "Removing raft logs" )
103 main.ONOSbench.onosRemoveRaftLogs()
104
105 main.log.info( "Uninstalling ONOS" )
106 for node in main.nodes:
107 main.ONOSbench.onosUninstall( node.ip_address )
108
109
110 main.log.info( "Killing any ONOS processes" )
111 killResults = main.TRUE
112 for node in main.nodes:
113 killed = main.ONOSbench.onosKill( node.ip_address )
114 killResults = killResults and killed
115
116 cleanInstallResult = main.TRUE
117 gitPullResult = main.TRUE
118
119 main.step( "Starting Mininet" )
120
121
122 topoName = "obelisk.py"
123 filePath = main.ONOSbench.home + "/tools/test/topos/"
124 main.ONOSbench.copyMininetFile( topoName, filePath,
125 main.Mininet1.user_name,
126 main.Mininet1.ip_address )
127 mnResult = main.Mininet1.startNet( )
128 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
129 onpass="Mininet Started",
130 onfail="Error starting Mininet" )
131
132 main.step( "Git checkout and pull " + gitBranch )
133 if PULLCODE:
134 main.ONOSbench.gitCheckout( gitBranch )
135 gitPullResult = main.ONOSbench.gitPull()
136
137 utilities.assert_lesser( expect=0, actual=gitPullResult,
138 onpass="Git pull successful",
139 onfail="Git pull failed" )
140 main.ONOSbench.getVersion( report=True )
141
142 main.step( "Using mvn clean install" )
143 cleanInstallResult = main.TRUE
144 if PULLCODE and gitPullResult == main.TRUE:
145 cleanInstallResult = main.ONOSbench.cleanInstall()
146 else:
147 main.log.warn( "Did not pull new code so skipping mvn " +
148 "clean install" )
149 utilities.assert_equals( expect=main.TRUE,
150 actual=cleanInstallResult,
151 onpass="MCI successful",
152 onfail="MCI failed" )
153
154
155
156
157
158 job = "HAsingleInstanceRestart"
159 plotName = "Plot-HA"
160 graphs = '<ac:structured-macro ac:name="html">\n'
161 graphs += '<ac:plain-text-body><![CDATA[\n'
162 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
163 '/plot/' + plotName + '/getPlot?index=0' +\
164 '&width=500&height=300"' +\
165 'noborder="0" width="500" height="300" scrolling="yes" ' +\
166 'seamless="seamless"></iframe>\n'
167 graphs += ']]></ac:plain-text-body>\n'
168 graphs += '</ac:structured-macro>\n'
169 main.log.wiki(graphs)
170
171 main.CLIs = []
172 main.nodes = []
173 ipList = []
174 for i in range( 1, main.numCtrls + 1 ):
175 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
176 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
177 ipList.append( main.nodes[ -1 ].ip_address )
178
179 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, "SingleHA",
180 main.Mininet1.ip_address,
181 cellAppString, ipList[ 0 ] )
182 cellResult = main.ONOSbench.setCell( "SingleHA" )
183 verifyResult = main.ONOSbench.verifyCell()
184 main.step( "Creating ONOS package" )
185 packageResult = main.ONOSbench.onosPackage()
186 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
187 onpass="ONOS package successful",
188 onfail="ONOS package failed" )
189
190 main.step( "Installing ONOS package" )
191 onosInstallResult = main.ONOSbench.onosInstall(
192 options="-f", node=main.nodes[0].ip_address )
193 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
194 onpass="ONOS install successful",
195 onfail="ONOS install failed" )
196
197 main.step( "Checking if ONOS is up yet" )
198 for i in range( 2 ):
199 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
200 if onos1Isup:
201 break
202 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
203 onpass="ONOS startup successful",
204 onfail="ONOS startup failed" )
205
206 main.log.step( "Starting ONOS CLI sessions" )
207 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
208 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
209 onpass="ONOS cli startup successful",
210 onfail="ONOS cli startup failed" )
211
212 if main.params[ 'tcpdump' ].lower() == "true":
213 main.step( "Start Packet Capture MN" )
214 main.Mininet2.startTcpdump(
215 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
216 + "-MN.pcap",
217 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
218 port=main.params[ 'MNtcpdump' ][ 'port' ] )
219
220 main.step( "App Ids check" )
221 appCheck = main.ONOScli1.appToIDCheck()
222 if appCheck != main.TRUE:
223 main.log.warn( main.CLIs[0].apps() )
224 main.log.warn( main.CLIs[0].appIDs() )
225 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
226 onpass="App Ids seem to be correct",
227 onfail="Something is wrong with app Ids" )
228
229 if cliResults == main.FALSE:
230 main.log.error( "Failed to start ONOS, stopping test" )
231 main.cleanup()
232 main.exit()
233
234 - def CASE2( self, main ):
235 """
236 Assign devices to controllers
237 """
238 import re
239 assert main.numCtrls, "main.numCtrls not defined"
240 assert main, "main not defined"
241 assert utilities.assert_equals, "utilities.assert_equals not defined"
242
243 main.case( "Assigning devices to controllers" )
244 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
245 "and check that an ONOS node becomes the " +\
246 "master of the device."
247 main.step( "Assign switches to controllers" )
248
249 ipList = []
250 for i in range( main.numCtrls ):
251 ipList.append( main.nodes[ i ].ip_address )
252 swList = []
253 for i in range( 1, 29 ):
254 swList.append( "s" + str( i ) )
255 main.Mininet1.assignSwController( sw=swList, ip=ipList )
256
257 mastershipCheck = main.TRUE
258 for i in range( 1, 29 ):
259 response = main.Mininet1.getSwController( "s" + str( i ) )
260 try:
261 main.log.info( str( response ) )
262 except Exception:
263 main.log.info( repr( response ) )
264 if re.search( "tcp:" + main.nodes[0].ip_address, response ):
265 mastershipCheck = mastershipCheck and main.TRUE
266 else:
267 mastershipCheck = main.FALSE
268 if mastershipCheck == main.TRUE:
269 main.log.info( "Switch mastership assigned correctly" )
270 utilities.assert_equals(
271 expect=main.TRUE,
272 actual=mastershipCheck,
273 onpass="Switch mastership assigned correctly",
274 onfail="Switches not assigned correctly to controllers" )
275
277 """
278 Assign mastership to controllers
279 """
280 assert main.numCtrls, "main.numCtrls not defined"
281 assert main, "main not defined"
282 assert utilities.assert_equals, "utilities.assert_equals not defined"
283 assert main.CLIs, "main.CLIs not defined"
284 assert main.nodes, "main.nodes not defined"
285
286 main.case( "Assigning Controller roles for switches" )
287 main.caseExplanation = "Check that ONOS is connected to each " +\
288 "device. Then manually assign" +\
289 " mastership to specific ONOS nodes using" +\
290 " 'device-role'"
291 main.step( "Assign mastership of switches to specific controllers" )
292 roleCall = main.TRUE
293 roleCheck = main.TRUE
294 try:
295 for i in range( 1, 29 ):
296 ip = main.nodes[ 0 ].ip_address
297
298 if i == 1:
299 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
300 elif i == 2:
301 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
302 elif i == 3:
303 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
304 elif i == 4:
305 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
306 elif i == 5:
307 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
308 elif i == 6:
309 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
310 elif i == 7:
311 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
312 elif i >= 8 and i <= 17:
313 dpid = '3' + str( i ).zfill( 3 )
314 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
315 elif i >= 18 and i <= 27:
316 dpid = '6' + str( i ).zfill( 3 )
317 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
318 elif i == 28:
319 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
320 else:
321 main.log.error( "You didn't write an else statement for " +
322 "switch s" + str( i ) )
323
324 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
325
326 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
327 ip )
328
329 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
330 if ip in master:
331 roleCheck = roleCheck and main.TRUE
332 else:
333 roleCheck = roleCheck and main.FALSE
334 main.log.error( "Error, controller " + ip + " is not" +
335 " master " + "of device " +
336 str( deviceId ) + ". Master is " +
337 repr( master ) + "." )
338 except ( AttributeError, AssertionError ):
339 main.log.exception( "Something is wrong with ONOS device view" )
340 main.log.info( main.ONOScli1.devices() )
341 utilities.assert_equals(
342 expect=main.TRUE,
343 actual=roleCall,
344 onpass="Re-assigned switch mastership to designated controller",
345 onfail="Something wrong with deviceRole calls" )
346
347 main.step( "Check mastership was correctly assigned" )
348 utilities.assert_equals(
349 expect=main.TRUE,
350 actual=roleCheck,
351 onpass="Switches were successfully reassigned to designated " +
352 "controller",
353 onfail="Switches were not successfully reassigned" )
354
355 - def CASE3( self, main ):
356 """
357 Assign intents
358 """
359 import time
360 import json
361 assert main.numCtrls, "main.numCtrls not defined"
362 assert main, "main not defined"
363 assert utilities.assert_equals, "utilities.assert_equals not defined"
364
365
366 main.case( "Adding host Intents" )
367 main.caseExplanation = "Discover hosts by using pingall then " +\
368 "assign predetermined host-to-host intents." +\
369 " After installation, check that the intent" +\
370 " is distributed to all nodes and the state" +\
371 " is INSTALLED"
372
373
374 main.step( "Install reactive forwarding app" )
375 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
376 utilities.assert_equals( expect=main.TRUE, actual=installResults,
377 onpass="Install fwd successful",
378 onfail="Install fwd failed" )
379
380 main.step( "Check app ids" )
381 appCheck = main.ONOScli1.appToIDCheck()
382 if appCheck != main.TRUE:
383 main.log.warn( main.CLIs[0].apps() )
384 main.log.warn( main.CLIs[0].appIDs() )
385 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
386 onpass="App Ids seem to be correct",
387 onfail="Something is wrong with app Ids" )
388
389 main.step( "Discovering Hosts( Via pingall for now )" )
390
391
392 pingResult = main.FALSE
393 for i in range(2):
394 time1 = time.time()
395 pingResult = main.Mininet1.pingall()
396 if i == 0:
397 utilities.assert_equals(
398 expect=main.TRUE,
399 actual=pingResult,
400 onpass="Reactive Pingall test passed",
401 onfail="Reactive Pingall failed, " +
402 "one or more ping pairs failed" )
403 time2 = time.time()
404 main.log.info( "Time for pingall: %2f seconds" %
405 ( time2 - time1 ) )
406
407 time.sleep( 11 )
408
409 main.step( "Uninstall reactive forwarding app" )
410 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
411 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
412 onpass="Uninstall fwd successful",
413 onfail="Uninstall fwd failed" )
414
415 main.step( "Check app ids" )
416 appCheck2 = main.ONOScli1.appToIDCheck()
417 if appCheck2 != main.TRUE:
418 main.log.warn( main.CLIs[0].apps() )
419 main.log.warn( main.CLIs[0].appIDs() )
420 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
421 onpass="App Ids seem to be correct",
422 onfail="Something is wrong with app Ids" )
423
424 main.step( "Add host intents via cli" )
425 intentIds = []
426
427
428 intentAddResult = True
429 hostResult = main.TRUE
430 for i in range( 8, 18 ):
431 main.log.info( "Adding host intent between h" + str( i ) +
432 " and h" + str( i + 10 ) )
433 host1 = "00:00:00:00:00:" + \
434 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
435 host2 = "00:00:00:00:00:" + \
436 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
437
438 host1Dict = main.ONOScli1.getHost( host1 )
439 host2Dict = main.ONOScli1.getHost( host2 )
440 host1Id = None
441 host2Id = None
442 if host1Dict and host2Dict:
443 host1Id = host1Dict.get( 'id', None )
444 host2Id = host2Dict.get( 'id', None )
445 if host1Id and host2Id:
446 tmpId = main.ONOScli1.addHostIntent( host1Id, host2Id )
447 if tmpId:
448 main.log.info( "Added intent with id: " + tmpId )
449 intentIds.append( tmpId )
450 else:
451 main.log.error( "addHostIntent returned: " +
452 repr( tmpId ) )
453 else:
454 main.log.error( "Error, getHost() failed for h" + str( i ) +
455 " and/or h" + str( i + 10 ) )
456 hosts = main.ONOScli1.hosts()
457 main.log.warn( "Hosts output: " )
458 try:
459 main.log.warn( json.dumps( json.loads( hosts ),
460 sort_keys=True,
461 indent=4,
462 separators=( ',', ': ' ) ) )
463 except ( ValueError, TypeError ):
464 main.log.warn( repr( hosts ) )
465 hostResult = main.FALSE
466 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
467 onpass="Found a host id for each host",
468 onfail="Error looking up host ids" )
469
470 intentStart = time.time()
471 onosIds = main.ONOScli1.getAllIntentsId()
472 main.log.info( "Submitted intents: " + str( intentIds ) )
473 main.log.info( "Intents in ONOS: " + str( onosIds ) )
474 for intent in intentIds:
475 if intent in onosIds:
476 pass
477 else:
478 intentAddResult = False
479 if intentAddResult:
480 intentStop = time.time()
481 else:
482 intentStop = None
483
484 intents = main.ONOScli1.intents()
485 intentStates = []
486 installedCheck = True
487 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
488 count = 0
489 try:
490 for intent in json.loads( intents ):
491 state = intent.get( 'state', None )
492 if "INSTALLED" not in state:
493 installedCheck = False
494 intentId = intent.get( 'id', None )
495 intentStates.append( ( intentId, state ) )
496 except ( ValueError, TypeError ):
497 main.log.exception( "Error parsing intents" )
498
499 tmplist = [ i for i, s in intentStates ]
500 missingIntents = False
501 for i in intentIds:
502 if i not in tmplist:
503 intentStates.append( ( i, " - " ) )
504 missingIntents = True
505 intentStates.sort()
506 for i, s in intentStates:
507 count += 1
508 main.log.info( "%-6s%-15s%-15s" %
509 ( str( count ), str( i ), str( s ) ) )
510 leaders = main.ONOScli1.leaders()
511 try:
512 missing = False
513 if leaders:
514 parsedLeaders = json.loads( leaders )
515 main.log.warn( json.dumps( parsedLeaders,
516 sort_keys=True,
517 indent=4,
518 separators=( ',', ': ' ) ) )
519
520 topics = []
521 for i in range( 14 ):
522 topics.append( "intent-partition-" + str( i ) )
523 main.log.debug( topics )
524 ONOStopics = [ j['topic'] for j in parsedLeaders ]
525 for topic in topics:
526 if topic not in ONOStopics:
527 main.log.error( "Error: " + topic +
528 " not in leaders" )
529 missing = True
530 else:
531 main.log.error( "leaders() returned None" )
532 except ( ValueError, TypeError ):
533 main.log.exception( "Error parsing leaders" )
534 main.log.error( repr( leaders ) )
535
536 if missing:
537 response = main.ONOScli1.leaders( jsonFormat=False)
538 main.log.warn( "ONOS1 leaders output: \n" +
539 str( response ) )
540
541 partitions = main.ONOScli1.partitions()
542 try:
543 if partitions :
544 parsedPartitions = json.loads( partitions )
545 main.log.warn( json.dumps( parsedPartitions,
546 sort_keys=True,
547 indent=4,
548 separators=( ',', ': ' ) ) )
549
550
551 else:
552 main.log.error( "partitions() returned None" )
553 except ( ValueError, TypeError ):
554 main.log.exception( "Error parsing partitions" )
555 main.log.error( repr( partitions ) )
556 pendingMap = main.ONOScli1.pendingMap()
557 try:
558 if pendingMap :
559 parsedPending = json.loads( pendingMap )
560 main.log.warn( json.dumps( parsedPending,
561 sort_keys=True,
562 indent=4,
563 separators=( ',', ': ' ) ) )
564
565 else:
566 main.log.error( "pendingMap() returned None" )
567 except ( ValueError, TypeError ):
568 main.log.exception( "Error parsing pending map" )
569 main.log.error( repr( pendingMap ) )
570
571 intentAddResult = bool( intentAddResult and not missingIntents and
572 installedCheck )
573 if not intentAddResult:
574 main.log.error( "Error in pushing host intents to ONOS" )
575
576 main.step( "Intent Anti-Entropy dispersion" )
577 for i in range(100):
578 correct = True
579 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
580 for cli in main.CLIs:
581 onosIds = []
582 ids = cli.getAllIntentsId()
583 onosIds.append( ids )
584 main.log.debug( "Intents in " + cli.name + ": " +
585 str( sorted( onosIds ) ) )
586 if sorted( ids ) != sorted( intentIds ):
587 main.log.warn( "Set of intent IDs doesn't match" )
588 correct = False
589 break
590 else:
591 intents = json.loads( cli.intents() )
592 for intent in intents:
593 if intent[ 'state' ] != "INSTALLED":
594 main.log.warn( "Intent " + intent[ 'id' ] +
595 " is " + intent[ 'state' ] )
596 correct = False
597 break
598 if correct:
599 break
600 else:
601 time.sleep(1)
602 if not intentStop:
603 intentStop = time.time()
604 global gossipTime
605 gossipTime = intentStop - intentStart
606 main.log.info( "It took about " + str( gossipTime ) +
607 " seconds for all intents to appear in each node" )
608
609
610 utilities.assert_greater_equals(
611 expect=40, actual=gossipTime,
612 onpass="ECM anti-entropy for intents worked within " +
613 "expected time",
614 onfail="Intent ECM anti-entropy took too long" )
615 if gossipTime <= 40:
616 intentAddResult = True
617
618 if not intentAddResult or "key" in pendingMap:
619 import time
620 installedCheck = True
621 main.log.info( "Sleeping 60 seconds to see if intents are found" )
622 time.sleep( 60 )
623 onosIds = main.ONOScli1.getAllIntentsId()
624 main.log.info( "Submitted intents: " + str( intentIds ) )
625 main.log.info( "Intents in ONOS: " + str( onosIds ) )
626
627 intents = main.ONOScli1.intents()
628 intentStates = []
629 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
630 count = 0
631 try:
632 for intent in json.loads( intents ):
633
634 state = intent.get( 'state', None )
635 if "INSTALLED" not in state:
636 installedCheck = False
637 intentId = intent.get( 'id', None )
638 intentStates.append( ( intentId, state ) )
639 except ( ValueError, TypeError ):
640 main.log.exception( "Error parsing intents" )
641
642 tmplist = [ i for i, s in intentStates ]
643 for i in intentIds:
644 if i not in tmplist:
645 intentStates.append( ( i, " - " ) )
646 intentStates.sort()
647 for i, s in intentStates:
648 count += 1
649 main.log.info( "%-6s%-15s%-15s" %
650 ( str( count ), str( i ), str( s ) ) )
651 leaders = main.ONOScli1.leaders()
652 try:
653 missing = False
654 if leaders:
655 parsedLeaders = json.loads( leaders )
656 main.log.warn( json.dumps( parsedLeaders,
657 sort_keys=True,
658 indent=4,
659 separators=( ',', ': ' ) ) )
660
661
662 topics = []
663 for i in range( 14 ):
664 topics.append( "intent-partition-" + str( i ) )
665
666 topics.append( "org.onosproject.election" )
667 main.log.debug( topics )
668 ONOStopics = [ j['topic'] for j in parsedLeaders ]
669 for topic in topics:
670 if topic not in ONOStopics:
671 main.log.error( "Error: " + topic +
672 " not in leaders" )
673 missing = True
674 else:
675 main.log.error( "leaders() returned None" )
676 except ( ValueError, TypeError ):
677 main.log.exception( "Error parsing leaders" )
678 main.log.error( repr( leaders ) )
679
680 if missing:
681 response = main.ONOScli1.leaders( jsonFormat=False)
682 main.log.warn( "ONOS1 leaders output: \n" +
683 str( response ) )
684 partitions = main.ONOScli1.partitions()
685 try:
686 if partitions :
687 parsedPartitions = json.loads( partitions )
688 main.log.warn( json.dumps( parsedPartitions,
689 sort_keys=True,
690 indent=4,
691 separators=( ',', ': ' ) ) )
692
693
694 else:
695 main.log.error( "partitions() returned None" )
696 except ( ValueError, TypeError ):
697 main.log.exception( "Error parsing partitions" )
698 main.log.error( repr( partitions ) )
699 pendingMap = main.ONOScli1.pendingMap()
700 try:
701 if pendingMap :
702 parsedPending = json.loads( pendingMap )
703 main.log.warn( json.dumps( parsedPending,
704 sort_keys=True,
705 indent=4,
706 separators=( ',', ': ' ) ) )
707
708 else:
709 main.log.error( "pendingMap() returned None" )
710 except ( ValueError, TypeError ):
711 main.log.exception( "Error parsing pending map" )
712 main.log.error( repr( pendingMap ) )
713
714 - def CASE4( self, main ):
715 """
716 Ping across added host intents
717 """
718 import json
719 import time
720 assert main.numCtrls, "main.numCtrls not defined"
721 assert main, "main not defined"
722 assert utilities.assert_equals, "utilities.assert_equals not defined"
723 main.case( "Verify connectivity by sendind traffic across Intents" )
724 main.caseExplanation = "Ping across added host intents to check " +\
725 "functionality and check the state of " +\
726 "the intent"
727 main.step( "Ping across added host intents" )
728 PingResult = main.TRUE
729 for i in range( 8, 18 ):
730 ping = main.Mininet1.pingHost( src="h" + str( i ),
731 target="h" + str( i + 10 ) )
732 PingResult = PingResult and ping
733 if ping == main.FALSE:
734 main.log.warn( "Ping failed between h" + str( i ) +
735 " and h" + str( i + 10 ) )
736 elif ping == main.TRUE:
737 main.log.info( "Ping test passed!" )
738
739 if PingResult == main.FALSE:
740 main.log.error(
741 "Intents have not been installed correctly, pings failed." )
742
743 main.log.warn( "ONOS1 intents: " )
744 try:
745 tmpIntents = main.ONOScli1.intents()
746 main.log.warn( json.dumps( json.loads( tmpIntents ),
747 sort_keys=True,
748 indent=4,
749 separators=( ',', ': ' ) ) )
750 except ( ValueError, TypeError ):
751 main.log.warn( repr( tmpIntents ) )
752 utilities.assert_equals(
753 expect=main.TRUE,
754 actual=PingResult,
755 onpass="Intents have been installed correctly and pings work",
756 onfail="Intents have not been installed correctly, pings failed." )
757
758 main.step( "Check Intent state" )
759 installedCheck = True
760
761 intents = main.ONOScli1.intents()
762 intentStates = []
763 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
764 count = 0
765
766 try:
767 for intent in json.loads( intents ):
768 state = intent.get( 'state', None )
769 if "INSTALLED" not in state:
770 installedCheck = False
771 intentId = intent.get( 'id', None )
772 intentStates.append( ( intentId, state ) )
773 except ( ValueError, TypeError ):
774 main.log.exception( "Error parsing intents." )
775
776 intentStates.sort()
777 for i, s in intentStates:
778 count += 1
779 main.log.info( "%-6s%-15s%-15s" %
780 ( str( count ), str( i ), str( s ) ) )
781 utilities.assert_equals( expect=True, actual=installedCheck,
782 onpass="Intents are all INSTALLED",
783 onfail="Intents are not all in " +
784 "INSTALLED state" )
785
786 main.step( "Check leadership of topics" )
787 leaders = main.ONOScli1.leaders()
788 topicCheck = main.TRUE
789 try:
790 if leaders:
791 parsedLeaders = json.loads( leaders )
792 main.log.warn( json.dumps( parsedLeaders,
793 sort_keys=True,
794 indent=4,
795 separators=( ',', ': ' ) ) )
796
797
798
799 topics = []
800 for i in range( 14 ):
801 topics.append( "intent-partition-" + str( i ) )
802
803
804
805 main.log.debug( topics )
806 ONOStopics = [ j['topic'] for j in parsedLeaders ]
807 for topic in topics:
808 if topic not in ONOStopics:
809 main.log.error( "Error: " + topic +
810 " not in leaders" )
811 topicCheck = main.FALSE
812 else:
813 main.log.error( "leaders() returned None" )
814 topicCheck = main.FALSE
815 except ( ValueError, TypeError ):
816 topicCheck = main.FALSE
817 main.log.exception( "Error parsing leaders" )
818 main.log.error( repr( leaders ) )
819
820 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
821 onpass="intent Partitions is in leaders",
822 onfail="Some topics were lost " )
823
824 partitions = main.ONOScli1.partitions()
825 try:
826 if partitions :
827 parsedPartitions = json.loads( partitions )
828 main.log.warn( json.dumps( parsedPartitions,
829 sort_keys=True,
830 indent=4,
831 separators=( ',', ': ' ) ) )
832
833
834 else:
835 main.log.error( "partitions() returned None" )
836 except ( ValueError, TypeError ):
837 main.log.exception( "Error parsing partitions" )
838 main.log.error( repr( partitions ) )
839
840 pendingMap = main.ONOScli1.pendingMap()
841 try:
842 if pendingMap :
843 parsedPending = json.loads( pendingMap )
844 main.log.warn( json.dumps( parsedPending,
845 sort_keys=True,
846 indent=4,
847 separators=( ',', ': ' ) ) )
848
849 else:
850 main.log.error( "pendingMap() returned None" )
851 except ( ValueError, TypeError ):
852 main.log.exception( "Error parsing pending map" )
853 main.log.error( repr( pendingMap ) )
854
855 if not installedCheck:
856 main.log.info( "Waiting 60 seconds to see if the state of " +
857 "intents change" )
858 time.sleep( 60 )
859
860 intents = main.ONOScli1.intents()
861 intentStates = []
862 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
863 count = 0
864
865 try:
866 for intent in json.loads( intents ):
867 state = intent.get( 'state', None )
868 if "INSTALLED" not in state:
869 installedCheck = False
870 intentId = intent.get( 'id', None )
871 intentStates.append( ( intentId, state ) )
872 except ( ValueError, TypeError ):
873 main.log.exception( "Error parsing intents." )
874 intentStates.sort()
875 for i, s in intentStates:
876 count += 1
877 main.log.info( "%-6s%-15s%-15s" %
878 ( str( count ), str( i ), str( s ) ) )
879 leaders = main.ONOScli1.leaders()
880 try:
881 missing = False
882 if leaders:
883 parsedLeaders = json.loads( leaders )
884 main.log.warn( json.dumps( parsedLeaders,
885 sort_keys=True,
886 indent=4,
887 separators=( ',', ': ' ) ) )
888
889
890 topics = []
891 for i in range( 14 ):
892 topics.append( "intent-partition-" + str( i ) )
893
894 topics.append( "org.onosproject.election" )
895 main.log.debug( topics )
896 ONOStopics = [ j['topic'] for j in parsedLeaders ]
897 for topic in topics:
898 if topic not in ONOStopics:
899 main.log.error( "Error: " + topic +
900 " not in leaders" )
901 missing = True
902 else:
903 main.log.error( "leaders() returned None" )
904 except ( ValueError, TypeError ):
905 main.log.exception( "Error parsing leaders" )
906 main.log.error( repr( leaders ) )
907 if missing:
908 response = main.ONOScli1.leaders( jsonFormat=False)
909 main.log.warn( "ONOS1 leaders output: \n" +
910 str( response ) )
911 partitions = main.ONOScli1.partitions()
912 try:
913 if partitions :
914 parsedPartitions = json.loads( partitions )
915 main.log.warn( json.dumps( parsedPartitions,
916 sort_keys=True,
917 indent=4,
918 separators=( ',', ': ' ) ) )
919
920
921 else:
922 main.log.error( "partitions() returned None" )
923 except ( ValueError, TypeError ):
924 main.log.exception( "Error parsing partitions" )
925 main.log.error( repr( partitions ) )
926 pendingMap = main.ONOScli1.pendingMap()
927 try:
928 if pendingMap :
929 parsedPending = json.loads( pendingMap )
930 main.log.warn( json.dumps( parsedPending,
931 sort_keys=True,
932 indent=4,
933 separators=( ',', ': ' ) ) )
934
935 else:
936 main.log.error( "pendingMap() returned None" )
937 except ( ValueError, TypeError ):
938 main.log.exception( "Error parsing pending map" )
939 main.log.error( repr( pendingMap ) )
940
941 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
942 main.step( "Wait a minute then ping again" )
943
944 PingResult = main.TRUE
945 for i in range( 8, 18 ):
946 ping = main.Mininet1.pingHost( src="h" + str( i ),
947 target="h" + str( i + 10 ) )
948 PingResult = PingResult and ping
949 if ping == main.FALSE:
950 main.log.warn( "Ping failed between h" + str( i ) +
951 " and h" + str( i + 10 ) )
952 elif ping == main.TRUE:
953 main.log.info( "Ping test passed!" )
954
955 if PingResult == main.FALSE:
956 main.log.error(
957 "Intents have not been installed correctly, pings failed." )
958
959 main.log.warn( "ONOS1 intents: " )
960 try:
961 tmpIntents = main.ONOScli1.intents()
962 main.log.warn( json.dumps( json.loads( tmpIntents ),
963 sort_keys=True,
964 indent=4,
965 separators=( ',', ': ' ) ) )
966 except ( ValueError, TypeError ):
967 main.log.warn( repr( tmpIntents ) )
968 utilities.assert_equals(
969 expect=main.TRUE,
970 actual=PingResult,
971 onpass="Intents have been installed correctly and pings work",
972 onfail="Intents have not been installed correctly, pings failed." )
973
974 - def CASE5( self, main ):
975 """
976 Reading state of ONOS
977 """
978 import json
979 assert main.numCtrls, "main.numCtrls not defined"
980 assert main, "main not defined"
981 assert utilities.assert_equals, "utilities.assert_equals not defined"
982
983 main.case( "Setting up and gathering data for current state" )
984
985
986
987
988 main.step( "Check that each switch has a master" )
989 global mastershipState
990 mastershipState = '[]'
991
992
993 rolesNotNull = main.ONOScli1.rolesNotNull()
994 utilities.assert_equals(
995 expect=main.TRUE,
996 actual=rolesNotNull,
997 onpass="Each device has a master",
998 onfail="Some devices don't have a master assigned" )
999
1000 main.step( "Get the Mastership of each switch" )
1001 ONOS1Mastership = main.ONOScli1.roles()
1002
1003 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1004 main.log.error( "Error in getting ONOS roles" )
1005 main.log.warn(
1006 "ONOS1 mastership response: " +
1007 repr( ONOS1Mastership ) )
1008 consistentMastership = main.FALSE
1009 else:
1010 mastershipState = ONOS1Mastership
1011 consistentMastership = main.TRUE
1012
1013 main.step( "Get the intents from each controller" )
1014 global intentState
1015 intentState = []
1016 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1017 intentCheck = main.FALSE
1018 if "Error" in ONOS1Intents or not ONOS1Intents:
1019 main.log.error( "Error in getting ONOS intents" )
1020 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1021 else:
1022 intentCheck = main.TRUE
1023
1024 main.step( "Get the flows from each controller" )
1025 global flowState
1026 flowState = []
1027 flowCheck = main.FALSE
1028 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
1029 if "Error" in ONOS1Flows or not ONOS1Flows:
1030 main.log.error( "Error in getting ONOS flows" )
1031 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
1032 else:
1033
1034 flowState = ONOS1Flows
1035 flowCheck = main.TRUE
1036
1037 main.step( "Get the OF Table entries" )
1038 global flows
1039 flows = []
1040 for i in range( 1, 29 ):
1041 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
1042 if flowCheck == main.FALSE:
1043 for table in flows:
1044 main.log.warn( table )
1045
1046
1047 main.step( "Collecting topology information from ONOS" )
1048 devices = []
1049 devices.append( main.ONOScli1.devices() )
1050 hosts = []
1051 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1052 ports = []
1053 ports.append( main.ONOScli1.ports() )
1054 links = []
1055 links.append( main.ONOScli1.links() )
1056 clusters = []
1057 clusters.append( main.ONOScli1.clusters() )
1058
1059 main.step( "Each host has an IP address" )
1060 ipResult = main.TRUE
1061 for controller in range( 0, len( hosts ) ):
1062 controllerStr = str( controller + 1 )
1063 for host in hosts[ controller ]:
1064 if host is None or host.get( 'ipAddresses', [] ) == []:
1065 main.log.error(
1066 "DEBUG:Error with host ips on controller" +
1067 controllerStr + ": " + str( host ) )
1068 ipResult = main.FALSE
1069 utilities.assert_equals(
1070 expect=main.TRUE,
1071 actual=ipResult,
1072 onpass="The ips of the hosts aren't empty",
1073 onfail="The ip of at least one host is missing" )
1074
1075
1076 main.step( "There is only one dataplane cluster" )
1077 try:
1078 numClusters = len( json.loads( clusters[ 0 ] ) )
1079 except ( ValueError, TypeError ):
1080 main.log.exception( "Error parsing clusters[0]: " +
1081 repr( clusters[ 0 ] ) )
1082 clusterResults = main.FALSE
1083 if numClusters == 1:
1084 clusterResults = main.TRUE
1085 utilities.assert_equals(
1086 expect=1,
1087 actual=numClusters,
1088 onpass="ONOS shows 1 SCC",
1089 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1090
1091 main.step( "Comparing ONOS topology to MN" )
1092 devicesResults = main.TRUE
1093 linksResults = main.TRUE
1094 hostsResults = main.TRUE
1095 mnSwitches = main.Mininet1.getSwitches()
1096 mnLinks = main.Mininet1.getLinks()
1097 mnHosts = main.Mininet1.getHosts()
1098 for controller in range( main.numCtrls ):
1099 controllerStr = str( controller + 1 )
1100 if devices[ controller ] and ports[ controller ] and\
1101 "Error" not in devices[ controller ] and\
1102 "Error" not in ports[ controller ]:
1103
1104 currentDevicesResult = main.Mininet1.compareSwitches(
1105 mnSwitches,
1106 json.loads( devices[ controller ] ),
1107 json.loads( ports[ controller ] ) )
1108 else:
1109 currentDevicesResult = main.FALSE
1110 utilities.assert_equals( expect=main.TRUE,
1111 actual=currentDevicesResult,
1112 onpass="ONOS" + controllerStr +
1113 " Switches view is correct",
1114 onfail="ONOS" + controllerStr +
1115 " Switches view is incorrect" )
1116 if links[ controller ] and "Error" not in links[ controller ]:
1117 currentLinksResult = main.Mininet1.compareLinks(
1118 mnSwitches, mnLinks,
1119 json.loads( links[ controller ] ) )
1120 else:
1121 currentLinksResult = main.FALSE
1122 utilities.assert_equals( expect=main.TRUE,
1123 actual=currentLinksResult,
1124 onpass="ONOS" + controllerStr +
1125 " links view is correct",
1126 onfail="ONOS" + controllerStr +
1127 " links view is incorrect" )
1128
1129 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1130 currentHostsResult = main.Mininet1.compareHosts(
1131 mnHosts,
1132 hosts[ controller ] )
1133 else:
1134 currentHostsResult = main.FALSE
1135 utilities.assert_equals( expect=main.TRUE,
1136 actual=currentHostsResult,
1137 onpass="ONOS" + controllerStr +
1138 " hosts exist in Mininet",
1139 onfail="ONOS" + controllerStr +
1140 " hosts don't match Mininet" )
1141
1142 devicesResults = devicesResults and currentDevicesResult
1143 linksResults = linksResults and currentLinksResult
1144 hostsResults = hostsResults and currentHostsResult
1145
1146 main.step( "Device information is correct" )
1147 utilities.assert_equals(
1148 expect=main.TRUE,
1149 actual=devicesResults,
1150 onpass="Device information is correct",
1151 onfail="Device information is incorrect" )
1152
1153 main.step( "Links are correct" )
1154 utilities.assert_equals(
1155 expect=main.TRUE,
1156 actual=linksResults,
1157 onpass="Link are correct",
1158 onfail="Links are incorrect" )
1159
1160 main.step( "Hosts are correct" )
1161 utilities.assert_equals(
1162 expect=main.TRUE,
1163 actual=hostsResults,
1164 onpass="Hosts are correct",
1165 onfail="Hosts are incorrect" )
1166
1167 - def CASE6( self, main ):
1168 """
1169 The Failure case.
1170 """
1171 import time
1172 assert main.numCtrls, "main.numCtrls not defined"
1173 assert main, "main not defined"
1174 assert utilities.assert_equals, "utilities.assert_equals not defined"
1175
1176
1177 try:
1178 iCounterValue = 0
1179 except NameError:
1180 main.log.error( "iCounterValue not defined, setting to 0" )
1181 iCounterValue = 0
1182
1183 main.case( "Restart ONOS node" )
1184 main.caseExplanation = "Killing ONOS process and restart cli " +\
1185 "sessions once onos is up."
1186 main.step( "Killing ONOS processes" )
1187 killResult = main.ONOSbench.onosKill( main.nodes[0].ip_address )
1188 start = time.time()
1189 utilities.assert_equals( expect=main.TRUE, actual=killResult,
1190 onpass="ONOS Killed",
1191 onfail="Error killing ONOS" )
1192
1193 main.step( "Checking if ONOS is up yet" )
1194 count = 0
1195 while count < 10:
1196 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
1197 if onos1Isup == main.TRUE:
1198 elapsed = time.time() - start
1199 break
1200 else:
1201 count = count + 1
1202 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
1203 onpass="ONOS is back up",
1204 onfail="ONOS failed to start" )
1205
1206 main.log.step( "Starting ONOS CLI sessions" )
1207 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
1208 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1209 onpass="ONOS cli startup successful",
1210 onfail="ONOS cli startup failed" )
1211
1212 if elapsed:
1213 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
1214 str( elapsed ) )
1215 main.restartTime = elapsed
1216 else:
1217 main.restartTime = -1
1218 time.sleep( 5 )
1219
1220 main.ONOScli1.electionTestRun()
1221
1222 - def CASE7( self, main ):
1223 """
1224 Check state after ONOS failure
1225 """
1226 import json
1227 assert main.numCtrls, "main.numCtrls not defined"
1228 assert main, "main not defined"
1229 assert utilities.assert_equals, "utilities.assert_equals not defined"
1230 main.case( "Running ONOS Constant State Tests" )
1231 main.step( "Check that each switch has a master" )
1232
1233 rolesNotNull = main.ONOScli1.rolesNotNull()
1234 utilities.assert_equals(
1235 expect=main.TRUE,
1236 actual=rolesNotNull,
1237 onpass="Each device has a master",
1238 onfail="Some devices don't have a master assigned" )
1239
1240 main.step( "Check if switch roles are consistent across all nodes" )
1241 ONOS1Mastership = main.ONOScli1.roles()
1242
1243 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1244 main.log.error( "Error in getting ONOS mastership" )
1245 main.log.warn( "ONOS1 mastership response: " +
1246 repr( ONOS1Mastership ) )
1247 consistentMastership = main.FALSE
1248 else:
1249 consistentMastership = main.TRUE
1250 utilities.assert_equals(
1251 expect=main.TRUE,
1252 actual=consistentMastership,
1253 onpass="Switch roles are consistent across all ONOS nodes",
1254 onfail="ONOS nodes have different views of switch roles" )
1255
1256 description2 = "Compare switch roles from before failure"
1257 main.step( description2 )
1258
1259 currentJson = json.loads( ONOS1Mastership )
1260 oldJson = json.loads( mastershipState )
1261 mastershipCheck = main.TRUE
1262 for i in range( 1, 29 ):
1263 switchDPID = str(
1264 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1265
1266 current = [ switch[ 'master' ] for switch in currentJson
1267 if switchDPID in switch[ 'id' ] ]
1268 old = [ switch[ 'master' ] for switch in oldJson
1269 if switchDPID in switch[ 'id' ] ]
1270 if current == old:
1271 mastershipCheck = mastershipCheck and main.TRUE
1272 else:
1273 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1274 mastershipCheck = main.FALSE
1275 utilities.assert_equals(
1276 expect=main.TRUE,
1277 actual=mastershipCheck,
1278 onpass="Mastership of Switches was not changed",
1279 onfail="Mastership of some switches changed" )
1280 mastershipCheck = mastershipCheck and consistentMastership
1281
1282 main.step( "Get the intents and compare across all nodes" )
1283 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1284 intentCheck = main.FALSE
1285 if "Error" in ONOS1Intents or not ONOS1Intents:
1286 main.log.error( "Error in getting ONOS intents" )
1287 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1288 else:
1289 intentCheck = main.TRUE
1290 utilities.assert_equals(
1291 expect=main.TRUE,
1292 actual=intentCheck,
1293 onpass="Intents are consistent across all ONOS nodes",
1294 onfail="ONOS nodes have different views of intents" )
1295
1296 intents = []
1297 intents.append( ONOS1Intents )
1298 intentStates = []
1299 for node in intents:
1300 nodeStates = []
1301
1302 for intent in json.loads( node ):
1303 nodeStates.append( intent[ 'state' ] )
1304 intentStates.append( nodeStates )
1305 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1306 main.log.info( dict( out ) )
1307
1308
1309
1310 """
1311 main.step( "Compare current intents with intents before the failure" )
1312 # NOTE: this requires case 5 to pass for intentState to be set.
1313 # maybe we should stop the test if that fails?
1314 sameIntents = main.FALSE
1315 if intentState and intentState == ONOSIntents[ 0 ]:
1316 sameIntents = main.TRUE
1317 main.log.info( "Intents are consistent with before failure" )
1318 # TODO: possibly the states have changed? we may need to figure out
1319 # what the acceptable states are
1320 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1321 sameIntents = main.TRUE
1322 try:
1323 before = json.loads( intentState )
1324 after = json.loads( ONOSIntents[ 0 ] )
1325 for intent in before:
1326 if intent not in after:
1327 sameIntents = main.FALSE
1328 main.log.debug( "Intent is not currently in ONOS " +
1329 "(at least in the same form):" )
1330 main.log.debug( json.dumps( intent ) )
1331 except ( ValueError, TypeError ):
1332 main.log.exception( "Exception printing intents" )
1333 main.log.debug( repr( ONOSIntents[0] ) )
1334 main.log.debug( repr( intentState ) )
1335 if sameIntents == main.FALSE:
1336 try:
1337 main.log.debug( "ONOS intents before: " )
1338 main.log.debug( json.dumps( json.loads( intentState ),
1339 sort_keys=True, indent=4,
1340 separators=( ',', ': ' ) ) )
1341 main.log.debug( "Current ONOS intents: " )
1342 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1343 sort_keys=True, indent=4,
1344 separators=( ',', ': ' ) ) )
1345 except ( ValueError, TypeError ):
1346 main.log.exception( "Exception printing intents" )
1347 main.log.debug( repr( ONOSIntents[0] ) )
1348 main.log.debug( repr( intentState ) )
1349 utilities.assert_equals(
1350 expect=main.TRUE,
1351 actual=sameIntents,
1352 onpass="Intents are consistent with before failure",
1353 onfail="The Intents changed during failure" )
1354 intentCheck = intentCheck and sameIntents
1355 """
1356 main.step( "Get the OF Table entries and compare to before " +
1357 "component failure" )
1358 FlowTables = main.TRUE
1359 flows2 = []
1360 for i in range( 28 ):
1361 main.log.info( "Checking flow table on s" + str( i + 1 ) )
1362 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
1363 flows2.append( tmpFlows )
1364 tempResult = main.Mininet1.flowComp(
1365 flow1=flows[ i ],
1366 flow2=tmpFlows )
1367 FlowTables = FlowTables and tempResult
1368 if FlowTables == main.FALSE:
1369 main.log.info( "Differences in flow table for switch: s" +
1370 str( i + 1 ) )
1371 utilities.assert_equals(
1372 expect=main.TRUE,
1373 actual=FlowTables,
1374 onpass="No changes were found in the flow tables",
1375 onfail="Changes were found in the flow tables" )
1376
1377 main.step( "Leadership Election is still functional" )
1378
1379
1380 leader = main.nodes[0].ip_address
1381 leaderResult = main.TRUE
1382 for controller in range( 1, main.numCtrls + 1 ):
1383
1384 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
1385 leaderN = node.electionTestLeader()
1386
1387
1388
1389 if leaderN == leader:
1390
1391 pass
1392 elif leaderN == main.FALSE:
1393
1394 main.log.error( "Something is wrong with " +
1395 "electionTestLeader function, check the" +
1396 " error logs" )
1397 leaderResult = main.FALSE
1398 elif leader != leaderN:
1399 leaderResult = main.FALSE
1400 main.log.error( "ONOS" + str( controller ) + " sees " +
1401 str( leaderN ) +
1402 " as the leader of the election app. " +
1403 "Leader should be " + str( leader ) )
1404 utilities.assert_equals(
1405 expect=main.TRUE,
1406 actual=leaderResult,
1407 onpass="Leadership election passed",
1408 onfail="Something went wrong with Leadership election" )
1409
1410 - def CASE8( self, main ):
1411 """
1412 Compare topo
1413 """
1414 import json
1415 import time
1416 assert main.numCtrls, "main.numCtrls not defined"
1417 assert main, "main not defined"
1418 assert utilities.assert_equals, "utilities.assert_equals not defined"
1419
1420 main.case( "Compare ONOS Topology view to Mininet topology" )
1421 main.caseExplanation = "Compare topology objects between Mininet" +\
1422 " and ONOS"
1423
1424 main.step( "Comparing ONOS topology to MN" )
1425 devicesResults = main.TRUE
1426 linksResults = main.TRUE
1427 hostsResults = main.TRUE
1428 hostAttachmentResults = True
1429 topoResult = main.FALSE
1430 elapsed = 0
1431 count = 0
1432 main.step( "Collecting topology information from ONOS" )
1433 startTime = time.time()
1434
1435 while topoResult == main.FALSE and elapsed < 60:
1436 count += 1
1437 cliStart = time.time()
1438 devices = []
1439 devices.append( main.ONOScli1.devices() )
1440 hosts = []
1441 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1442 ipResult = main.TRUE
1443 for controller in range( 0, len( hosts ) ):
1444 controllerStr = str( controller + 1 )
1445 for host in hosts[ controller ]:
1446 if host is None or host.get( 'ipAddresses', [] ) == []:
1447 main.log.error(
1448 "DEBUG:Error with host ips on controller" +
1449 controllerStr + ": " + str( host ) )
1450 ipResult = main.FALSE
1451 ports = []
1452 ports.append( main.ONOScli1.ports() )
1453 links = []
1454 links.append( main.ONOScli1.links() )
1455 clusters = []
1456 clusters.append( main.ONOScli1.clusters() )
1457
1458 elapsed = time.time() - startTime
1459 cliTime = time.time() - cliStart
1460 print "CLI time: " + str( cliTime )
1461
1462 mnSwitches = main.Mininet1.getSwitches()
1463 mnLinks = main.Mininet1.getLinks()
1464 mnHosts = main.Mininet1.getHosts()
1465 for controller in range( main.numCtrls ):
1466 controllerStr = str( controller + 1 )
1467 if devices[ controller ] and ports[ controller ] and\
1468 "Error" not in devices[ controller ] and\
1469 "Error" not in ports[ controller ]:
1470
1471 currentDevicesResult = main.Mininet1.compareSwitches(
1472 mnSwitches,
1473 json.loads( devices[ controller ] ),
1474 json.loads( ports[ controller ] ) )
1475 else:
1476 currentDevicesResult = main.FALSE
1477 utilities.assert_equals( expect=main.TRUE,
1478 actual=currentDevicesResult,
1479 onpass="ONOS" + controllerStr +
1480 " Switches view is correct",
1481 onfail="ONOS" + controllerStr +
1482 " Switches view is incorrect" )
1483
1484 if links[ controller ] and "Error" not in links[ controller ]:
1485 currentLinksResult = main.Mininet1.compareLinks(
1486 mnSwitches, mnLinks,
1487 json.loads( links[ controller ] ) )
1488 else:
1489 currentLinksResult = main.FALSE
1490 utilities.assert_equals( expect=main.TRUE,
1491 actual=currentLinksResult,
1492 onpass="ONOS" + controllerStr +
1493 " links view is correct",
1494 onfail="ONOS" + controllerStr +
1495 " links view is incorrect" )
1496
1497 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1498 currentHostsResult = main.Mininet1.compareHosts(
1499 mnHosts,
1500 hosts[ controller ] )
1501 else:
1502 currentHostsResult = main.FALSE
1503 utilities.assert_equals( expect=main.TRUE,
1504 actual=currentHostsResult,
1505 onpass="ONOS" + controllerStr +
1506 " hosts exist in Mininet",
1507 onfail="ONOS" + controllerStr +
1508 " hosts don't match Mininet" )
1509
1510 hostAttachment = True
1511 zeroHosts = False
1512
1513
1514 mappings = {}
1515 for i in range( 1, 29 ):
1516
1517 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
1518 if i == 1:
1519 deviceId = "1000".zfill(16)
1520 elif i == 2:
1521 deviceId = "2000".zfill(16)
1522 elif i == 3:
1523 deviceId = "3000".zfill(16)
1524 elif i == 4:
1525 deviceId = "3004".zfill(16)
1526 elif i == 5:
1527 deviceId = "5000".zfill(16)
1528 elif i == 6:
1529 deviceId = "6000".zfill(16)
1530 elif i == 7:
1531 deviceId = "6007".zfill(16)
1532 elif i >= 8 and i <= 17:
1533 dpid = '3' + str( i ).zfill( 3 )
1534 deviceId = dpid.zfill(16)
1535 elif i >= 18 and i <= 27:
1536 dpid = '6' + str( i ).zfill( 3 )
1537 deviceId = dpid.zfill(16)
1538 elif i == 28:
1539 deviceId = "2800".zfill(16)
1540 mappings[ macId ] = deviceId
1541 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1542 if hosts[ controller ] == []:
1543 main.log.warn( "There are no hosts discovered" )
1544 zeroHosts = True
1545 else:
1546 for host in hosts[ controller ]:
1547 mac = None
1548 location = None
1549 device = None
1550 port = None
1551 try:
1552 mac = host.get( 'mac' )
1553 assert mac, "mac field could not be found for this host object"
1554
1555 location = host.get( 'location' )
1556 assert location, "location field could not be found for this host object"
1557
1558
1559 device = str( location.get( 'elementId' ) ).split(':')[1]
1560 assert device, "elementId field could not be found for this host location object"
1561
1562 port = location.get( 'port' )
1563 assert port, "port field could not be found for this host location object"
1564
1565
1566 if mac and device and port:
1567 if str( port ) != "1":
1568 main.log.error( "The attachment port is incorrect for " +
1569 "host " + str( mac ) +
1570 ". Expected: 1 Actual: " + str( port) )
1571 hostAttachment = False
1572 if device != mappings[ str( mac ) ]:
1573 main.log.error( "The attachment device is incorrect for " +
1574 "host " + str( mac ) +
1575 ". Expected: " + mappings[ str( mac ) ] +
1576 " Actual: " + device )
1577 hostAttachment = False
1578 else:
1579 hostAttachment = False
1580 except AssertionError:
1581 main.log.exception( "Json object not as expected" )
1582 main.log.error( repr( host ) )
1583 hostAttachment = False
1584 else:
1585 main.log.error( "No hosts json output or \"Error\"" +
1586 " in output. hosts = " +
1587 repr( hosts[ controller ] ) )
1588 if zeroHosts is False:
1589 hostAttachment = True
1590
1591 devicesResults = devicesResults and currentDevicesResult
1592 linksResults = linksResults and currentLinksResult
1593 hostsResults = hostsResults and currentHostsResult
1594 hostAttachmentResults = hostAttachmentResults and\
1595 hostAttachment
1596
1597
1598
1599 numClusters = len( json.loads( clusters[ 0 ] ) )
1600 clusterResults = main.FALSE
1601 if numClusters == 1:
1602 clusterResults = main.TRUE
1603 utilities.assert_equals(
1604 expect=1,
1605 actual=numClusters,
1606 onpass="ONOS shows 1 SCC",
1607 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1608
1609 topoResult = ( devicesResults and linksResults
1610 and hostsResults and ipResult and clusterResults and
1611 hostAttachmentResults )
1612
1613 topoResult = topoResult and int( count <= 2 )
1614 note = "note it takes about " + str( int( cliTime ) ) + \
1615 " seconds for the test to make all the cli calls to fetch " +\
1616 "the topology from each ONOS instance"
1617 main.log.info(
1618 "Very crass estimate for topology discovery/convergence( " +
1619 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1620 str( count ) + " tries" )
1621 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1622 onpass="Topology Check Test successful",
1623 onfail="Topology Check Test NOT successful" )
1624
1625 - def CASE9( self, main ):
1626 """
1627 Link s3-s28 down
1628 """
1629 import time
1630 assert main.numCtrls, "main.numCtrls not defined"
1631 assert main, "main not defined"
1632 assert utilities.assert_equals, "utilities.assert_equals not defined"
1633
1634
1635 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1636
1637 description = "Turn off a link to ensure that Link Discovery " +\
1638 "is working properly"
1639 main.case( description )
1640
1641 main.step( "Kill Link between s3 and s28" )
1642 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
1643 main.log.info( "Waiting " + str( linkSleep ) +
1644 " seconds for link down to be discovered" )
1645 time.sleep( linkSleep )
1646 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
1647 onpass="Link down successful",
1648 onfail="Failed to bring link down" )
1649
1650
1652 """
1653 Link s3-s28 up
1654 """
1655 import time
1656 assert main.numCtrls, "main.numCtrls not defined"
1657 assert main, "main not defined"
1658 assert utilities.assert_equals, "utilities.assert_equals not defined"
1659
1660
1661 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1662
1663 description = "Restore a link to ensure that Link Discovery is " + \
1664 "working properly"
1665 main.case( description )
1666
1667 main.step( "Bring link between s3 and s28 back up" )
1668 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
1669 main.log.info( "Waiting " + str( linkSleep ) +
1670 " seconds for link up to be discovered" )
1671 time.sleep( linkSleep )
1672 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
1673 onpass="Link up successful",
1674 onfail="Failed to bring link up" )
1675
1676
1678 """
1679 Switch Down
1680 """
1681
1682 import time
1683 assert main.numCtrls, "main.numCtrls not defined"
1684 assert main, "main not defined"
1685 assert utilities.assert_equals, "utilities.assert_equals not defined"
1686
1687 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1688
1689 description = "Killing a switch to ensure it is discovered correctly"
1690 main.case( description )
1691 switch = main.params[ 'kill' ][ 'switch' ]
1692 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1693
1694
1695 main.step( "Kill " + switch )
1696 main.log.info( "Deleting " + switch )
1697 main.Mininet1.delSwitch( switch )
1698 main.log.info( "Waiting " + str( switchSleep ) +
1699 " seconds for switch down to be discovered" )
1700 time.sleep( switchSleep )
1701 device = main.ONOScli1.getDevice( dpid=switchDPID )
1702
1703 main.log.warn( str( device ) )
1704 result = main.FALSE
1705 if device and device[ 'available' ] is False:
1706 result = main.TRUE
1707 utilities.assert_equals( expect=main.TRUE, actual=result,
1708 onpass="Kill switch successful",
1709 onfail="Failed to kill switch?" )
1710
1712 """
1713 Switch Up
1714 """
1715
1716 import time
1717 assert main.numCtrls, "main.numCtrls not defined"
1718 assert main, "main not defined"
1719 assert utilities.assert_equals, "utilities.assert_equals not defined"
1720
1721 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1722 switch = main.params[ 'kill' ][ 'switch' ]
1723 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1724 links = main.params[ 'kill' ][ 'links' ].split()
1725 description = "Adding a switch to ensure it is discovered correctly"
1726 main.case( description )
1727
1728 main.step( "Add back " + switch )
1729 main.Mininet1.addSwitch( switch, dpid=switchDPID )
1730 for peer in links:
1731 main.Mininet1.addLink( switch, peer )
1732 ipList = []
1733 for i in range( main.numCtrls ):
1734 ipList.append( main.nodes[ i ].ip_address )
1735 main.Mininet1.assignSwController( sw=switch, ip=ipList )
1736 main.log.info( "Waiting " + str( switchSleep ) +
1737 " seconds for switch up to be discovered" )
1738 time.sleep( switchSleep )
1739 device = main.ONOScli1.getDevice( dpid=switchDPID )
1740
1741 main.log.warn( str( device ) )
1742 result = main.FALSE
1743 if device and device[ 'available' ]:
1744 result = main.TRUE
1745 utilities.assert_equals( expect=main.TRUE, actual=result,
1746 onpass="add switch successful",
1747 onfail="Failed to add switch?" )
1748
1750 """
1751 Clean up
1752 """
1753 import os
1754 import time
1755 assert main.numCtrls, "main.numCtrls not defined"
1756 assert main, "main not defined"
1757 assert utilities.assert_equals, "utilities.assert_equals not defined"
1758
1759 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
1760 'blue': '\033[94m', 'green': '\033[92m',
1761 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
1762 main.case( "Test Cleanup" )
1763 main.step( "Killing tcpdumps" )
1764 main.Mininet2.stopTcpdump()
1765
1766 testname = main.TEST
1767 if main.params[ 'BACKUP' ] == "True":
1768 main.step( "Copying MN pcap and ONOS log files to test station" )
1769 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
1770 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
1771
1772
1773
1774
1775
1776
1777 logFolder = "/opt/onos/log/"
1778 logFiles = [ "karaf.log", "karaf.log.1" ]
1779
1780 dstDir = "~/packet_captures/"
1781 for f in logFiles:
1782 main.ONOSbench.handle.sendline( "scp sdn@" + main.nodes[0].ip_address + ":" +
1783 logFolder + f + " " +
1784 teststationUser + "@" +
1785 teststationIP + ":" + dstDir +
1786 str( testname ) + "-ONOS1-" + f )
1787 main.ONOSbench.handle.expect( "\$" )
1788
1789
1790
1791 logFolder = "/opt/onos/var/"
1792 logFiles = [ "stderr.log", "stdout.log" ]
1793
1794 dstDir = "~/packet_captures/"
1795 for f in logFiles:
1796 main.ONOSbench.handle.sendline( "scp sdn@" + main.nodes[0].ip_address + ":" +
1797 logFolder + f + " " +
1798 teststationUser + "@" +
1799 teststationIP + ":" + dstDir +
1800 str( testname ) + "-ONOS1-" + f )
1801 main.ONOSbench.handle.expect( "\$" )
1802
1803 time.sleep( 10 )
1804 main.step( "Packing and rotating pcap archives" )
1805 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
1806
1807 main.step( "Stopping Mininet" )
1808 mnResult = main.Mininet1.stopNet()
1809 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
1810 onpass="Mininet stopped",
1811 onfail="MN cleanup NOT successful" )
1812
1813 main.step( "Checking ONOS Logs for errors" )
1814 print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
1815 colors[ 'end' ]
1816 print main.ONOSbench.checkLogs( main.nodes[0].ip_address, restart=True )
1817
1818 try:
1819 timerLog = open( main.logdir + "/Timers.csv", 'w')
1820
1821 labels = "Gossip Intents, Restart"
1822 data = str( gossipTime ) + ", " + str( main.restartTime )
1823 timerLog.write( labels + "\n" + data )
1824 timerLog.close()
1825 except NameError, e:
1826 main.log.exception(e)
1827
1829 """
1830 start election app on all onos nodes
1831 """
1832 assert main.numCtrls, "main.numCtrls not defined"
1833 assert main, "main not defined"
1834 assert utilities.assert_equals, "utilities.assert_equals not defined"
1835
1836 main.case("Start Leadership Election app")
1837 main.step( "Install leadership election app" )
1838 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
1839 utilities.assert_equals(
1840 expect=main.TRUE,
1841 actual=appResult,
1842 onpass="Election app installed",
1843 onfail="Something went wrong with installing Leadership election" )
1844
1845 main.step( "Run for election on each node" )
1846 leaderResult = main.ONOScli1.electionTestRun()
1847
1848 leader = main.ONOScli1.electionTestLeader()
1849
1850 if leader == main.nodes[0].ip_address:
1851
1852 pass
1853 elif leader is None:
1854
1855 main.log.error( "No leader was elected" )
1856 leaderResult = main.FALSE
1857 elif leader == main.FALSE:
1858
1859
1860
1861 main.log.error( "Something is wrong with electionTestLeader" +
1862 " function, check the error logs" )
1863 leaderResult = main.FALSE
1864 else:
1865
1866 main.log.error(
1867 "Unexpected response from electionTestLeader function:'" +
1868 str( leader ) +
1869 "'" )
1870 leaderResult = main.FALSE
1871 utilities.assert_equals(
1872 expect=main.TRUE,
1873 actual=leaderResult,
1874 onpass="Successfully ran for leadership",
1875 onfail="Failed to run for leadership" )
1876
1878 """
1879 Check that Leadership Election is still functional
1880 """
1881 assert main.numCtrls, "main.numCtrls not defined"
1882 assert main, "main not defined"
1883 assert utilities.assert_equals, "utilities.assert_equals not defined"
1884 leaderResult = main.TRUE
1885 description = "Check that Leadership Election is still functional"
1886 main.case( description )
1887 main.step( "Find current leader and withdraw" )
1888 leader = main.ONOScli1.electionTestLeader()
1889
1890 withdrawResult = main.FALSE
1891 if leader == main.nodes[0].ip_address:
1892 oldLeader = getattr( main, "ONOScli1" )
1893 elif leader is None or leader == main.FALSE:
1894 main.log.error(
1895 "Leader for the election app should be an ONOS node," +
1896 "instead got '" + str( leader ) + "'" )
1897 leaderResult = main.FALSE
1898 oldLeader = None
1899 else:
1900 main.log.error( "Leader election --- why am I HERE?!?")
1901 leaderResult = main.FALSE
1902 oldLeader = None
1903 if oldLeader:
1904 withdrawResult = oldLeader.electionTestWithdraw()
1905 utilities.assert_equals(
1906 expect=main.TRUE,
1907 actual=withdrawResult,
1908 onpass="Node was withdrawn from election",
1909 onfail="Node was not withdrawn from election" )
1910
1911 main.step( "Make sure new leader is elected" )
1912 leaderN = main.ONOScli1.electionTestLeader()
1913 if leaderN == leader:
1914 main.log.error( "ONOS still sees " + str( leaderN ) +
1915 " as leader after they withdrew" )
1916 leaderResult = main.FALSE
1917 elif leaderN == main.FALSE:
1918
1919
1920
1921 main.log.error( "Something is wrong with electionTestLeader " +
1922 "function, check the error logs" )
1923 leaderResult = main.FALSE
1924 elif leaderN is None:
1925 main.log.info(
1926 "There is no leader after the app withdrew from election" )
1927 leaderResult = main.TRUE
1928 utilities.assert_equals(
1929 expect=main.TRUE,
1930 actual=leaderResult,
1931 onpass="Leadership election passed",
1932 onfail="Something went wrong with Leadership election" )
1933
1934 main.step( "Run for election on old leader( just so everyone " +
1935 "is in the hat )" )
1936 if oldLeader:
1937 runResult = oldLeader.electionTestRun()
1938 else:
1939 runResult = main.FALSE
1940 utilities.assert_equals(
1941 expect=main.TRUE,
1942 actual=runResult,
1943 onpass="App re-ran for election",
1944 onfail="App failed to run for election" )
1945
1946 main.step( "Node became leader when it ran for election" )
1947 afterRun = main.ONOScli1.electionTestLeader()
1948
1949 if afterRun == main.nodes[0].ip_address:
1950 afterResult = main.TRUE
1951 else:
1952 afterResult = main.FALSE
1953
1954 utilities.assert_equals(
1955 expect=main.TRUE,
1956 actual=afterResult,
1957 onpass="Old leader successfully re-ran for election",
1958 onfail="Something went wrong with Leadership election after " +
1959 "the old leader re-ran for election" )
1960
1962 """
1963 Install Distributed Primitives app
1964 """
1965 assert main.numCtrls, "main.numCtrls not defined"
1966 assert main, "main not defined"
1967 assert utilities.assert_equals, "utilities.assert_equals not defined"
1968 assert main.CLIs, "main.CLIs not defined"
1969 assert main.nodes, "main.nodes not defined"
1970
1971
1972 global pCounterName
1973 global iCounterName
1974 global pCounterValue
1975 global iCounterValue
1976 global onosSet
1977 global onosSetName
1978 pCounterName = "TestON-Partitions"
1979 iCounterName = "TestON-inMemory"
1980 pCounterValue = 0
1981 iCounterValue = 0
1982 onosSet = set([])
1983 onosSetName = "TestON-set"
1984
1985 description = "Install Primitives app"
1986 main.case( description )
1987 main.step( "Install Primitives app" )
1988 appName = "org.onosproject.distributedprimitives"
1989 appResults = main.CLIs[0].activateApp( appName )
1990 utilities.assert_equals( expect=main.TRUE,
1991 actual=appResults,
1992 onpass="Primitives app activated",
1993 onfail="Primitives app not activated" )
1994
1996 """
1997 Check for basic functionality with distributed primitives
1998 """
1999
2000 assert main.numCtrls, "main.numCtrls not defined"
2001 assert main, "main not defined"
2002 assert utilities.assert_equals, "utilities.assert_equals not defined"
2003 assert main.CLIs, "main.CLIs not defined"
2004 assert main.nodes, "main.nodes not defined"
2005 assert pCounterName, "pCounterName not defined"
2006 assert iCounterName, "iCounterName not defined"
2007 assert onosSetName, "onosSetName not defined"
2008
2009 try:
2010 pCounterValue
2011 except NameError:
2012 main.log.error( "pCounterValue not defined, setting to 0" )
2013 pCounterValue = 0
2014 try:
2015 iCounterValue
2016 except NameError:
2017 main.log.error( "iCounterValue not defined, setting to 0" )
2018 iCounterValue = 0
2019 try:
2020 onosSet
2021 except NameError:
2022 main.log.error( "onosSet not defined, setting to empty Set" )
2023 onosSet = set([])
2024
2025 addValue = "a"
2026 addAllValue = "a b c d e f"
2027 retainValue = "c d e f"
2028
2029 description = "Check for basic functionality with distributed " +\
2030 "primitives"
2031 main.case( description )
2032 main.caseExplanation = "Test the methods of the distributed " +\
2033 "primitives (counters and sets) throught the cli"
2034
2035
2036 main.step( "Increment then get a default counter on each node" )
2037 pCounters = []
2038 threads = []
2039 addedPValues = []
2040 for i in range( main.numCtrls ):
2041 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2042 name="counterAddAndGet-" + str( i ),
2043 args=[ pCounterName ] )
2044 pCounterValue += 1
2045 addedPValues.append( pCounterValue )
2046 threads.append( t )
2047 t.start()
2048
2049 for t in threads:
2050 t.join()
2051 pCounters.append( t.result )
2052
2053 pCounterResults = True
2054 for i in addedPValues:
2055 tmpResult = i in pCounters
2056 pCounterResults = pCounterResults and tmpResult
2057 if not tmpResult:
2058 main.log.error( str( i ) + " is not in partitioned "
2059 "counter incremented results" )
2060 utilities.assert_equals( expect=True,
2061 actual=pCounterResults,
2062 onpass="Default counter incremented",
2063 onfail="Error incrementing default" +
2064 " counter" )
2065
2066 main.step( "Get then Increment a default counter on each node" )
2067 pCounters = []
2068 threads = []
2069 addedPValues = []
2070 for i in range( main.numCtrls ):
2071 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2072 name="counterGetAndAdd-" + str( i ),
2073 args=[ pCounterName ] )
2074 addedPValues.append( pCounterValue )
2075 pCounterValue += 1
2076 threads.append( t )
2077 t.start()
2078
2079 for t in threads:
2080 t.join()
2081 pCounters.append( t.result )
2082
2083 pCounterResults = True
2084 for i in addedPValues:
2085 tmpResult = i in pCounters
2086 pCounterResults = pCounterResults and tmpResult
2087 if not tmpResult:
2088 main.log.error( str( i ) + " is not in partitioned "
2089 "counter incremented results" )
2090 utilities.assert_equals( expect=True,
2091 actual=pCounterResults,
2092 onpass="Default counter incremented",
2093 onfail="Error incrementing default" +
2094 " counter" )
2095
2096 main.step( "Counters we added have the correct values" )
2097 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2098 utilities.assert_equals( expect=main.TRUE,
2099 actual=incrementCheck,
2100 onpass="Added counters are correct",
2101 onfail="Added counters are incorrect" )
2102
2103 main.step( "Add -8 to then get a default counter on each node" )
2104 pCounters = []
2105 threads = []
2106 addedPValues = []
2107 for i in range( main.numCtrls ):
2108 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2109 name="counterIncrement-" + str( i ),
2110 args=[ pCounterName ],
2111 kwargs={ "delta": -8 } )
2112 pCounterValue += -8
2113 addedPValues.append( pCounterValue )
2114 threads.append( t )
2115 t.start()
2116
2117 for t in threads:
2118 t.join()
2119 pCounters.append( t.result )
2120
2121 pCounterResults = True
2122 for i in addedPValues:
2123 tmpResult = i in pCounters
2124 pCounterResults = pCounterResults and tmpResult
2125 if not tmpResult:
2126 main.log.error( str( i ) + " is not in partitioned "
2127 "counter incremented results" )
2128 utilities.assert_equals( expect=True,
2129 actual=pCounterResults,
2130 onpass="Default counter incremented",
2131 onfail="Error incrementing default" +
2132 " counter" )
2133
2134 main.step( "Add 5 to then get a default counter on each node" )
2135 pCounters = []
2136 threads = []
2137 addedPValues = []
2138 for i in range( main.numCtrls ):
2139 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2140 name="counterIncrement-" + str( i ),
2141 args=[ pCounterName ],
2142 kwargs={ "delta": 5 } )
2143 pCounterValue += 5
2144 addedPValues.append( pCounterValue )
2145 threads.append( t )
2146 t.start()
2147
2148 for t in threads:
2149 t.join()
2150 pCounters.append( t.result )
2151
2152 pCounterResults = True
2153 for i in addedPValues:
2154 tmpResult = i in pCounters
2155 pCounterResults = pCounterResults and tmpResult
2156 if not tmpResult:
2157 main.log.error( str( i ) + " is not in partitioned "
2158 "counter incremented results" )
2159 utilities.assert_equals( expect=True,
2160 actual=pCounterResults,
2161 onpass="Default counter incremented",
2162 onfail="Error incrementing default" +
2163 " counter" )
2164
2165 main.step( "Get then add 5 to a default counter on each node" )
2166 pCounters = []
2167 threads = []
2168 addedPValues = []
2169 for i in range( main.numCtrls ):
2170 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2171 name="counterIncrement-" + str( i ),
2172 args=[ pCounterName ],
2173 kwargs={ "delta": 5 } )
2174 addedPValues.append( pCounterValue )
2175 pCounterValue += 5
2176 threads.append( t )
2177 t.start()
2178
2179 for t in threads:
2180 t.join()
2181 pCounters.append( t.result )
2182
2183 pCounterResults = True
2184 for i in addedPValues:
2185 tmpResult = i in pCounters
2186 pCounterResults = pCounterResults and tmpResult
2187 if not tmpResult:
2188 main.log.error( str( i ) + " is not in partitioned "
2189 "counter incremented results" )
2190 utilities.assert_equals( expect=True,
2191 actual=pCounterResults,
2192 onpass="Default counter incremented",
2193 onfail="Error incrementing default" +
2194 " counter" )
2195
2196 main.step( "Counters we added have the correct values" )
2197 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2198 utilities.assert_equals( expect=main.TRUE,
2199 actual=incrementCheck,
2200 onpass="Added counters are correct",
2201 onfail="Added counters are incorrect" )
2202
2203
2204 main.step( "Increment and get an in-memory counter on each node" )
2205 iCounters = []
2206 addedIValues = []
2207 threads = []
2208 for i in range( main.numCtrls ):
2209 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2210 name="icounterIncrement-" + str( i ),
2211 args=[ iCounterName ],
2212 kwargs={ "inMemory": True } )
2213 iCounterValue += 1
2214 addedIValues.append( iCounterValue )
2215 threads.append( t )
2216 t.start()
2217
2218 for t in threads:
2219 t.join()
2220 iCounters.append( t.result )
2221
2222 iCounterResults = True
2223 for i in addedIValues:
2224 tmpResult = i in iCounters
2225 iCounterResults = iCounterResults and tmpResult
2226 if not tmpResult:
2227 main.log.error( str( i ) + " is not in the in-memory "
2228 "counter incremented results" )
2229 utilities.assert_equals( expect=True,
2230 actual=iCounterResults,
2231 onpass="In-memory counter incremented",
2232 onfail="Error incrementing in-memory" +
2233 " counter" )
2234
2235 main.step( "Get then Increment a in-memory counter on each node" )
2236 iCounters = []
2237 threads = []
2238 addedIValues = []
2239 for i in range( main.numCtrls ):
2240 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2241 name="counterGetAndAdd-" + str( i ),
2242 args=[ iCounterName ],
2243 kwargs={ "inMemory": True } )
2244 addedIValues.append( iCounterValue )
2245 iCounterValue += 1
2246 threads.append( t )
2247 t.start()
2248
2249 for t in threads:
2250 t.join()
2251 iCounters.append( t.result )
2252
2253 iCounterResults = True
2254 for i in addedIValues:
2255 tmpResult = i in iCounters
2256 iCounterResults = iCounterResults and tmpResult
2257 if not tmpResult:
2258 main.log.error( str( i ) + " is not in in-memory "
2259 "counter incremented results" )
2260 utilities.assert_equals( expect=True,
2261 actual=iCounterResults,
2262 onpass="In-memory counter incremented",
2263 onfail="Error incrementing in-memory" +
2264 " counter" )
2265
2266 main.step( "Counters we added have the correct values" )
2267 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2268 utilities.assert_equals( expect=main.TRUE,
2269 actual=incrementCheck,
2270 onpass="Added counters are correct",
2271 onfail="Added counters are incorrect" )
2272
2273 main.step( "Add -8 to then get a in-memory counter on each node" )
2274 iCounters = []
2275 threads = []
2276 addedIValues = []
2277 for i in range( main.numCtrls ):
2278 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2279 name="counterIncrement-" + str( i ),
2280 args=[ iCounterName ],
2281 kwargs={ "delta": -8, "inMemory": True } )
2282 iCounterValue += -8
2283 addedIValues.append( iCounterValue )
2284 threads.append( t )
2285 t.start()
2286
2287 for t in threads:
2288 t.join()
2289 iCounters.append( t.result )
2290
2291 iCounterResults = True
2292 for i in addedIValues:
2293 tmpResult = i in iCounters
2294 iCounterResults = iCounterResults and tmpResult
2295 if not tmpResult:
2296 main.log.error( str( i ) + " is not in in-memory "
2297 "counter incremented results" )
2298 utilities.assert_equals( expect=True,
2299 actual=pCounterResults,
2300 onpass="In-memory counter incremented",
2301 onfail="Error incrementing in-memory" +
2302 " counter" )
2303
2304 main.step( "Add 5 to then get a in-memory counter on each node" )
2305 iCounters = []
2306 threads = []
2307 addedIValues = []
2308 for i in range( main.numCtrls ):
2309 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2310 name="counterIncrement-" + str( i ),
2311 args=[ iCounterName ],
2312 kwargs={ "delta": 5, "inMemory": True } )
2313 iCounterValue += 5
2314 addedIValues.append( iCounterValue )
2315 threads.append( t )
2316 t.start()
2317
2318 for t in threads:
2319 t.join()
2320 iCounters.append( t.result )
2321
2322 iCounterResults = True
2323 for i in addedIValues:
2324 tmpResult = i in iCounters
2325 iCounterResults = iCounterResults and tmpResult
2326 if not tmpResult:
2327 main.log.error( str( i ) + " is not in in-memory "
2328 "counter incremented results" )
2329 utilities.assert_equals( expect=True,
2330 actual=pCounterResults,
2331 onpass="In-memory counter incremented",
2332 onfail="Error incrementing in-memory" +
2333 " counter" )
2334
2335 main.step( "Get then add 5 to a in-memory counter on each node" )
2336 iCounters = []
2337 threads = []
2338 addedIValues = []
2339 for i in range( main.numCtrls ):
2340 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2341 name="counterIncrement-" + str( i ),
2342 args=[ iCounterName ],
2343 kwargs={ "delta": 5, "inMemory": True } )
2344 addedIValues.append( iCounterValue )
2345 iCounterValue += 5
2346 threads.append( t )
2347 t.start()
2348
2349 for t in threads:
2350 t.join()
2351 iCounters.append( t.result )
2352
2353 iCounterResults = True
2354 for i in addedIValues:
2355 tmpResult = i in iCounters
2356 iCounterResults = iCounterResults and tmpResult
2357 if not tmpResult:
2358 main.log.error( str( i ) + " is not in in-memory "
2359 "counter incremented results" )
2360 utilities.assert_equals( expect=True,
2361 actual=iCounterResults,
2362 onpass="In-memory counter incremented",
2363 onfail="Error incrementing in-memory" +
2364 " counter" )
2365
2366 main.step( "Counters we added have the correct values" )
2367 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2368 utilities.assert_equals( expect=main.TRUE,
2369 actual=incrementCheck,
2370 onpass="Added counters are correct",
2371 onfail="Added counters are incorrect" )
2372
2373 main.step( "Check counters are consistant across nodes" )
2374 onosCounters = []
2375 threads = []
2376 for i in range( main.numCtrls ):
2377 t = main.Thread( target=main.CLIs[i].counters,
2378 name="counters-" + str( i ) )
2379 threads.append( t )
2380 t.start()
2381 for t in threads:
2382 t.join()
2383 onosCounters.append( t.result )
2384 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
2385 if all( tmp ):
2386 main.log.info( "Counters are consistent across all nodes" )
2387 consistentCounterResults = main.TRUE
2388 else:
2389 main.log.error( "Counters are not consistent across all nodes" )
2390 consistentCounterResults = main.FALSE
2391 utilities.assert_equals( expect=main.TRUE,
2392 actual=consistentCounterResults,
2393 onpass="ONOS counters are consistent " +
2394 "across nodes",
2395 onfail="ONOS Counters are inconsistent " +
2396 "across nodes" )
2397
2398 main.step( "Counters we added have the correct values" )
2399 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2400 incrementCheck = incrementCheck and \
2401 main.Counters.counterCheck( iCounterName, iCounterValue )
2402 utilities.assert_equals( expect=main.TRUE,
2403 actual=incrementCheck,
2404 onpass="Added counters are correct",
2405 onfail="Added counters are incorrect" )
2406
2407
2408 main.step( "Distributed Set get" )
2409 size = len( onosSet )
2410 getResponses = []
2411 threads = []
2412 for i in range( main.numCtrls ):
2413 t = main.Thread( target=main.CLIs[i].setTestGet,
2414 name="setTestGet-" + str( i ),
2415 args=[ onosSetName ] )
2416 threads.append( t )
2417 t.start()
2418 for t in threads:
2419 t.join()
2420 getResponses.append( t.result )
2421
2422 getResults = main.TRUE
2423 for i in range( main.numCtrls ):
2424 if isinstance( getResponses[ i ], list):
2425 current = set( getResponses[ i ] )
2426 if len( current ) == len( getResponses[ i ] ):
2427
2428 if onosSet != current:
2429 main.log.error( "ONOS" + str( i + 1 ) +
2430 " has incorrect view" +
2431 " of set " + onosSetName + ":\n" +
2432 str( getResponses[ i ] ) )
2433 main.log.debug( "Expected: " + str( onosSet ) )
2434 main.log.debug( "Actual: " + str( current ) )
2435 getResults = main.FALSE
2436 else:
2437
2438 main.log.error( "ONOS" + str( i + 1 ) +
2439 " has repeat elements in" +
2440 " set " + onosSetName + ":\n" +
2441 str( getResponses[ i ] ) )
2442 getResults = main.FALSE
2443 elif getResponses[ i ] == main.ERROR:
2444 getResults = main.FALSE
2445 utilities.assert_equals( expect=main.TRUE,
2446 actual=getResults,
2447 onpass="Set elements are correct",
2448 onfail="Set elements are incorrect" )
2449
2450 main.step( "Distributed Set size" )
2451 sizeResponses = []
2452 threads = []
2453 for i in range( main.numCtrls ):
2454 t = main.Thread( target=main.CLIs[i].setTestSize,
2455 name="setTestSize-" + str( i ),
2456 args=[ onosSetName ] )
2457 threads.append( t )
2458 t.start()
2459 for t in threads:
2460 t.join()
2461 sizeResponses.append( t.result )
2462
2463 sizeResults = main.TRUE
2464 for i in range( main.numCtrls ):
2465 if size != sizeResponses[ i ]:
2466 sizeResults = main.FALSE
2467 main.log.error( "ONOS" + str( i + 1 ) +
2468 " expected a size of " + str( size ) +
2469 " for set " + onosSetName +
2470 " but got " + str( sizeResponses[ i ] ) )
2471 utilities.assert_equals( expect=main.TRUE,
2472 actual=sizeResults,
2473 onpass="Set sizes are correct",
2474 onfail="Set sizes are incorrect" )
2475
2476 main.step( "Distributed Set add()" )
2477 onosSet.add( addValue )
2478 addResponses = []
2479 threads = []
2480 for i in range( main.numCtrls ):
2481 t = main.Thread( target=main.CLIs[i].setTestAdd,
2482 name="setTestAdd-" + str( i ),
2483 args=[ onosSetName, addValue ] )
2484 threads.append( t )
2485 t.start()
2486 for t in threads:
2487 t.join()
2488 addResponses.append( t.result )
2489
2490
2491
2492
2493 addResults = main.TRUE
2494 for i in range( main.numCtrls ):
2495 if addResponses[ i ] == main.TRUE:
2496
2497 pass
2498 elif addResponses[ i ] == main.FALSE:
2499
2500 pass
2501 elif addResponses[ i ] == main.ERROR:
2502
2503 addResults = main.FALSE
2504 else:
2505
2506 addResults = main.FALSE
2507 if addResults != main.TRUE:
2508 main.log.error( "Error executing set add" )
2509
2510
2511 size = len( onosSet )
2512 getResponses = []
2513 threads = []
2514 for i in range( main.numCtrls ):
2515 t = main.Thread( target=main.CLIs[i].setTestGet,
2516 name="setTestGet-" + str( i ),
2517 args=[ onosSetName ] )
2518 threads.append( t )
2519 t.start()
2520 for t in threads:
2521 t.join()
2522 getResponses.append( t.result )
2523 getResults = main.TRUE
2524 for i in range( main.numCtrls ):
2525 if isinstance( getResponses[ i ], list):
2526 current = set( getResponses[ i ] )
2527 if len( current ) == len( getResponses[ i ] ):
2528
2529 if onosSet != current:
2530 main.log.error( "ONOS" + str( i + 1 ) +
2531 " has incorrect view" +
2532 " of set " + onosSetName + ":\n" +
2533 str( getResponses[ i ] ) )
2534 main.log.debug( "Expected: " + str( onosSet ) )
2535 main.log.debug( "Actual: " + str( current ) )
2536 getResults = main.FALSE
2537 else:
2538
2539 main.log.error( "ONOS" + str( i + 1 ) +
2540 " has repeat elements in" +
2541 " set " + onosSetName + ":\n" +
2542 str( getResponses[ i ] ) )
2543 getResults = main.FALSE
2544 elif getResponses[ i ] == main.ERROR:
2545 getResults = main.FALSE
2546 sizeResponses = []
2547 threads = []
2548 for i in range( main.numCtrls ):
2549 t = main.Thread( target=main.CLIs[i].setTestSize,
2550 name="setTestSize-" + str( i ),
2551 args=[ onosSetName ] )
2552 threads.append( t )
2553 t.start()
2554 for t in threads:
2555 t.join()
2556 sizeResponses.append( t.result )
2557 sizeResults = main.TRUE
2558 for i in range( main.numCtrls ):
2559 if size != sizeResponses[ i ]:
2560 sizeResults = main.FALSE
2561 main.log.error( "ONOS" + str( i + 1 ) +
2562 " expected a size of " + str( size ) +
2563 " for set " + onosSetName +
2564 " but got " + str( sizeResponses[ i ] ) )
2565 addResults = addResults and getResults and sizeResults
2566 utilities.assert_equals( expect=main.TRUE,
2567 actual=addResults,
2568 onpass="Set add correct",
2569 onfail="Set add was incorrect" )
2570
2571 main.step( "Distributed Set addAll()" )
2572 onosSet.update( addAllValue.split() )
2573 addResponses = []
2574 threads = []
2575 for i in range( main.numCtrls ):
2576 t = main.Thread( target=main.CLIs[i].setTestAdd,
2577 name="setTestAddAll-" + str( i ),
2578 args=[ onosSetName, addAllValue ] )
2579 threads.append( t )
2580 t.start()
2581 for t in threads:
2582 t.join()
2583 addResponses.append( t.result )
2584
2585
2586
2587
2588 addAllResults = main.TRUE
2589 for i in range( main.numCtrls ):
2590 if addResponses[ i ] == main.TRUE:
2591
2592 pass
2593 elif addResponses[ i ] == main.FALSE:
2594
2595 pass
2596 elif addResponses[ i ] == main.ERROR:
2597
2598 addAllResults = main.FALSE
2599 else:
2600
2601 addAllResults = main.FALSE
2602 if addAllResults != main.TRUE:
2603 main.log.error( "Error executing set addAll" )
2604
2605
2606 size = len( onosSet )
2607 getResponses = []
2608 threads = []
2609 for i in range( main.numCtrls ):
2610 t = main.Thread( target=main.CLIs[i].setTestGet,
2611 name="setTestGet-" + str( i ),
2612 args=[ onosSetName ] )
2613 threads.append( t )
2614 t.start()
2615 for t in threads:
2616 t.join()
2617 getResponses.append( t.result )
2618 getResults = main.TRUE
2619 for i in range( main.numCtrls ):
2620 if isinstance( getResponses[ i ], list):
2621 current = set( getResponses[ i ] )
2622 if len( current ) == len( getResponses[ i ] ):
2623
2624 if onosSet != current:
2625 main.log.error( "ONOS" + str( i + 1 ) +
2626 " has incorrect view" +
2627 " of set " + onosSetName + ":\n" +
2628 str( getResponses[ i ] ) )
2629 main.log.debug( "Expected: " + str( onosSet ) )
2630 main.log.debug( "Actual: " + str( current ) )
2631 getResults = main.FALSE
2632 else:
2633
2634 main.log.error( "ONOS" + str( i + 1 ) +
2635 " has repeat elements in" +
2636 " set " + onosSetName + ":\n" +
2637 str( getResponses[ i ] ) )
2638 getResults = main.FALSE
2639 elif getResponses[ i ] == main.ERROR:
2640 getResults = main.FALSE
2641 sizeResponses = []
2642 threads = []
2643 for i in range( main.numCtrls ):
2644 t = main.Thread( target=main.CLIs[i].setTestSize,
2645 name="setTestSize-" + str( i ),
2646 args=[ onosSetName ] )
2647 threads.append( t )
2648 t.start()
2649 for t in threads:
2650 t.join()
2651 sizeResponses.append( t.result )
2652 sizeResults = main.TRUE
2653 for i in range( main.numCtrls ):
2654 if size != sizeResponses[ i ]:
2655 sizeResults = main.FALSE
2656 main.log.error( "ONOS" + str( i + 1 ) +
2657 " expected a size of " + str( size ) +
2658 " for set " + onosSetName +
2659 " but got " + str( sizeResponses[ i ] ) )
2660 addAllResults = addAllResults and getResults and sizeResults
2661 utilities.assert_equals( expect=main.TRUE,
2662 actual=addAllResults,
2663 onpass="Set addAll correct",
2664 onfail="Set addAll was incorrect" )
2665
2666 main.step( "Distributed Set contains()" )
2667 containsResponses = []
2668 threads = []
2669 for i in range( main.numCtrls ):
2670 t = main.Thread( target=main.CLIs[i].setTestGet,
2671 name="setContains-" + str( i ),
2672 args=[ onosSetName ],
2673 kwargs={ "values": addValue } )
2674 threads.append( t )
2675 t.start()
2676 for t in threads:
2677 t.join()
2678
2679 containsResponses.append( t.result )
2680
2681 containsResults = main.TRUE
2682 for i in range( main.numCtrls ):
2683 if containsResponses[ i ] == main.ERROR:
2684 containsResults = main.FALSE
2685 else:
2686 containsResults = containsResults and\
2687 containsResponses[ i ][ 1 ]
2688 utilities.assert_equals( expect=main.TRUE,
2689 actual=containsResults,
2690 onpass="Set contains is functional",
2691 onfail="Set contains failed" )
2692
2693 main.step( "Distributed Set containsAll()" )
2694 containsAllResponses = []
2695 threads = []
2696 for i in range( main.numCtrls ):
2697 t = main.Thread( target=main.CLIs[i].setTestGet,
2698 name="setContainsAll-" + str( i ),
2699 args=[ onosSetName ],
2700 kwargs={ "values": addAllValue } )
2701 threads.append( t )
2702 t.start()
2703 for t in threads:
2704 t.join()
2705
2706 containsAllResponses.append( t.result )
2707
2708 containsAllResults = main.TRUE
2709 for i in range( main.numCtrls ):
2710 if containsResponses[ i ] == main.ERROR:
2711 containsResults = main.FALSE
2712 else:
2713 containsResults = containsResults and\
2714 containsResponses[ i ][ 1 ]
2715 utilities.assert_equals( expect=main.TRUE,
2716 actual=containsAllResults,
2717 onpass="Set containsAll is functional",
2718 onfail="Set containsAll failed" )
2719
2720 main.step( "Distributed Set remove()" )
2721 onosSet.remove( addValue )
2722 removeResponses = []
2723 threads = []
2724 for i in range( main.numCtrls ):
2725 t = main.Thread( target=main.CLIs[i].setTestRemove,
2726 name="setTestRemove-" + str( i ),
2727 args=[ onosSetName, addValue ] )
2728 threads.append( t )
2729 t.start()
2730 for t in threads:
2731 t.join()
2732 removeResponses.append( t.result )
2733
2734
2735
2736
2737 removeResults = main.TRUE
2738 for i in range( main.numCtrls ):
2739 if removeResponses[ i ] == main.TRUE:
2740
2741 pass
2742 elif removeResponses[ i ] == main.FALSE:
2743
2744 pass
2745 elif removeResponses[ i ] == main.ERROR:
2746
2747 removeResults = main.FALSE
2748 else:
2749
2750 removeResults = main.FALSE
2751 if removeResults != main.TRUE:
2752 main.log.error( "Error executing set remove" )
2753
2754
2755 size = len( onosSet )
2756 getResponses = []
2757 threads = []
2758 for i in range( main.numCtrls ):
2759 t = main.Thread( target=main.CLIs[i].setTestGet,
2760 name="setTestGet-" + str( i ),
2761 args=[ onosSetName ] )
2762 threads.append( t )
2763 t.start()
2764 for t in threads:
2765 t.join()
2766 getResponses.append( t.result )
2767 getResults = main.TRUE
2768 for i in range( main.numCtrls ):
2769 if isinstance( getResponses[ i ], list):
2770 current = set( getResponses[ i ] )
2771 if len( current ) == len( getResponses[ i ] ):
2772
2773 if onosSet != current:
2774 main.log.error( "ONOS" + str( i + 1 ) +
2775 " has incorrect view" +
2776 " of set " + onosSetName + ":\n" +
2777 str( getResponses[ i ] ) )
2778 main.log.debug( "Expected: " + str( onosSet ) )
2779 main.log.debug( "Actual: " + str( current ) )
2780 getResults = main.FALSE
2781 else:
2782
2783 main.log.error( "ONOS" + str( i + 1 ) +
2784 " has repeat elements in" +
2785 " set " + onosSetName + ":\n" +
2786 str( getResponses[ i ] ) )
2787 getResults = main.FALSE
2788 elif getResponses[ i ] == main.ERROR:
2789 getResults = main.FALSE
2790 sizeResponses = []
2791 threads = []
2792 for i in range( main.numCtrls ):
2793 t = main.Thread( target=main.CLIs[i].setTestSize,
2794 name="setTestSize-" + str( i ),
2795 args=[ onosSetName ] )
2796 threads.append( t )
2797 t.start()
2798 for t in threads:
2799 t.join()
2800 sizeResponses.append( t.result )
2801 sizeResults = main.TRUE
2802 for i in range( main.numCtrls ):
2803 if size != sizeResponses[ i ]:
2804 sizeResults = main.FALSE
2805 main.log.error( "ONOS" + str( i + 1 ) +
2806 " expected a size of " + str( size ) +
2807 " for set " + onosSetName +
2808 " but got " + str( sizeResponses[ i ] ) )
2809 removeResults = removeResults and getResults and sizeResults
2810 utilities.assert_equals( expect=main.TRUE,
2811 actual=removeResults,
2812 onpass="Set remove correct",
2813 onfail="Set remove was incorrect" )
2814
2815 main.step( "Distributed Set removeAll()" )
2816 onosSet.difference_update( addAllValue.split() )
2817 removeAllResponses = []
2818 threads = []
2819 try:
2820 for i in range( main.numCtrls ):
2821 t = main.Thread( target=main.CLIs[i].setTestRemove,
2822 name="setTestRemoveAll-" + str( i ),
2823 args=[ onosSetName, addAllValue ] )
2824 threads.append( t )
2825 t.start()
2826 for t in threads:
2827 t.join()
2828 removeAllResponses.append( t.result )
2829 except Exception, e:
2830 main.log.exception(e)
2831
2832
2833
2834
2835 removeAllResults = main.TRUE
2836 for i in range( main.numCtrls ):
2837 if removeAllResponses[ i ] == main.TRUE:
2838
2839 pass
2840 elif removeAllResponses[ i ] == main.FALSE:
2841
2842 pass
2843 elif removeAllResponses[ i ] == main.ERROR:
2844
2845 removeAllResults = main.FALSE
2846 else:
2847
2848 removeAllResults = main.FALSE
2849 if removeAllResults != main.TRUE:
2850 main.log.error( "Error executing set removeAll" )
2851
2852
2853 size = len( onosSet )
2854 getResponses = []
2855 threads = []
2856 for i in range( main.numCtrls ):
2857 t = main.Thread( target=main.CLIs[i].setTestGet,
2858 name="setTestGet-" + str( i ),
2859 args=[ onosSetName ] )
2860 threads.append( t )
2861 t.start()
2862 for t in threads:
2863 t.join()
2864 getResponses.append( t.result )
2865 getResults = main.TRUE
2866 for i in range( main.numCtrls ):
2867 if isinstance( getResponses[ i ], list):
2868 current = set( getResponses[ i ] )
2869 if len( current ) == len( getResponses[ i ] ):
2870
2871 if onosSet != current:
2872 main.log.error( "ONOS" + str( i + 1 ) +
2873 " has incorrect view" +
2874 " of set " + onosSetName + ":\n" +
2875 str( getResponses[ i ] ) )
2876 main.log.debug( "Expected: " + str( onosSet ) )
2877 main.log.debug( "Actual: " + str( current ) )
2878 getResults = main.FALSE
2879 else:
2880
2881 main.log.error( "ONOS" + str( i + 1 ) +
2882 " has repeat elements in" +
2883 " set " + onosSetName + ":\n" +
2884 str( getResponses[ i ] ) )
2885 getResults = main.FALSE
2886 elif getResponses[ i ] == main.ERROR:
2887 getResults = main.FALSE
2888 sizeResponses = []
2889 threads = []
2890 for i in range( main.numCtrls ):
2891 t = main.Thread( target=main.CLIs[i].setTestSize,
2892 name="setTestSize-" + str( i ),
2893 args=[ onosSetName ] )
2894 threads.append( t )
2895 t.start()
2896 for t in threads:
2897 t.join()
2898 sizeResponses.append( t.result )
2899 sizeResults = main.TRUE
2900 for i in range( main.numCtrls ):
2901 if size != sizeResponses[ i ]:
2902 sizeResults = main.FALSE
2903 main.log.error( "ONOS" + str( i + 1 ) +
2904 " expected a size of " + str( size ) +
2905 " for set " + onosSetName +
2906 " but got " + str( sizeResponses[ i ] ) )
2907 removeAllResults = removeAllResults and getResults and sizeResults
2908 utilities.assert_equals( expect=main.TRUE,
2909 actual=removeAllResults,
2910 onpass="Set removeAll correct",
2911 onfail="Set removeAll was incorrect" )
2912
2913 main.step( "Distributed Set addAll()" )
2914 onosSet.update( addAllValue.split() )
2915 addResponses = []
2916 threads = []
2917 for i in range( main.numCtrls ):
2918 t = main.Thread( target=main.CLIs[i].setTestAdd,
2919 name="setTestAddAll-" + str( i ),
2920 args=[ onosSetName, addAllValue ] )
2921 threads.append( t )
2922 t.start()
2923 for t in threads:
2924 t.join()
2925 addResponses.append( t.result )
2926
2927
2928
2929
2930 addAllResults = main.TRUE
2931 for i in range( main.numCtrls ):
2932 if addResponses[ i ] == main.TRUE:
2933
2934 pass
2935 elif addResponses[ i ] == main.FALSE:
2936
2937 pass
2938 elif addResponses[ i ] == main.ERROR:
2939
2940 addAllResults = main.FALSE
2941 else:
2942
2943 addAllResults = main.FALSE
2944 if addAllResults != main.TRUE:
2945 main.log.error( "Error executing set addAll" )
2946
2947
2948 size = len( onosSet )
2949 getResponses = []
2950 threads = []
2951 for i in range( main.numCtrls ):
2952 t = main.Thread( target=main.CLIs[i].setTestGet,
2953 name="setTestGet-" + str( i ),
2954 args=[ onosSetName ] )
2955 threads.append( t )
2956 t.start()
2957 for t in threads:
2958 t.join()
2959 getResponses.append( t.result )
2960 getResults = main.TRUE
2961 for i in range( main.numCtrls ):
2962 if isinstance( getResponses[ i ], list):
2963 current = set( getResponses[ i ] )
2964 if len( current ) == len( getResponses[ i ] ):
2965
2966 if onosSet != current:
2967 main.log.error( "ONOS" + str( i + 1 ) +
2968 " has incorrect view" +
2969 " of set " + onosSetName + ":\n" +
2970 str( getResponses[ i ] ) )
2971 main.log.debug( "Expected: " + str( onosSet ) )
2972 main.log.debug( "Actual: " + str( current ) )
2973 getResults = main.FALSE
2974 else:
2975
2976 main.log.error( "ONOS" + str( i + 1 ) +
2977 " has repeat elements in" +
2978 " set " + onosSetName + ":\n" +
2979 str( getResponses[ i ] ) )
2980 getResults = main.FALSE
2981 elif getResponses[ i ] == main.ERROR:
2982 getResults = main.FALSE
2983 sizeResponses = []
2984 threads = []
2985 for i in range( main.numCtrls ):
2986 t = main.Thread( target=main.CLIs[i].setTestSize,
2987 name="setTestSize-" + str( i ),
2988 args=[ onosSetName ] )
2989 threads.append( t )
2990 t.start()
2991 for t in threads:
2992 t.join()
2993 sizeResponses.append( t.result )
2994 sizeResults = main.TRUE
2995 for i in range( main.numCtrls ):
2996 if size != sizeResponses[ i ]:
2997 sizeResults = main.FALSE
2998 main.log.error( "ONOS" + str( i + 1 ) +
2999 " expected a size of " + str( size ) +
3000 " for set " + onosSetName +
3001 " but got " + str( sizeResponses[ i ] ) )
3002 addAllResults = addAllResults and getResults and sizeResults
3003 utilities.assert_equals( expect=main.TRUE,
3004 actual=addAllResults,
3005 onpass="Set addAll correct",
3006 onfail="Set addAll was incorrect" )
3007
3008 main.step( "Distributed Set clear()" )
3009 onosSet.clear()
3010 clearResponses = []
3011 threads = []
3012 for i in range( main.numCtrls ):
3013 t = main.Thread( target=main.CLIs[i].setTestRemove,
3014 name="setTestClear-" + str( i ),
3015 args=[ onosSetName, " "],
3016 kwargs={ "clear": True } )
3017 threads.append( t )
3018 t.start()
3019 for t in threads:
3020 t.join()
3021 clearResponses.append( t.result )
3022
3023
3024
3025
3026 clearResults = main.TRUE
3027 for i in range( main.numCtrls ):
3028 if clearResponses[ i ] == main.TRUE:
3029
3030 pass
3031 elif clearResponses[ i ] == main.FALSE:
3032
3033 pass
3034 elif clearResponses[ i ] == main.ERROR:
3035
3036 clearResults = main.FALSE
3037 else:
3038
3039 clearResults = main.FALSE
3040 if clearResults != main.TRUE:
3041 main.log.error( "Error executing set clear" )
3042
3043
3044 size = len( onosSet )
3045 getResponses = []
3046 threads = []
3047 for i in range( main.numCtrls ):
3048 t = main.Thread( target=main.CLIs[i].setTestGet,
3049 name="setTestGet-" + str( i ),
3050 args=[ onosSetName ] )
3051 threads.append( t )
3052 t.start()
3053 for t in threads:
3054 t.join()
3055 getResponses.append( t.result )
3056 getResults = main.TRUE
3057 for i in range( main.numCtrls ):
3058 if isinstance( getResponses[ i ], list):
3059 current = set( getResponses[ i ] )
3060 if len( current ) == len( getResponses[ i ] ):
3061
3062 if onosSet != current:
3063 main.log.error( "ONOS" + str( i + 1 ) +
3064 " has incorrect view" +
3065 " of set " + onosSetName + ":\n" +
3066 str( getResponses[ i ] ) )
3067 main.log.debug( "Expected: " + str( onosSet ) )
3068 main.log.debug( "Actual: " + str( current ) )
3069 getResults = main.FALSE
3070 else:
3071
3072 main.log.error( "ONOS" + str( i + 1 ) +
3073 " has repeat elements in" +
3074 " set " + onosSetName + ":\n" +
3075 str( getResponses[ i ] ) )
3076 getResults = main.FALSE
3077 elif getResponses[ i ] == main.ERROR:
3078 getResults = main.FALSE
3079 sizeResponses = []
3080 threads = []
3081 for i in range( main.numCtrls ):
3082 t = main.Thread( target=main.CLIs[i].setTestSize,
3083 name="setTestSize-" + str( i ),
3084 args=[ onosSetName ] )
3085 threads.append( t )
3086 t.start()
3087 for t in threads:
3088 t.join()
3089 sizeResponses.append( t.result )
3090 sizeResults = main.TRUE
3091 for i in range( main.numCtrls ):
3092 if size != sizeResponses[ i ]:
3093 sizeResults = main.FALSE
3094 main.log.error( "ONOS" + str( i + 1 ) +
3095 " expected a size of " + str( size ) +
3096 " for set " + onosSetName +
3097 " but got " + str( sizeResponses[ i ] ) )
3098 clearResults = clearResults and getResults and sizeResults
3099 utilities.assert_equals( expect=main.TRUE,
3100 actual=clearResults,
3101 onpass="Set clear correct",
3102 onfail="Set clear was incorrect" )
3103
3104 main.step( "Distributed Set addAll()" )
3105 onosSet.update( addAllValue.split() )
3106 addResponses = []
3107 threads = []
3108 for i in range( main.numCtrls ):
3109 t = main.Thread( target=main.CLIs[i].setTestAdd,
3110 name="setTestAddAll-" + str( i ),
3111 args=[ onosSetName, addAllValue ] )
3112 threads.append( t )
3113 t.start()
3114 for t in threads:
3115 t.join()
3116 addResponses.append( t.result )
3117
3118
3119
3120
3121 addAllResults = main.TRUE
3122 for i in range( main.numCtrls ):
3123 if addResponses[ i ] == main.TRUE:
3124
3125 pass
3126 elif addResponses[ i ] == main.FALSE:
3127
3128 pass
3129 elif addResponses[ i ] == main.ERROR:
3130
3131 addAllResults = main.FALSE
3132 else:
3133
3134 addAllResults = main.FALSE
3135 if addAllResults != main.TRUE:
3136 main.log.error( "Error executing set addAll" )
3137
3138
3139 size = len( onosSet )
3140 getResponses = []
3141 threads = []
3142 for i in range( main.numCtrls ):
3143 t = main.Thread( target=main.CLIs[i].setTestGet,
3144 name="setTestGet-" + str( i ),
3145 args=[ onosSetName ] )
3146 threads.append( t )
3147 t.start()
3148 for t in threads:
3149 t.join()
3150 getResponses.append( t.result )
3151 getResults = main.TRUE
3152 for i in range( main.numCtrls ):
3153 if isinstance( getResponses[ i ], list):
3154 current = set( getResponses[ i ] )
3155 if len( current ) == len( getResponses[ i ] ):
3156
3157 if onosSet != current:
3158 main.log.error( "ONOS" + str( i + 1 ) +
3159 " has incorrect view" +
3160 " of set " + onosSetName + ":\n" +
3161 str( getResponses[ i ] ) )
3162 main.log.debug( "Expected: " + str( onosSet ) )
3163 main.log.debug( "Actual: " + str( current ) )
3164 getResults = main.FALSE
3165 else:
3166
3167 main.log.error( "ONOS" + str( i + 1 ) +
3168 " has repeat elements in" +
3169 " set " + onosSetName + ":\n" +
3170 str( getResponses[ i ] ) )
3171 getResults = main.FALSE
3172 elif getResponses[ i ] == main.ERROR:
3173 getResults = main.FALSE
3174 sizeResponses = []
3175 threads = []
3176 for i in range( main.numCtrls ):
3177 t = main.Thread( target=main.CLIs[i].setTestSize,
3178 name="setTestSize-" + str( i ),
3179 args=[ onosSetName ] )
3180 threads.append( t )
3181 t.start()
3182 for t in threads:
3183 t.join()
3184 sizeResponses.append( t.result )
3185 sizeResults = main.TRUE
3186 for i in range( main.numCtrls ):
3187 if size != sizeResponses[ i ]:
3188 sizeResults = main.FALSE
3189 main.log.error( "ONOS" + str( i + 1 ) +
3190 " expected a size of " + str( size ) +
3191 " for set " + onosSetName +
3192 " but got " + str( sizeResponses[ i ] ) )
3193 addAllResults = addAllResults and getResults and sizeResults
3194 utilities.assert_equals( expect=main.TRUE,
3195 actual=addAllResults,
3196 onpass="Set addAll correct",
3197 onfail="Set addAll was incorrect" )
3198
3199 main.step( "Distributed Set retain()" )
3200 onosSet.intersection_update( retainValue.split() )
3201 retainResponses = []
3202 threads = []
3203 for i in range( main.numCtrls ):
3204 t = main.Thread( target=main.CLIs[i].setTestRemove,
3205 name="setTestRetain-" + str( i ),
3206 args=[ onosSetName, retainValue ],
3207 kwargs={ "retain": True } )
3208 threads.append( t )
3209 t.start()
3210 for t in threads:
3211 t.join()
3212 retainResponses.append( t.result )
3213
3214
3215
3216
3217 retainResults = main.TRUE
3218 for i in range( main.numCtrls ):
3219 if retainResponses[ i ] == main.TRUE:
3220
3221 pass
3222 elif retainResponses[ i ] == main.FALSE:
3223
3224 pass
3225 elif retainResponses[ i ] == main.ERROR:
3226
3227 retainResults = main.FALSE
3228 else:
3229
3230 retainResults = main.FALSE
3231 if retainResults != main.TRUE:
3232 main.log.error( "Error executing set retain" )
3233
3234
3235 size = len( onosSet )
3236 getResponses = []
3237 threads = []
3238 for i in range( main.numCtrls ):
3239 t = main.Thread( target=main.CLIs[i].setTestGet,
3240 name="setTestGet-" + str( i ),
3241 args=[ onosSetName ] )
3242 threads.append( t )
3243 t.start()
3244 for t in threads:
3245 t.join()
3246 getResponses.append( t.result )
3247 getResults = main.TRUE
3248 for i in range( main.numCtrls ):
3249 if isinstance( getResponses[ i ], list):
3250 current = set( getResponses[ i ] )
3251 if len( current ) == len( getResponses[ i ] ):
3252
3253 if onosSet != current:
3254 main.log.error( "ONOS" + str( i + 1 ) +
3255 " has incorrect view" +
3256 " of set " + onosSetName + ":\n" +
3257 str( getResponses[ i ] ) )
3258 main.log.debug( "Expected: " + str( onosSet ) )
3259 main.log.debug( "Actual: " + str( current ) )
3260 getResults = main.FALSE
3261 else:
3262
3263 main.log.error( "ONOS" + str( i + 1 ) +
3264 " has repeat elements in" +
3265 " set " + onosSetName + ":\n" +
3266 str( getResponses[ i ] ) )
3267 getResults = main.FALSE
3268 elif getResponses[ i ] == main.ERROR:
3269 getResults = main.FALSE
3270 sizeResponses = []
3271 threads = []
3272 for i in range( main.numCtrls ):
3273 t = main.Thread( target=main.CLIs[i].setTestSize,
3274 name="setTestSize-" + str( i ),
3275 args=[ onosSetName ] )
3276 threads.append( t )
3277 t.start()
3278 for t in threads:
3279 t.join()
3280 sizeResponses.append( t.result )
3281 sizeResults = main.TRUE
3282 for i in range( main.numCtrls ):
3283 if size != sizeResponses[ i ]:
3284 sizeResults = main.FALSE
3285 main.log.error( "ONOS" + str( i + 1 ) +
3286 " expected a size of " +
3287 str( size ) + " for set " + onosSetName +
3288 " but got " + str( sizeResponses[ i ] ) )
3289 retainResults = retainResults and getResults and sizeResults
3290 utilities.assert_equals( expect=main.TRUE,
3291 actual=retainResults,
3292 onpass="Set retain correct",
3293 onfail="Set retain was incorrect" )
3294