blob: a2602d1329c3e0f602e1ea7f3cca381f5e4be7dc [file] [log] [blame]
Jon Hall85794ff2015-07-08 14:12:30 -07001"""
2Description: This test is to determine if a single
3 instance ONOS 'cluster' can handle a restart
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAsingleInstanceRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hall85794ff2015-07-08 14:12:30 -070051 main.log.info( "ONOS Single node cluster restart " +
52 "HA test - initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall85794ff2015-07-08 14:12:30 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall85794ff2015-07-08 14:12:30 -070070
Jon Halle1a3b752015-07-22 13:02:46 -070071 try:
72 fileName = "Counters"
73 path = main.params[ 'imports' ][ 'path' ]
74 main.Counters = imp.load_source( fileName,
75 path + fileName + ".py" )
76 except Exception as e:
77 main.log.exception( e )
78 main.cleanup()
79 main.exit()
80
81 main.CLIs = []
82 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -070083 ipList = []
84 for i in range( 1, int( main.ONOSbench.maxNodes ) + 1 ):
85 try:
Jon Halle1a3b752015-07-22 13:02:46 -070086 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
87 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
88 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -070089 except AttributeError:
90 break
Jon Hall85794ff2015-07-08 14:12:30 -070091
Jon Hall5cf14d52015-07-16 12:15:19 -070092 main.step( "Create cell file" )
93 cellAppString = main.params[ 'ENV' ][ 'appString' ]
94 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
95 main.Mininet1.ip_address,
96 cellAppString, ipList )
Jon Hall85794ff2015-07-08 14:12:30 -070097 main.step( "Applying cell variable to environment" )
98 cellResult = main.ONOSbench.setCell( cellName )
99 verifyResult = main.ONOSbench.verifyCell()
100
101 # FIXME:this is short term fix
102 main.log.info( "Removing raft logs" )
103 main.ONOSbench.onosRemoveRaftLogs()
104
105 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700106 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700107 main.ONOSbench.onosUninstall( node.ip_address )
108
109 # Make sure ONOS is DEAD
110 main.log.info( "Killing any ONOS processes" )
111 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700112 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700113 killed = main.ONOSbench.onosKill( node.ip_address )
114 killResults = killResults and killed
115
116 cleanInstallResult = main.TRUE
117 gitPullResult = main.TRUE
118
119 main.step( "Starting Mininet" )
120 # scp topo file to mininet
121 # TODO: move to params?
122 topoName = "obelisk.py"
123 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700124 main.ONOSbench.scp( main.Mininet1,
125 filePath + topoName,
126 main.Mininet1.home,
127 direction="to" )
Jon Hall85794ff2015-07-08 14:12:30 -0700128 mnResult = main.Mininet1.startNet( )
129 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
130 onpass="Mininet Started",
131 onfail="Error starting Mininet" )
132
133 main.step( "Git checkout and pull " + gitBranch )
134 if PULLCODE:
135 main.ONOSbench.gitCheckout( gitBranch )
136 gitPullResult = main.ONOSbench.gitPull()
137 # values of 1 or 3 are good
138 utilities.assert_lesser( expect=0, actual=gitPullResult,
139 onpass="Git pull successful",
140 onfail="Git pull failed" )
141 main.ONOSbench.getVersion( report=True )
142
143 main.step( "Using mvn clean install" )
144 cleanInstallResult = main.TRUE
145 if PULLCODE and gitPullResult == main.TRUE:
146 cleanInstallResult = main.ONOSbench.cleanInstall()
147 else:
148 main.log.warn( "Did not pull new code so skipping mvn " +
149 "clean install" )
150 utilities.assert_equals( expect=main.TRUE,
151 actual=cleanInstallResult,
152 onpass="MCI successful",
153 onfail="MCI failed" )
154 # GRAPHS
155 # NOTE: important params here:
156 # job = name of Jenkins job
157 # Plot Name = Plot-HA, only can be used if multiple plots
158 # index = The number of the graph under plot name
Jon Hall5cf14d52015-07-16 12:15:19 -0700159 job = "HAsingleInstanceRestart"
Jon Hall85794ff2015-07-08 14:12:30 -0700160 plotName = "Plot-HA"
161 graphs = '<ac:structured-macro ac:name="html">\n'
162 graphs += '<ac:plain-text-body><![CDATA[\n'
163 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
164 '/plot/' + plotName + '/getPlot?index=0' +\
165 '&width=500&height=300"' +\
166 'noborder="0" width="500" height="300" scrolling="yes" ' +\
167 'seamless="seamless"></iframe>\n'
168 graphs += ']]></ac:plain-text-body>\n'
169 graphs += '</ac:structured-macro>\n'
170 main.log.wiki(graphs)
171
Jon Halle1a3b752015-07-22 13:02:46 -0700172 main.CLIs = []
173 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700174 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700175 for i in range( 1, main.numCtrls + 1 ):
176 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
177 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
178 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700179
180 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, "SingleHA",
181 main.Mininet1.ip_address,
182 cellAppString, ipList[ 0 ] )
Jon Hall85794ff2015-07-08 14:12:30 -0700183 cellResult = main.ONOSbench.setCell( "SingleHA" )
184 verifyResult = main.ONOSbench.verifyCell()
185 main.step( "Creating ONOS package" )
186 packageResult = main.ONOSbench.onosPackage()
187 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
188 onpass="ONOS package successful",
189 onfail="ONOS package failed" )
190
191 main.step( "Installing ONOS package" )
192 onosInstallResult = main.ONOSbench.onosInstall(
Jon Halle1a3b752015-07-22 13:02:46 -0700193 options="-f", node=main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700194 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
195 onpass="ONOS install successful",
196 onfail="ONOS install failed" )
197
198 main.step( "Checking if ONOS is up yet" )
199 for i in range( 2 ):
Jon Halle1a3b752015-07-22 13:02:46 -0700200 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700201 if onos1Isup:
202 break
203 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
204 onpass="ONOS startup successful",
205 onfail="ONOS startup failed" )
206
207 main.log.step( "Starting ONOS CLI sessions" )
Jon Halle1a3b752015-07-22 13:02:46 -0700208 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700209 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
210 onpass="ONOS cli startup successful",
211 onfail="ONOS cli startup failed" )
212
213 if main.params[ 'tcpdump' ].lower() == "true":
214 main.step( "Start Packet Capture MN" )
215 main.Mininet2.startTcpdump(
216 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
217 + "-MN.pcap",
218 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
219 port=main.params[ 'MNtcpdump' ][ 'port' ] )
220
221 main.step( "App Ids check" )
222 appCheck = main.ONOScli1.appToIDCheck()
223 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700224 main.log.warn( main.CLIs[0].apps() )
225 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700226 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
227 onpass="App Ids seem to be correct",
228 onfail="Something is wrong with app Ids" )
229
230 if cliResults == main.FALSE:
231 main.log.error( "Failed to start ONOS, stopping test" )
232 main.cleanup()
233 main.exit()
234
235 def CASE2( self, main ):
236 """
237 Assign devices to controllers
238 """
239 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700240 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700241 assert main, "main not defined"
242 assert utilities.assert_equals, "utilities.assert_equals not defined"
243
244 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700245 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700246 "and check that an ONOS node becomes the " +\
247 "master of the device."
248 main.step( "Assign switches to controllers" )
249
250 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700251 for i in range( main.numCtrls ):
252 ipList.append( main.nodes[ i ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700253 swList = []
254 for i in range( 1, 29 ):
255 swList.append( "s" + str( i ) )
256 main.Mininet1.assignSwController( sw=swList, ip=ipList )
257
258 mastershipCheck = main.TRUE
259 for i in range( 1, 29 ):
260 response = main.Mininet1.getSwController( "s" + str( i ) )
261 try:
262 main.log.info( str( response ) )
263 except Exception:
264 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700265 if re.search( "tcp:" + main.nodes[0].ip_address, response ):
Jon Hall85794ff2015-07-08 14:12:30 -0700266 mastershipCheck = mastershipCheck and main.TRUE
267 else:
268 mastershipCheck = main.FALSE
269 if mastershipCheck == main.TRUE:
270 main.log.info( "Switch mastership assigned correctly" )
271 utilities.assert_equals(
272 expect=main.TRUE,
273 actual=mastershipCheck,
274 onpass="Switch mastership assigned correctly",
275 onfail="Switches not assigned correctly to controllers" )
276
277 def CASE21( self, main ):
278 """
279 Assign mastership to controllers
280 """
Jon Halle1a3b752015-07-22 13:02:46 -0700281 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700282 assert main, "main not defined"
283 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700284 assert main.CLIs, "main.CLIs not defined"
285 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700286
287 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700288 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700289 "device. Then manually assign" +\
290 " mastership to specific ONOS nodes using" +\
291 " 'device-role'"
292 main.step( "Assign mastership of switches to specific controllers" )
293 roleCall = main.TRUE
294 roleCheck = main.TRUE
295 try:
296 for i in range( 1, 29 ): # switches 1 through 28
Jon Halle1a3b752015-07-22 13:02:46 -0700297 ip = main.nodes[ 0 ].ip_address # ONOS1
Jon Hall85794ff2015-07-08 14:12:30 -0700298 # set up correct variables:
299 if i == 1:
300 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
301 elif i == 2:
302 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
303 elif i == 3:
304 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
305 elif i == 4:
306 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
307 elif i == 5:
308 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
309 elif i == 6:
310 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
311 elif i == 7:
312 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
313 elif i >= 8 and i <= 17:
314 dpid = '3' + str( i ).zfill( 3 )
315 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
316 elif i >= 18 and i <= 27:
317 dpid = '6' + str( i ).zfill( 3 )
318 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
319 elif i == 28:
320 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
321 else:
322 main.log.error( "You didn't write an else statement for " +
323 "switch s" + str( i ) )
324 # Assign switch
325 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
326 # TODO: make this controller dynamic
327 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
328 ip )
329 # Check assignment
330 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
331 if ip in master:
332 roleCheck = roleCheck and main.TRUE
333 else:
334 roleCheck = roleCheck and main.FALSE
335 main.log.error( "Error, controller " + ip + " is not" +
336 " master " + "of device " +
337 str( deviceId ) + ". Master is " +
338 repr( master ) + "." )
339 except ( AttributeError, AssertionError ):
340 main.log.exception( "Something is wrong with ONOS device view" )
341 main.log.info( main.ONOScli1.devices() )
342 utilities.assert_equals(
343 expect=main.TRUE,
344 actual=roleCall,
345 onpass="Re-assigned switch mastership to designated controller",
346 onfail="Something wrong with deviceRole calls" )
347
348 main.step( "Check mastership was correctly assigned" )
349 utilities.assert_equals(
350 expect=main.TRUE,
351 actual=roleCheck,
352 onpass="Switches were successfully reassigned to designated " +
353 "controller",
354 onfail="Switches were not successfully reassigned" )
355
356 def CASE3( self, main ):
357 """
358 Assign intents
359 """
360 import time
361 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700362 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700363 assert main, "main not defined"
364 assert utilities.assert_equals, "utilities.assert_equals not defined"
365 # NOTE: we must reinstall intents until we have a persistant intent
366 # datastore!
367 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700368 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700369 "assign predetermined host-to-host intents." +\
370 " After installation, check that the intent" +\
371 " is distributed to all nodes and the state" +\
372 " is INSTALLED"
373
374 # install onos-app-fwd
375 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700376 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700377 utilities.assert_equals( expect=main.TRUE, actual=installResults,
378 onpass="Install fwd successful",
379 onfail="Install fwd failed" )
380
381 main.step( "Check app ids" )
382 appCheck = main.ONOScli1.appToIDCheck()
383 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700384 main.log.warn( main.CLIs[0].apps() )
385 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700386 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
387 onpass="App Ids seem to be correct",
388 onfail="Something is wrong with app Ids" )
389
390 main.step( "Discovering Hosts( Via pingall for now )" )
391 # FIXME: Once we have a host discovery mechanism, use that instead
392 # REACTIVE FWD test
393 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700394 passMsg = "Reactive Pingall test passed"
395 time1 = time.time()
396 pingResult = main.Mininet1.pingall()
397 time2 = time.time()
398 if not pingResult:
399 main.log.warn("First pingall failed. Trying again...")
Jon Hall85794ff2015-07-08 14:12:30 -0700400 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700401 passMsg += " on the second try"
402 utilities.assert_equals(
403 expect=main.TRUE,
404 actual=pingResult,
405 onpass= passMsg,
406 onfail="Reactive Pingall failed, " +
407 "one or more ping pairs failed" )
408 main.log.info( "Time for pingall: %2f seconds" %
409 ( time2 - time1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700410 # timeout for fwd flows
411 time.sleep( 11 )
412 # uninstall onos-app-fwd
413 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700414 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700415 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
416 onpass="Uninstall fwd successful",
417 onfail="Uninstall fwd failed" )
418
419 main.step( "Check app ids" )
420 appCheck2 = main.ONOScli1.appToIDCheck()
421 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700422 main.log.warn( main.CLIs[0].apps() )
423 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700424 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
425 onpass="App Ids seem to be correct",
426 onfail="Something is wrong with app Ids" )
427
428 main.step( "Add host intents via cli" )
429 intentIds = []
430 # TODO: move the host numbers to params
431 # Maybe look at all the paths we ping?
432 intentAddResult = True
433 hostResult = main.TRUE
434 for i in range( 8, 18 ):
435 main.log.info( "Adding host intent between h" + str( i ) +
436 " and h" + str( i + 10 ) )
437 host1 = "00:00:00:00:00:" + \
438 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
439 host2 = "00:00:00:00:00:" + \
440 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
441 # NOTE: getHost can return None
442 host1Dict = main.ONOScli1.getHost( host1 )
443 host2Dict = main.ONOScli1.getHost( host2 )
444 host1Id = None
445 host2Id = None
446 if host1Dict and host2Dict:
447 host1Id = host1Dict.get( 'id', None )
448 host2Id = host2Dict.get( 'id', None )
449 if host1Id and host2Id:
450 tmpId = main.ONOScli1.addHostIntent( host1Id, host2Id )
451 if tmpId:
452 main.log.info( "Added intent with id: " + tmpId )
453 intentIds.append( tmpId )
454 else:
455 main.log.error( "addHostIntent returned: " +
456 repr( tmpId ) )
457 else:
458 main.log.error( "Error, getHost() failed for h" + str( i ) +
459 " and/or h" + str( i + 10 ) )
460 hosts = main.ONOScli1.hosts()
461 main.log.warn( "Hosts output: " )
462 try:
463 main.log.warn( json.dumps( json.loads( hosts ),
464 sort_keys=True,
465 indent=4,
466 separators=( ',', ': ' ) ) )
467 except ( ValueError, TypeError ):
468 main.log.warn( repr( hosts ) )
469 hostResult = main.FALSE
470 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
471 onpass="Found a host id for each host",
472 onfail="Error looking up host ids" )
473
474 intentStart = time.time()
475 onosIds = main.ONOScli1.getAllIntentsId()
476 main.log.info( "Submitted intents: " + str( intentIds ) )
477 main.log.info( "Intents in ONOS: " + str( onosIds ) )
478 for intent in intentIds:
479 if intent in onosIds:
480 pass # intent submitted is in onos
481 else:
482 intentAddResult = False
483 if intentAddResult:
484 intentStop = time.time()
485 else:
486 intentStop = None
487 # Print the intent states
488 intents = main.ONOScli1.intents()
489 intentStates = []
490 installedCheck = True
491 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
492 count = 0
493 try:
494 for intent in json.loads( intents ):
495 state = intent.get( 'state', None )
496 if "INSTALLED" not in state:
497 installedCheck = False
498 intentId = intent.get( 'id', None )
499 intentStates.append( ( intentId, state ) )
500 except ( ValueError, TypeError ):
501 main.log.exception( "Error parsing intents" )
502 # add submitted intents not in the store
503 tmplist = [ i for i, s in intentStates ]
504 missingIntents = False
505 for i in intentIds:
506 if i not in tmplist:
507 intentStates.append( ( i, " - " ) )
508 missingIntents = True
509 intentStates.sort()
510 for i, s in intentStates:
511 count += 1
512 main.log.info( "%-6s%-15s%-15s" %
513 ( str( count ), str( i ), str( s ) ) )
514 leaders = main.ONOScli1.leaders()
515 try:
516 missing = False
517 if leaders:
518 parsedLeaders = json.loads( leaders )
519 main.log.warn( json.dumps( parsedLeaders,
520 sort_keys=True,
521 indent=4,
522 separators=( ',', ': ' ) ) )
523 # check for all intent partitions
524 topics = []
525 for i in range( 14 ):
526 topics.append( "intent-partition-" + str( i ) )
527 main.log.debug( topics )
528 ONOStopics = [ j['topic'] for j in parsedLeaders ]
529 for topic in topics:
530 if topic not in ONOStopics:
531 main.log.error( "Error: " + topic +
532 " not in leaders" )
533 missing = True
534 else:
535 main.log.error( "leaders() returned None" )
536 except ( ValueError, TypeError ):
537 main.log.exception( "Error parsing leaders" )
538 main.log.error( repr( leaders ) )
539 # Check all nodes
540 if missing:
541 response = main.ONOScli1.leaders( jsonFormat=False)
542 main.log.warn( "ONOS1 leaders output: \n" +
543 str( response ) )
544
545 partitions = main.ONOScli1.partitions()
546 try:
547 if partitions :
548 parsedPartitions = json.loads( partitions )
549 main.log.warn( json.dumps( parsedPartitions,
550 sort_keys=True,
551 indent=4,
552 separators=( ',', ': ' ) ) )
553 # TODO check for a leader in all paritions
554 # TODO check for consistency among nodes
555 else:
556 main.log.error( "partitions() returned None" )
557 except ( ValueError, TypeError ):
558 main.log.exception( "Error parsing partitions" )
559 main.log.error( repr( partitions ) )
560 pendingMap = main.ONOScli1.pendingMap()
561 try:
562 if pendingMap :
563 parsedPending = json.loads( pendingMap )
564 main.log.warn( json.dumps( parsedPending,
565 sort_keys=True,
566 indent=4,
567 separators=( ',', ': ' ) ) )
568 # TODO check something here?
569 else:
570 main.log.error( "pendingMap() returned None" )
571 except ( ValueError, TypeError ):
572 main.log.exception( "Error parsing pending map" )
573 main.log.error( repr( pendingMap ) )
574
575 intentAddResult = bool( intentAddResult and not missingIntents and
576 installedCheck )
577 if not intentAddResult:
578 main.log.error( "Error in pushing host intents to ONOS" )
579
580 main.step( "Intent Anti-Entropy dispersion" )
581 for i in range(100):
582 correct = True
583 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700584 for cli in main.CLIs:
Jon Hall85794ff2015-07-08 14:12:30 -0700585 onosIds = []
586 ids = cli.getAllIntentsId()
587 onosIds.append( ids )
588 main.log.debug( "Intents in " + cli.name + ": " +
589 str( sorted( onosIds ) ) )
590 if sorted( ids ) != sorted( intentIds ):
591 main.log.warn( "Set of intent IDs doesn't match" )
592 correct = False
593 break
594 else:
595 intents = json.loads( cli.intents() )
596 for intent in intents:
597 if intent[ 'state' ] != "INSTALLED":
598 main.log.warn( "Intent " + intent[ 'id' ] +
599 " is " + intent[ 'state' ] )
600 correct = False
601 break
602 if correct:
603 break
604 else:
605 time.sleep(1)
606 if not intentStop:
607 intentStop = time.time()
608 global gossipTime
609 gossipTime = intentStop - intentStart
610 main.log.info( "It took about " + str( gossipTime ) +
611 " seconds for all intents to appear in each node" )
612 # FIXME: make this time configurable/calculate based off of number of
613 # nodes and gossip rounds
614 utilities.assert_greater_equals(
615 expect=40, actual=gossipTime,
616 onpass="ECM anti-entropy for intents worked within " +
617 "expected time",
618 onfail="Intent ECM anti-entropy took too long" )
619 if gossipTime <= 40:
620 intentAddResult = True
621
622 if not intentAddResult or "key" in pendingMap:
623 import time
624 installedCheck = True
625 main.log.info( "Sleeping 60 seconds to see if intents are found" )
626 time.sleep( 60 )
627 onosIds = main.ONOScli1.getAllIntentsId()
628 main.log.info( "Submitted intents: " + str( intentIds ) )
629 main.log.info( "Intents in ONOS: " + str( onosIds ) )
630 # Print the intent states
631 intents = main.ONOScli1.intents()
632 intentStates = []
633 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
634 count = 0
635 try:
636 for intent in json.loads( intents ):
637 # Iter through intents of a node
638 state = intent.get( 'state', None )
639 if "INSTALLED" not in state:
640 installedCheck = False
641 intentId = intent.get( 'id', None )
642 intentStates.append( ( intentId, state ) )
643 except ( ValueError, TypeError ):
644 main.log.exception( "Error parsing intents" )
645 # add submitted intents not in the store
646 tmplist = [ i for i, s in intentStates ]
647 for i in intentIds:
648 if i not in tmplist:
649 intentStates.append( ( i, " - " ) )
650 intentStates.sort()
651 for i, s in intentStates:
652 count += 1
653 main.log.info( "%-6s%-15s%-15s" %
654 ( str( count ), str( i ), str( s ) ) )
655 leaders = main.ONOScli1.leaders()
656 try:
657 missing = False
658 if leaders:
659 parsedLeaders = json.loads( leaders )
660 main.log.warn( json.dumps( parsedLeaders,
661 sort_keys=True,
662 indent=4,
663 separators=( ',', ': ' ) ) )
664 # check for all intent partitions
665 # check for election
666 topics = []
667 for i in range( 14 ):
668 topics.append( "intent-partition-" + str( i ) )
669 # FIXME: this should only be after we start the app
670 topics.append( "org.onosproject.election" )
671 main.log.debug( topics )
672 ONOStopics = [ j['topic'] for j in parsedLeaders ]
673 for topic in topics:
674 if topic not in ONOStopics:
675 main.log.error( "Error: " + topic +
676 " not in leaders" )
677 missing = True
678 else:
679 main.log.error( "leaders() returned None" )
680 except ( ValueError, TypeError ):
681 main.log.exception( "Error parsing leaders" )
682 main.log.error( repr( leaders ) )
683 # Check all nodes
684 if missing:
685 response = main.ONOScli1.leaders( jsonFormat=False)
686 main.log.warn( "ONOS1 leaders output: \n" +
687 str( response ) )
688 partitions = main.ONOScli1.partitions()
689 try:
690 if partitions :
691 parsedPartitions = json.loads( partitions )
692 main.log.warn( json.dumps( parsedPartitions,
693 sort_keys=True,
694 indent=4,
695 separators=( ',', ': ' ) ) )
696 # TODO check for a leader in all paritions
697 # TODO check for consistency among nodes
698 else:
699 main.log.error( "partitions() returned None" )
700 except ( ValueError, TypeError ):
701 main.log.exception( "Error parsing partitions" )
702 main.log.error( repr( partitions ) )
703 pendingMap = main.ONOScli1.pendingMap()
704 try:
705 if pendingMap :
706 parsedPending = json.loads( pendingMap )
707 main.log.warn( json.dumps( parsedPending,
708 sort_keys=True,
709 indent=4,
710 separators=( ',', ': ' ) ) )
711 # TODO check something here?
712 else:
713 main.log.error( "pendingMap() returned None" )
714 except ( ValueError, TypeError ):
715 main.log.exception( "Error parsing pending map" )
716 main.log.error( repr( pendingMap ) )
717
718 def CASE4( self, main ):
719 """
720 Ping across added host intents
721 """
722 import json
723 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700724 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700725 assert main, "main not defined"
726 assert utilities.assert_equals, "utilities.assert_equals not defined"
727 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700728 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700729 "functionality and check the state of " +\
730 "the intent"
731 main.step( "Ping across added host intents" )
732 PingResult = main.TRUE
733 for i in range( 8, 18 ):
734 ping = main.Mininet1.pingHost( src="h" + str( i ),
735 target="h" + str( i + 10 ) )
736 PingResult = PingResult and ping
737 if ping == main.FALSE:
738 main.log.warn( "Ping failed between h" + str( i ) +
739 " and h" + str( i + 10 ) )
740 elif ping == main.TRUE:
741 main.log.info( "Ping test passed!" )
742 # Don't set PingResult or you'd override failures
743 if PingResult == main.FALSE:
744 main.log.error(
745 "Intents have not been installed correctly, pings failed." )
746 # TODO: pretty print
747 main.log.warn( "ONOS1 intents: " )
748 try:
749 tmpIntents = main.ONOScli1.intents()
750 main.log.warn( json.dumps( json.loads( tmpIntents ),
751 sort_keys=True,
752 indent=4,
753 separators=( ',', ': ' ) ) )
754 except ( ValueError, TypeError ):
755 main.log.warn( repr( tmpIntents ) )
756 utilities.assert_equals(
757 expect=main.TRUE,
758 actual=PingResult,
759 onpass="Intents have been installed correctly and pings work",
760 onfail="Intents have not been installed correctly, pings failed." )
761
762 main.step( "Check Intent state" )
763 installedCheck = True
764 # Print the intent states
765 intents = main.ONOScli1.intents()
766 intentStates = []
767 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
768 count = 0
769 # Iter through intents of a node
770 try:
771 for intent in json.loads( intents ):
772 state = intent.get( 'state', None )
773 if "INSTALLED" not in state:
774 installedCheck = False
775 intentId = intent.get( 'id', None )
776 intentStates.append( ( intentId, state ) )
777 except ( ValueError, TypeError ):
778 main.log.exception( "Error parsing intents." )
779 # Print states
780 intentStates.sort()
781 for i, s in intentStates:
782 count += 1
783 main.log.info( "%-6s%-15s%-15s" %
784 ( str( count ), str( i ), str( s ) ) )
785 utilities.assert_equals( expect=True, actual=installedCheck,
786 onpass="Intents are all INSTALLED",
787 onfail="Intents are not all in " +
788 "INSTALLED state" )
789
790 main.step( "Check leadership of topics" )
791 leaders = main.ONOScli1.leaders()
792 topicCheck = main.TRUE
793 try:
794 if leaders:
795 parsedLeaders = json.loads( leaders )
796 main.log.warn( json.dumps( parsedLeaders,
797 sort_keys=True,
798 indent=4,
799 separators=( ',', ': ' ) ) )
800 # check for all intent partitions
801 # check for election
802 # TODO: Look at Devices as topics now that it uses this system
803 topics = []
804 for i in range( 14 ):
805 topics.append( "intent-partition-" + str( i ) )
806 # FIXME: this should only be after we start the app
807 # FIXME: topics.append( "org.onosproject.election" )
808 # Print leaders output
809 main.log.debug( topics )
810 ONOStopics = [ j['topic'] for j in parsedLeaders ]
811 for topic in topics:
812 if topic not in ONOStopics:
813 main.log.error( "Error: " + topic +
814 " not in leaders" )
815 topicCheck = main.FALSE
816 else:
817 main.log.error( "leaders() returned None" )
818 topicCheck = main.FALSE
819 except ( ValueError, TypeError ):
820 topicCheck = main.FALSE
821 main.log.exception( "Error parsing leaders" )
822 main.log.error( repr( leaders ) )
823 # TODO: Check for a leader of these topics
824 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
825 onpass="intent Partitions is in leaders",
826 onfail="Some topics were lost " )
827 # Print partitions
828 partitions = main.ONOScli1.partitions()
829 try:
830 if partitions :
831 parsedPartitions = json.loads( partitions )
832 main.log.warn( json.dumps( parsedPartitions,
833 sort_keys=True,
834 indent=4,
835 separators=( ',', ': ' ) ) )
836 # TODO check for a leader in all paritions
837 # TODO check for consistency among nodes
838 else:
839 main.log.error( "partitions() returned None" )
840 except ( ValueError, TypeError ):
841 main.log.exception( "Error parsing partitions" )
842 main.log.error( repr( partitions ) )
843 # Print Pending Map
844 pendingMap = main.ONOScli1.pendingMap()
845 try:
846 if pendingMap :
847 parsedPending = json.loads( pendingMap )
848 main.log.warn( json.dumps( parsedPending,
849 sort_keys=True,
850 indent=4,
851 separators=( ',', ': ' ) ) )
852 # TODO check something here?
853 else:
854 main.log.error( "pendingMap() returned None" )
855 except ( ValueError, TypeError ):
856 main.log.exception( "Error parsing pending map" )
857 main.log.error( repr( pendingMap ) )
858
859 if not installedCheck:
860 main.log.info( "Waiting 60 seconds to see if the state of " +
861 "intents change" )
862 time.sleep( 60 )
863 # Print the intent states
864 intents = main.ONOScli1.intents()
865 intentStates = []
866 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
867 count = 0
868 # Iter through intents of a node
869 try:
870 for intent in json.loads( intents ):
871 state = intent.get( 'state', None )
872 if "INSTALLED" not in state:
873 installedCheck = False
874 intentId = intent.get( 'id', None )
875 intentStates.append( ( intentId, state ) )
876 except ( ValueError, TypeError ):
877 main.log.exception( "Error parsing intents." )
878 intentStates.sort()
879 for i, s in intentStates:
880 count += 1
881 main.log.info( "%-6s%-15s%-15s" %
882 ( str( count ), str( i ), str( s ) ) )
883 leaders = main.ONOScli1.leaders()
884 try:
885 missing = False
886 if leaders:
887 parsedLeaders = json.loads( leaders )
888 main.log.warn( json.dumps( parsedLeaders,
889 sort_keys=True,
890 indent=4,
891 separators=( ',', ': ' ) ) )
892 # check for all intent partitions
893 # check for election
894 topics = []
895 for i in range( 14 ):
896 topics.append( "intent-partition-" + str( i ) )
897 # FIXME: this should only be after we start the app
898 topics.append( "org.onosproject.election" )
899 main.log.debug( topics )
900 ONOStopics = [ j['topic'] for j in parsedLeaders ]
901 for topic in topics:
902 if topic not in ONOStopics:
903 main.log.error( "Error: " + topic +
904 " not in leaders" )
905 missing = True
906 else:
907 main.log.error( "leaders() returned None" )
908 except ( ValueError, TypeError ):
909 main.log.exception( "Error parsing leaders" )
910 main.log.error( repr( leaders ) )
911 if missing:
912 response = main.ONOScli1.leaders( jsonFormat=False)
913 main.log.warn( "ONOS1 leaders output: \n" +
914 str( response ) )
915 partitions = main.ONOScli1.partitions()
916 try:
917 if partitions :
918 parsedPartitions = json.loads( partitions )
919 main.log.warn( json.dumps( parsedPartitions,
920 sort_keys=True,
921 indent=4,
922 separators=( ',', ': ' ) ) )
923 # TODO check for a leader in all paritions
924 # TODO check for consistency among nodes
925 else:
926 main.log.error( "partitions() returned None" )
927 except ( ValueError, TypeError ):
928 main.log.exception( "Error parsing partitions" )
929 main.log.error( repr( partitions ) )
930 pendingMap = main.ONOScli1.pendingMap()
931 try:
932 if pendingMap :
933 parsedPending = json.loads( pendingMap )
934 main.log.warn( json.dumps( parsedPending,
935 sort_keys=True,
936 indent=4,
937 separators=( ',', ': ' ) ) )
938 # TODO check something here?
939 else:
940 main.log.error( "pendingMap() returned None" )
941 except ( ValueError, TypeError ):
942 main.log.exception( "Error parsing pending map" )
943 main.log.error( repr( pendingMap ) )
944 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -0700945 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700946 main.step( "Wait a minute then ping again" )
947 # the wait is above
948 PingResult = main.TRUE
949 for i in range( 8, 18 ):
950 ping = main.Mininet1.pingHost( src="h" + str( i ),
951 target="h" + str( i + 10 ) )
952 PingResult = PingResult and ping
953 if ping == main.FALSE:
954 main.log.warn( "Ping failed between h" + str( i ) +
955 " and h" + str( i + 10 ) )
956 elif ping == main.TRUE:
957 main.log.info( "Ping test passed!" )
958 # Don't set PingResult or you'd override failures
959 if PingResult == main.FALSE:
960 main.log.error(
961 "Intents have not been installed correctly, pings failed." )
962 # TODO: pretty print
963 main.log.warn( "ONOS1 intents: " )
964 try:
965 tmpIntents = main.ONOScli1.intents()
966 main.log.warn( json.dumps( json.loads( tmpIntents ),
967 sort_keys=True,
968 indent=4,
969 separators=( ',', ': ' ) ) )
970 except ( ValueError, TypeError ):
971 main.log.warn( repr( tmpIntents ) )
972 utilities.assert_equals(
973 expect=main.TRUE,
974 actual=PingResult,
975 onpass="Intents have been installed correctly and pings work",
976 onfail="Intents have not been installed correctly, pings failed." )
977
978 def CASE5( self, main ):
979 """
980 Reading state of ONOS
981 """
982 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700983 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700984 assert main, "main not defined"
985 assert utilities.assert_equals, "utilities.assert_equals not defined"
986
987 main.case( "Setting up and gathering data for current state" )
988 # The general idea for this test case is to pull the state of
989 # ( intents,flows, topology,... ) from each ONOS node
990 # We can then compare them with each other and also with past states
991
992 main.step( "Check that each switch has a master" )
993 global mastershipState
994 mastershipState = '[]'
995
996 # Assert that each device has a master
997 rolesNotNull = main.ONOScli1.rolesNotNull()
998 utilities.assert_equals(
999 expect=main.TRUE,
1000 actual=rolesNotNull,
1001 onpass="Each device has a master",
1002 onfail="Some devices don't have a master assigned" )
1003
1004 main.step( "Get the Mastership of each switch" )
1005 ONOS1Mastership = main.ONOScli1.roles()
1006 # TODO: Make this a meaningful check
1007 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1008 main.log.error( "Error in getting ONOS roles" )
1009 main.log.warn(
1010 "ONOS1 mastership response: " +
1011 repr( ONOS1Mastership ) )
1012 consistentMastership = main.FALSE
1013 else:
1014 mastershipState = ONOS1Mastership
1015 consistentMastership = main.TRUE
1016
1017 main.step( "Get the intents from each controller" )
1018 global intentState
1019 intentState = []
1020 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1021 intentCheck = main.FALSE
1022 if "Error" in ONOS1Intents or not ONOS1Intents:
1023 main.log.error( "Error in getting ONOS intents" )
1024 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1025 else:
1026 intentCheck = main.TRUE
1027
1028 main.step( "Get the flows from each controller" )
1029 global flowState
1030 flowState = []
1031 flowCheck = main.FALSE
1032 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
1033 if "Error" in ONOS1Flows or not ONOS1Flows:
1034 main.log.error( "Error in getting ONOS flows" )
1035 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
1036 else:
1037 # TODO: Do a better check, maybe compare flows on switches?
1038 flowState = ONOS1Flows
1039 flowCheck = main.TRUE
1040
1041 main.step( "Get the OF Table entries" )
1042 global flows
1043 flows = []
1044 for i in range( 1, 29 ):
Jon Hall9043c902015-07-30 14:23:44 -07001045 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001046 if flowCheck == main.FALSE:
1047 for table in flows:
1048 main.log.warn( table )
1049 # TODO: Compare switch flow tables with ONOS flow tables
1050
1051 main.step( "Collecting topology information from ONOS" )
1052 devices = []
1053 devices.append( main.ONOScli1.devices() )
1054 hosts = []
1055 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1056 ports = []
1057 ports.append( main.ONOScli1.ports() )
1058 links = []
1059 links.append( main.ONOScli1.links() )
1060 clusters = []
1061 clusters.append( main.ONOScli1.clusters() )
1062
1063 main.step( "Each host has an IP address" )
1064 ipResult = main.TRUE
1065 for controller in range( 0, len( hosts ) ):
1066 controllerStr = str( controller + 1 )
1067 for host in hosts[ controller ]:
1068 if host is None or host.get( 'ipAddresses', [] ) == []:
1069 main.log.error(
1070 "DEBUG:Error with host ips on controller" +
1071 controllerStr + ": " + str( host ) )
1072 ipResult = main.FALSE
1073 utilities.assert_equals(
1074 expect=main.TRUE,
1075 actual=ipResult,
1076 onpass="The ips of the hosts aren't empty",
1077 onfail="The ip of at least one host is missing" )
1078
1079 # there should always only be one cluster
1080 main.step( "There is only one dataplane cluster" )
1081 try:
1082 numClusters = len( json.loads( clusters[ 0 ] ) )
1083 except ( ValueError, TypeError ):
1084 main.log.exception( "Error parsing clusters[0]: " +
1085 repr( clusters[ 0 ] ) )
1086 clusterResults = main.FALSE
1087 if numClusters == 1:
1088 clusterResults = main.TRUE
1089 utilities.assert_equals(
1090 expect=1,
1091 actual=numClusters,
1092 onpass="ONOS shows 1 SCC",
1093 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1094
1095 main.step( "Comparing ONOS topology to MN" )
1096 devicesResults = main.TRUE
1097 linksResults = main.TRUE
1098 hostsResults = main.TRUE
1099 mnSwitches = main.Mininet1.getSwitches()
1100 mnLinks = main.Mininet1.getLinks()
1101 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001102 for controller in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07001103 controllerStr = str( controller + 1 )
1104 if devices[ controller ] and ports[ controller ] and\
1105 "Error" not in devices[ controller ] and\
1106 "Error" not in ports[ controller ]:
1107
1108 currentDevicesResult = main.Mininet1.compareSwitches(
1109 mnSwitches,
1110 json.loads( devices[ controller ] ),
1111 json.loads( ports[ controller ] ) )
1112 else:
1113 currentDevicesResult = main.FALSE
1114 utilities.assert_equals( expect=main.TRUE,
1115 actual=currentDevicesResult,
1116 onpass="ONOS" + controllerStr +
1117 " Switches view is correct",
1118 onfail="ONOS" + controllerStr +
1119 " Switches view is incorrect" )
1120 if links[ controller ] and "Error" not in links[ controller ]:
1121 currentLinksResult = main.Mininet1.compareLinks(
1122 mnSwitches, mnLinks,
1123 json.loads( links[ controller ] ) )
1124 else:
1125 currentLinksResult = main.FALSE
1126 utilities.assert_equals( expect=main.TRUE,
1127 actual=currentLinksResult,
1128 onpass="ONOS" + controllerStr +
1129 " links view is correct",
1130 onfail="ONOS" + controllerStr +
1131 " links view is incorrect" )
1132
1133 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1134 currentHostsResult = main.Mininet1.compareHosts(
1135 mnHosts,
1136 hosts[ controller ] )
1137 else:
1138 currentHostsResult = main.FALSE
1139 utilities.assert_equals( expect=main.TRUE,
1140 actual=currentHostsResult,
1141 onpass="ONOS" + controllerStr +
1142 " hosts exist in Mininet",
1143 onfail="ONOS" + controllerStr +
1144 " hosts don't match Mininet" )
1145
1146 devicesResults = devicesResults and currentDevicesResult
1147 linksResults = linksResults and currentLinksResult
1148 hostsResults = hostsResults and currentHostsResult
1149
1150 main.step( "Device information is correct" )
1151 utilities.assert_equals(
1152 expect=main.TRUE,
1153 actual=devicesResults,
1154 onpass="Device information is correct",
1155 onfail="Device information is incorrect" )
1156
1157 main.step( "Links are correct" )
1158 utilities.assert_equals(
1159 expect=main.TRUE,
1160 actual=linksResults,
1161 onpass="Link are correct",
1162 onfail="Links are incorrect" )
1163
1164 main.step( "Hosts are correct" )
1165 utilities.assert_equals(
1166 expect=main.TRUE,
1167 actual=hostsResults,
1168 onpass="Hosts are correct",
1169 onfail="Hosts are incorrect" )
1170
1171 def CASE6( self, main ):
1172 """
1173 The Failure case.
1174 """
1175 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001176 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001177 assert main, "main not defined"
1178 assert utilities.assert_equals, "utilities.assert_equals not defined"
1179
1180 # Reset non-persistent variables
1181 try:
1182 iCounterValue = 0
1183 except NameError:
1184 main.log.error( "iCounterValue not defined, setting to 0" )
1185 iCounterValue = 0
1186
1187 main.case( "Restart ONOS node" )
Jon Hall783bbf92015-07-23 14:33:19 -07001188 main.caseExplanation = "Killing ONOS process and restart cli " +\
Jon Hall85794ff2015-07-08 14:12:30 -07001189 "sessions once onos is up."
Jon Hall96091e62015-09-21 17:34:17 -07001190
1191 main.step( "Checking ONOS Logs for errors" )
1192 for node in main.nodes:
1193 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1194 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1195
Jon Hall85794ff2015-07-08 14:12:30 -07001196 main.step( "Killing ONOS processes" )
Jon Halle1a3b752015-07-22 13:02:46 -07001197 killResult = main.ONOSbench.onosKill( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001198 start = time.time()
1199 utilities.assert_equals( expect=main.TRUE, actual=killResult,
1200 onpass="ONOS Killed",
1201 onfail="Error killing ONOS" )
1202
1203 main.step( "Checking if ONOS is up yet" )
1204 count = 0
1205 while count < 10:
Jon Halle1a3b752015-07-22 13:02:46 -07001206 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001207 if onos1Isup == main.TRUE:
1208 elapsed = time.time() - start
1209 break
1210 else:
1211 count = count + 1
1212 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
1213 onpass="ONOS is back up",
1214 onfail="ONOS failed to start" )
1215
1216 main.log.step( "Starting ONOS CLI sessions" )
Jon Halle1a3b752015-07-22 13:02:46 -07001217 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001218 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1219 onpass="ONOS cli startup successful",
1220 onfail="ONOS cli startup failed" )
1221
1222 if elapsed:
1223 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
1224 str( elapsed ) )
1225 main.restartTime = elapsed
1226 else:
1227 main.restartTime = -1
1228 time.sleep( 5 )
1229 # rerun on election apps
1230 main.ONOScli1.electionTestRun()
1231
1232 def CASE7( self, main ):
1233 """
1234 Check state after ONOS failure
1235 """
1236 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001237 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001238 assert main, "main not defined"
1239 assert utilities.assert_equals, "utilities.assert_equals not defined"
1240 main.case( "Running ONOS Constant State Tests" )
1241 main.step( "Check that each switch has a master" )
1242 # Assert that each device has a master
1243 rolesNotNull = main.ONOScli1.rolesNotNull()
1244 utilities.assert_equals(
1245 expect=main.TRUE,
1246 actual=rolesNotNull,
1247 onpass="Each device has a master",
1248 onfail="Some devices don't have a master assigned" )
1249
1250 main.step( "Check if switch roles are consistent across all nodes" )
1251 ONOS1Mastership = main.ONOScli1.roles()
1252 # FIXME: Refactor this whole case for single instance
1253 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1254 main.log.error( "Error in getting ONOS mastership" )
1255 main.log.warn( "ONOS1 mastership response: " +
1256 repr( ONOS1Mastership ) )
1257 consistentMastership = main.FALSE
1258 else:
1259 consistentMastership = main.TRUE
1260 utilities.assert_equals(
1261 expect=main.TRUE,
1262 actual=consistentMastership,
1263 onpass="Switch roles are consistent across all ONOS nodes",
1264 onfail="ONOS nodes have different views of switch roles" )
1265
1266 description2 = "Compare switch roles from before failure"
1267 main.step( description2 )
1268
1269 currentJson = json.loads( ONOS1Mastership )
1270 oldJson = json.loads( mastershipState )
1271 mastershipCheck = main.TRUE
1272 for i in range( 1, 29 ):
1273 switchDPID = str(
1274 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1275
1276 current = [ switch[ 'master' ] for switch in currentJson
1277 if switchDPID in switch[ 'id' ] ]
1278 old = [ switch[ 'master' ] for switch in oldJson
1279 if switchDPID in switch[ 'id' ] ]
1280 if current == old:
1281 mastershipCheck = mastershipCheck and main.TRUE
1282 else:
1283 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1284 mastershipCheck = main.FALSE
1285 utilities.assert_equals(
1286 expect=main.TRUE,
1287 actual=mastershipCheck,
1288 onpass="Mastership of Switches was not changed",
1289 onfail="Mastership of some switches changed" )
1290 mastershipCheck = mastershipCheck and consistentMastership
1291
1292 main.step( "Get the intents and compare across all nodes" )
1293 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1294 intentCheck = main.FALSE
1295 if "Error" in ONOS1Intents or not ONOS1Intents:
1296 main.log.error( "Error in getting ONOS intents" )
1297 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1298 else:
1299 intentCheck = main.TRUE
1300 utilities.assert_equals(
1301 expect=main.TRUE,
1302 actual=intentCheck,
1303 onpass="Intents are consistent across all ONOS nodes",
1304 onfail="ONOS nodes have different views of intents" )
1305 # Print the intent states
1306 intents = []
1307 intents.append( ONOS1Intents )
1308 intentStates = []
1309 for node in intents: # Iter through ONOS nodes
1310 nodeStates = []
1311 # Iter through intents of a node
1312 for intent in json.loads( node ):
1313 nodeStates.append( intent[ 'state' ] )
1314 intentStates.append( nodeStates )
1315 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1316 main.log.info( dict( out ) )
1317
1318 # NOTE: Store has no durability, so intents are lost across system
1319 # restarts
1320 """
1321 main.step( "Compare current intents with intents before the failure" )
1322 # NOTE: this requires case 5 to pass for intentState to be set.
1323 # maybe we should stop the test if that fails?
1324 sameIntents = main.FALSE
1325 if intentState and intentState == ONOSIntents[ 0 ]:
1326 sameIntents = main.TRUE
1327 main.log.info( "Intents are consistent with before failure" )
1328 # TODO: possibly the states have changed? we may need to figure out
1329 # what the acceptable states are
1330 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1331 sameIntents = main.TRUE
1332 try:
1333 before = json.loads( intentState )
1334 after = json.loads( ONOSIntents[ 0 ] )
1335 for intent in before:
1336 if intent not in after:
1337 sameIntents = main.FALSE
1338 main.log.debug( "Intent is not currently in ONOS " +
1339 "(at least in the same form):" )
1340 main.log.debug( json.dumps( intent ) )
1341 except ( ValueError, TypeError ):
1342 main.log.exception( "Exception printing intents" )
1343 main.log.debug( repr( ONOSIntents[0] ) )
1344 main.log.debug( repr( intentState ) )
1345 if sameIntents == main.FALSE:
1346 try:
1347 main.log.debug( "ONOS intents before: " )
1348 main.log.debug( json.dumps( json.loads( intentState ),
1349 sort_keys=True, indent=4,
1350 separators=( ',', ': ' ) ) )
1351 main.log.debug( "Current ONOS intents: " )
1352 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1353 sort_keys=True, indent=4,
1354 separators=( ',', ': ' ) ) )
1355 except ( ValueError, TypeError ):
1356 main.log.exception( "Exception printing intents" )
1357 main.log.debug( repr( ONOSIntents[0] ) )
1358 main.log.debug( repr( intentState ) )
1359 utilities.assert_equals(
1360 expect=main.TRUE,
1361 actual=sameIntents,
1362 onpass="Intents are consistent with before failure",
1363 onfail="The Intents changed during failure" )
1364 intentCheck = intentCheck and sameIntents
1365 """
1366 main.step( "Get the OF Table entries and compare to before " +
1367 "component failure" )
1368 FlowTables = main.TRUE
1369 flows2 = []
1370 for i in range( 28 ):
1371 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall9043c902015-07-30 14:23:44 -07001372 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001373 flows2.append( tmpFlows )
Jon Hall9043c902015-07-30 14:23:44 -07001374 tempResult = main.Mininet1.flowComp(
Jon Hall85794ff2015-07-08 14:12:30 -07001375 flow1=flows[ i ],
1376 flow2=tmpFlows )
1377 FlowTables = FlowTables and tempResult
1378 if FlowTables == main.FALSE:
1379 main.log.info( "Differences in flow table for switch: s" +
1380 str( i + 1 ) )
1381 utilities.assert_equals(
1382 expect=main.TRUE,
1383 actual=FlowTables,
1384 onpass="No changes were found in the flow tables",
1385 onfail="Changes were found in the flow tables" )
1386
1387 main.step( "Leadership Election is still functional" )
1388 # Test of LeadershipElection
1389
Jon Halle1a3b752015-07-22 13:02:46 -07001390 leader = main.nodes[0].ip_address
Jon Hall85794ff2015-07-08 14:12:30 -07001391 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001392 for controller in range( 1, main.numCtrls + 1 ):
Jon Hall85794ff2015-07-08 14:12:30 -07001393 # loop through ONOScli handlers
1394 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
1395 leaderN = node.electionTestLeader()
1396 # verify leader is ONOS1
1397 # NOTE even though we restarted ONOS, it is the only one so onos 1
1398 # must be leader
1399 if leaderN == leader:
1400 # all is well
1401 pass
1402 elif leaderN == main.FALSE:
1403 # error in response
1404 main.log.error( "Something is wrong with " +
1405 "electionTestLeader function, check the" +
1406 " error logs" )
1407 leaderResult = main.FALSE
1408 elif leader != leaderN:
1409 leaderResult = main.FALSE
1410 main.log.error( "ONOS" + str( controller ) + " sees " +
1411 str( leaderN ) +
1412 " as the leader of the election app. " +
1413 "Leader should be " + str( leader ) )
1414 utilities.assert_equals(
1415 expect=main.TRUE,
1416 actual=leaderResult,
1417 onpass="Leadership election passed",
1418 onfail="Something went wrong with Leadership election" )
1419
1420 def CASE8( self, main ):
1421 """
1422 Compare topo
1423 """
1424 import json
1425 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001426 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001427 assert main, "main not defined"
1428 assert utilities.assert_equals, "utilities.assert_equals not defined"
1429
1430 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07001431 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall85794ff2015-07-08 14:12:30 -07001432 " and ONOS"
1433
1434 main.step( "Comparing ONOS topology to MN" )
Jon Hall85794ff2015-07-08 14:12:30 -07001435 topoResult = main.FALSE
1436 elapsed = 0
1437 count = 0
1438 main.step( "Collecting topology information from ONOS" )
1439 startTime = time.time()
1440 # Give time for Gossip to work
1441 while topoResult == main.FALSE and elapsed < 60:
Jon Hall96091e62015-09-21 17:34:17 -07001442 devicesResults = main.TRUE
1443 linksResults = main.TRUE
1444 hostsResults = main.TRUE
1445 hostAttachmentResults = True
Jon Hall85794ff2015-07-08 14:12:30 -07001446 count += 1
1447 cliStart = time.time()
1448 devices = []
1449 devices.append( main.ONOScli1.devices() )
1450 hosts = []
1451 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1452 ipResult = main.TRUE
1453 for controller in range( 0, len( hosts ) ):
1454 controllerStr = str( controller + 1 )
1455 for host in hosts[ controller ]:
1456 if host is None or host.get( 'ipAddresses', [] ) == []:
1457 main.log.error(
1458 "DEBUG:Error with host ips on controller" +
1459 controllerStr + ": " + str( host ) )
1460 ipResult = main.FALSE
1461 ports = []
1462 ports.append( main.ONOScli1.ports() )
1463 links = []
1464 links.append( main.ONOScli1.links() )
1465 clusters = []
1466 clusters.append( main.ONOScli1.clusters() )
1467
1468 elapsed = time.time() - startTime
1469 cliTime = time.time() - cliStart
1470 print "CLI time: " + str( cliTime )
1471
1472 mnSwitches = main.Mininet1.getSwitches()
1473 mnLinks = main.Mininet1.getLinks()
1474 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001475 for controller in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07001476 controllerStr = str( controller + 1 )
1477 if devices[ controller ] and ports[ controller ] and\
1478 "Error" not in devices[ controller ] and\
1479 "Error" not in ports[ controller ]:
1480
1481 currentDevicesResult = main.Mininet1.compareSwitches(
1482 mnSwitches,
1483 json.loads( devices[ controller ] ),
1484 json.loads( ports[ controller ] ) )
1485 else:
1486 currentDevicesResult = main.FALSE
1487 utilities.assert_equals( expect=main.TRUE,
1488 actual=currentDevicesResult,
1489 onpass="ONOS" + controllerStr +
1490 " Switches view is correct",
1491 onfail="ONOS" + controllerStr +
1492 " Switches view is incorrect" )
1493
1494 if links[ controller ] and "Error" not in links[ controller ]:
1495 currentLinksResult = main.Mininet1.compareLinks(
1496 mnSwitches, mnLinks,
1497 json.loads( links[ controller ] ) )
1498 else:
1499 currentLinksResult = main.FALSE
1500 utilities.assert_equals( expect=main.TRUE,
1501 actual=currentLinksResult,
1502 onpass="ONOS" + controllerStr +
1503 " links view is correct",
1504 onfail="ONOS" + controllerStr +
1505 " links view is incorrect" )
1506
1507 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1508 currentHostsResult = main.Mininet1.compareHosts(
1509 mnHosts,
1510 hosts[ controller ] )
1511 else:
1512 currentHostsResult = main.FALSE
1513 utilities.assert_equals( expect=main.TRUE,
1514 actual=currentHostsResult,
1515 onpass="ONOS" + controllerStr +
1516 " hosts exist in Mininet",
1517 onfail="ONOS" + controllerStr +
1518 " hosts don't match Mininet" )
1519 # CHECKING HOST ATTACHMENT POINTS
1520 hostAttachment = True
1521 zeroHosts = False
1522 # FIXME: topo-HA/obelisk specific mappings:
1523 # key is mac and value is dpid
1524 mappings = {}
1525 for i in range( 1, 29 ): # hosts 1 through 28
1526 # set up correct variables:
1527 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
1528 if i == 1:
1529 deviceId = "1000".zfill(16)
1530 elif i == 2:
1531 deviceId = "2000".zfill(16)
1532 elif i == 3:
1533 deviceId = "3000".zfill(16)
1534 elif i == 4:
1535 deviceId = "3004".zfill(16)
1536 elif i == 5:
1537 deviceId = "5000".zfill(16)
1538 elif i == 6:
1539 deviceId = "6000".zfill(16)
1540 elif i == 7:
1541 deviceId = "6007".zfill(16)
1542 elif i >= 8 and i <= 17:
1543 dpid = '3' + str( i ).zfill( 3 )
1544 deviceId = dpid.zfill(16)
1545 elif i >= 18 and i <= 27:
1546 dpid = '6' + str( i ).zfill( 3 )
1547 deviceId = dpid.zfill(16)
1548 elif i == 28:
1549 deviceId = "2800".zfill(16)
1550 mappings[ macId ] = deviceId
1551 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1552 if hosts[ controller ] == []:
1553 main.log.warn( "There are no hosts discovered" )
1554 zeroHosts = True
1555 else:
1556 for host in hosts[ controller ]:
1557 mac = None
1558 location = None
1559 device = None
1560 port = None
1561 try:
1562 mac = host.get( 'mac' )
1563 assert mac, "mac field could not be found for this host object"
1564
1565 location = host.get( 'location' )
1566 assert location, "location field could not be found for this host object"
1567
1568 # Trim the protocol identifier off deviceId
1569 device = str( location.get( 'elementId' ) ).split(':')[1]
1570 assert device, "elementId field could not be found for this host location object"
1571
1572 port = location.get( 'port' )
1573 assert port, "port field could not be found for this host location object"
1574
1575 # Now check if this matches where they should be
1576 if mac and device and port:
1577 if str( port ) != "1":
1578 main.log.error( "The attachment port is incorrect for " +
1579 "host " + str( mac ) +
1580 ". Expected: 1 Actual: " + str( port) )
1581 hostAttachment = False
1582 if device != mappings[ str( mac ) ]:
1583 main.log.error( "The attachment device is incorrect for " +
1584 "host " + str( mac ) +
1585 ". Expected: " + mappings[ str( mac ) ] +
1586 " Actual: " + device )
1587 hostAttachment = False
1588 else:
1589 hostAttachment = False
1590 except AssertionError:
1591 main.log.exception( "Json object not as expected" )
1592 main.log.error( repr( host ) )
1593 hostAttachment = False
1594 else:
1595 main.log.error( "No hosts json output or \"Error\"" +
1596 " in output. hosts = " +
1597 repr( hosts[ controller ] ) )
1598 if zeroHosts is False:
1599 hostAttachment = True
1600
Jon Hall85794ff2015-07-08 14:12:30 -07001601 devicesResults = devicesResults and currentDevicesResult
1602 linksResults = linksResults and currentLinksResult
1603 hostsResults = hostsResults and currentHostsResult
1604 hostAttachmentResults = hostAttachmentResults and\
1605 hostAttachment
1606
1607 # "consistent" results don't make sense for single instance
1608 # there should always only be one cluster
1609 numClusters = len( json.loads( clusters[ 0 ] ) )
1610 clusterResults = main.FALSE
1611 if numClusters == 1:
1612 clusterResults = main.TRUE
1613 utilities.assert_equals(
1614 expect=1,
1615 actual=numClusters,
1616 onpass="ONOS shows 1 SCC",
1617 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1618
1619 topoResult = ( devicesResults and linksResults
1620 and hostsResults and ipResult and clusterResults and
1621 hostAttachmentResults )
1622
1623 topoResult = topoResult and int( count <= 2 )
1624 note = "note it takes about " + str( int( cliTime ) ) + \
1625 " seconds for the test to make all the cli calls to fetch " +\
1626 "the topology from each ONOS instance"
1627 main.log.info(
1628 "Very crass estimate for topology discovery/convergence( " +
1629 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1630 str( count ) + " tries" )
1631 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1632 onpass="Topology Check Test successful",
1633 onfail="Topology Check Test NOT successful" )
1634
1635 def CASE9( self, main ):
1636 """
1637 Link s3-s28 down
1638 """
1639 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001640 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001641 assert main, "main not defined"
1642 assert utilities.assert_equals, "utilities.assert_equals not defined"
1643 # NOTE: You should probably run a topology check after this
1644
1645 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1646
1647 description = "Turn off a link to ensure that Link Discovery " +\
1648 "is working properly"
1649 main.case( description )
1650
1651 main.step( "Kill Link between s3 and s28" )
1652 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
1653 main.log.info( "Waiting " + str( linkSleep ) +
1654 " seconds for link down to be discovered" )
1655 time.sleep( linkSleep )
1656 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
1657 onpass="Link down successful",
1658 onfail="Failed to bring link down" )
1659 # TODO do some sort of check here
1660
1661 def CASE10( self, main ):
1662 """
1663 Link s3-s28 up
1664 """
1665 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001666 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001667 assert main, "main not defined"
1668 assert utilities.assert_equals, "utilities.assert_equals not defined"
1669 # NOTE: You should probably run a topology check after this
1670
1671 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1672
1673 description = "Restore a link to ensure that Link Discovery is " + \
1674 "working properly"
1675 main.case( description )
1676
1677 main.step( "Bring link between s3 and s28 back up" )
1678 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
1679 main.log.info( "Waiting " + str( linkSleep ) +
1680 " seconds for link up to be discovered" )
1681 time.sleep( linkSleep )
1682 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
1683 onpass="Link up successful",
1684 onfail="Failed to bring link up" )
1685 # TODO do some sort of check here
1686
1687 def CASE11( self, main ):
1688 """
1689 Switch Down
1690 """
1691 # NOTE: You should probably run a topology check after this
1692 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001693 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001694 assert main, "main not defined"
1695 assert utilities.assert_equals, "utilities.assert_equals not defined"
1696
1697 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1698
1699 description = "Killing a switch to ensure it is discovered correctly"
1700 main.case( description )
1701 switch = main.params[ 'kill' ][ 'switch' ]
1702 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1703
1704 # TODO: Make this switch parameterizable
1705 main.step( "Kill " + switch )
1706 main.log.info( "Deleting " + switch )
1707 main.Mininet1.delSwitch( switch )
1708 main.log.info( "Waiting " + str( switchSleep ) +
1709 " seconds for switch down to be discovered" )
1710 time.sleep( switchSleep )
1711 device = main.ONOScli1.getDevice( dpid=switchDPID )
1712 # Peek at the deleted switch
1713 main.log.warn( str( device ) )
1714 result = main.FALSE
1715 if device and device[ 'available' ] is False:
1716 result = main.TRUE
1717 utilities.assert_equals( expect=main.TRUE, actual=result,
1718 onpass="Kill switch successful",
1719 onfail="Failed to kill switch?" )
1720
1721 def CASE12( self, main ):
1722 """
1723 Switch Up
1724 """
1725 # NOTE: You should probably run a topology check after this
1726 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001727 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001728 assert main, "main not defined"
1729 assert utilities.assert_equals, "utilities.assert_equals not defined"
1730
1731 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1732 switch = main.params[ 'kill' ][ 'switch' ]
1733 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1734 links = main.params[ 'kill' ][ 'links' ].split()
1735 description = "Adding a switch to ensure it is discovered correctly"
1736 main.case( description )
1737
1738 main.step( "Add back " + switch )
1739 main.Mininet1.addSwitch( switch, dpid=switchDPID )
1740 for peer in links:
1741 main.Mininet1.addLink( switch, peer )
1742 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07001743 for i in range( main.numCtrls ):
1744 ipList.append( main.nodes[ i ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001745 main.Mininet1.assignSwController( sw=switch, ip=ipList )
1746 main.log.info( "Waiting " + str( switchSleep ) +
1747 " seconds for switch up to be discovered" )
1748 time.sleep( switchSleep )
1749 device = main.ONOScli1.getDevice( dpid=switchDPID )
1750 # Peek at the deleted switch
1751 main.log.warn( str( device ) )
1752 result = main.FALSE
1753 if device and device[ 'available' ]:
1754 result = main.TRUE
1755 utilities.assert_equals( expect=main.TRUE, actual=result,
1756 onpass="add switch successful",
1757 onfail="Failed to add switch?" )
1758
1759 def CASE13( self, main ):
1760 """
1761 Clean up
1762 """
1763 import os
1764 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001765 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001766 assert main, "main not defined"
1767 assert utilities.assert_equals, "utilities.assert_equals not defined"
1768 # printing colors to terminal
1769 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
1770 'blue': '\033[94m', 'green': '\033[92m',
1771 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
1772 main.case( "Test Cleanup" )
1773 main.step( "Killing tcpdumps" )
1774 main.Mininet2.stopTcpdump()
1775
1776 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07001777 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall85794ff2015-07-08 14:12:30 -07001778 main.step( "Copying MN pcap and ONOS log files to test station" )
1779 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
1780 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07001781 # NOTE: MN Pcap file is being saved to logdir.
1782 # We scp this file as MN and TestON aren't necessarily the same vm
1783
1784 # FIXME: To be replaced with a Jenkin's post script
Jon Hall85794ff2015-07-08 14:12:30 -07001785 # TODO: Load these from params
1786 # NOTE: must end in /
1787 logFolder = "/opt/onos/log/"
1788 logFiles = [ "karaf.log", "karaf.log.1" ]
1789 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07001790 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07001791 for node in main.nodes:
1792 dstName = main.logdir + "/" + node.name + "-" + f
1793 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
1794 logFolder + f, dstName )
Jon Hall85794ff2015-07-08 14:12:30 -07001795 # std*.log's
1796 # NOTE: must end in /
1797 logFolder = "/opt/onos/var/"
1798 logFiles = [ "stderr.log", "stdout.log" ]
1799 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07001800 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07001801 for node in main.nodes:
1802 dstName = main.logdir + "/" + node.name + "-" + f
1803 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
1804 logFolder + f, dstName )
1805 else:
1806 main.log.debug( "skipping saving log files" )
Jon Hall85794ff2015-07-08 14:12:30 -07001807
Jon Hall85794ff2015-07-08 14:12:30 -07001808 main.step( "Stopping Mininet" )
1809 mnResult = main.Mininet1.stopNet()
1810 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
1811 onpass="Mininet stopped",
1812 onfail="MN cleanup NOT successful" )
1813
1814 main.step( "Checking ONOS Logs for errors" )
Jon Hall96091e62015-09-21 17:34:17 -07001815 for node in main.nodes:
1816 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1817 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001818
1819 try:
1820 timerLog = open( main.logdir + "/Timers.csv", 'w')
1821 # Overwrite with empty line and close
1822 labels = "Gossip Intents, Restart"
1823 data = str( gossipTime ) + ", " + str( main.restartTime )
1824 timerLog.write( labels + "\n" + data )
1825 timerLog.close()
1826 except NameError, e:
1827 main.log.exception(e)
1828
1829 def CASE14( self, main ):
1830 """
1831 start election app on all onos nodes
1832 """
Jon Halle1a3b752015-07-22 13:02:46 -07001833 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001834 assert main, "main not defined"
1835 assert utilities.assert_equals, "utilities.assert_equals not defined"
1836
1837 main.case("Start Leadership Election app")
1838 main.step( "Install leadership election app" )
1839 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
1840 utilities.assert_equals(
1841 expect=main.TRUE,
1842 actual=appResult,
1843 onpass="Election app installed",
1844 onfail="Something went wrong with installing Leadership election" )
1845
1846 main.step( "Run for election on each node" )
1847 leaderResult = main.ONOScli1.electionTestRun()
1848 # check for leader
1849 leader = main.ONOScli1.electionTestLeader()
1850 # verify leader is ONOS1
Jon Halle1a3b752015-07-22 13:02:46 -07001851 if leader == main.nodes[0].ip_address:
Jon Hall85794ff2015-07-08 14:12:30 -07001852 # all is well
1853 pass
1854 elif leader is None:
1855 # No leader elected
1856 main.log.error( "No leader was elected" )
1857 leaderResult = main.FALSE
1858 elif leader == main.FALSE:
1859 # error in response
1860 # TODO: add check for "Command not found:" in the driver, this
1861 # means the app isn't loaded
1862 main.log.error( "Something is wrong with electionTestLeader" +
1863 " function, check the error logs" )
1864 leaderResult = main.FALSE
1865 else:
1866 # error in response
1867 main.log.error(
1868 "Unexpected response from electionTestLeader function:'" +
1869 str( leader ) +
1870 "'" )
1871 leaderResult = main.FALSE
1872 utilities.assert_equals(
1873 expect=main.TRUE,
1874 actual=leaderResult,
1875 onpass="Successfully ran for leadership",
1876 onfail="Failed to run for leadership" )
1877
1878 def CASE15( self, main ):
1879 """
1880 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07001881 15.1 Run election on each node
1882 15.2 Check that each node has the same leaders and candidates
1883 15.3 Find current leader and withdraw
1884 15.4 Check that a new node was elected leader
1885 15.5 Check that that new leader was the candidate of old leader
1886 15.6 Run for election on old leader
1887 15.7 Check that oldLeader is a candidate, and leader if only 1 node
1888 15.8 Make sure that the old leader was added to the candidate list
1889
1890 old and new variable prefixes refer to data from before vs after
1891 withdrawl and later before withdrawl vs after re-election
Jon Hall85794ff2015-07-08 14:12:30 -07001892 """
acsmars71adceb2015-08-31 15:09:26 -07001893 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001894 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001895 assert main, "main not defined"
1896 assert utilities.assert_equals, "utilities.assert_equals not defined"
acsmars71adceb2015-08-31 15:09:26 -07001897 assert main.CLIs, "main.CLIs not defined"
1898 assert main.nodes, "main.nodes not defined"
1899
Jon Hall85794ff2015-07-08 14:12:30 -07001900 description = "Check that Leadership Election is still functional"
1901 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07001902 # NOTE: Need to re-run since being a canidate is not persistant
1903 # TODO: add check for "Command not found:" in the driver, this
1904 # means the election test app isn't loaded
acsmars2c2fcdd2015-08-25 17:14:13 -07001905
acsmars71adceb2015-08-31 15:09:26 -07001906 oldLeaders = [] # leaders by node before withdrawl from candidates
1907 newLeaders = [] # leaders by node after withdrawl from candidates
1908 oldAllCandidates = [] # list of lists of each nodes' candidates before
1909 newAllCandidates = [] # list of lists of each nodes' candidates after
1910 oldCandidates = [] # list of candidates from node 0 before withdrawl
1911 newCandidates = [] # list of candidates from node 0 after withdrawl
1912 oldLeader = '' # the old leader from oldLeaders, None if not same
1913 newLeader = '' # the new leaders fron newLoeaders, None if not same
1914 oldLeaderCLI = None # the CLI of the old leader used for re-electing
1915 expectNoLeader = False # True when there is only one leader
1916 if main.numCtrls == 1:
1917 expectNoLeader = True
acsmars2c2fcdd2015-08-25 17:14:13 -07001918
acsmars71adceb2015-08-31 15:09:26 -07001919 main.step( "Run for election on each node" )
1920 electionResult = main.TRUE
1921
1922 for cli in main.CLIs: # run test election on each node
1923 if cli.electionTestRun() == main.FALSE:
1924 electionResult = main.FALSE
1925
1926 utilities.assert_equals(
1927 expect=main.TRUE,
1928 actual=electionResult,
1929 onpass="All nodes successfully ran for leadership",
1930 onfail="At least one node failed to run for leadership" )
1931
acsmars3a72bde2015-09-02 14:16:22 -07001932 if electionResult == main.FALSE:
1933 main.log.error(
1934 "Skipping Test Case because Election Test App isn't loaded" )
1935 main.skipCase()
1936
acsmars71adceb2015-08-31 15:09:26 -07001937 main.step( "Check that each node shows the same leader and candidates" )
1938 sameResult = main.TRUE
1939 failMessage = "Nodes have different leaders"
1940 for cli in main.CLIs:
1941 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
1942 oldAllCandidates.append( node )
1943 oldLeaders.append( node[ 0 ] )
1944 oldCandidates = oldAllCandidates[ 0 ]
1945
1946 # Check that each node has the same leader. Defines oldLeader
1947 if len( set( oldLeaders ) ) != 1:
1948 sameResult = main.FALSE
1949 main.log.error( "More than one leader present:" + str( oldLeaders ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001950 oldLeader = None
1951 else:
acsmars71adceb2015-08-31 15:09:26 -07001952 oldLeader = oldLeaders[ 0 ]
1953
1954 # Check that each node's candidate list is the same
1955 for candidates in oldAllCandidates:
1956 if set( candidates ) != set( oldCandidates ):
1957 sameResult = main.FALSE
1958 failMessage += "and candidates"
1959
1960 utilities.assert_equals(
1961 expect=main.TRUE,
1962 actual=sameResult,
1963 onpass="Leadership is consistent for the election topic",
1964 onfail=failMessage )
1965
1966 main.step( "Find current leader and withdraw" )
1967 withdrawResult = main.TRUE
1968 # do some sanity checking on leader before using it
1969 if oldLeader is None:
1970 main.log.error( "Leadership isn't consistent." )
1971 withdrawResult = main.FALSE
1972 # Get the CLI of the oldLeader
1973 for i in range( len( main.CLIs ) ):
1974 if oldLeader == main.nodes[ i ].ip_address:
1975 oldLeaderCLI = main.CLIs[ i ]
1976 break
1977 else: # FOR/ELSE statement
1978 main.log.error( "Leader election, could not find current leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07001979 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07001980 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall85794ff2015-07-08 14:12:30 -07001981 utilities.assert_equals(
1982 expect=main.TRUE,
1983 actual=withdrawResult,
1984 onpass="Node was withdrawn from election",
1985 onfail="Node was not withdrawn from election" )
1986
acsmars71adceb2015-08-31 15:09:26 -07001987 main.step( "Check that a new node was elected leader" )
1988
1989 # FIXME: use threads
1990 newLeaderResult = main.TRUE
1991 failMessage = "Nodes have different leaders"
1992
1993 # Get new leaders and candidates
1994 for cli in main.CLIs:
1995 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
1996 # elections might no have finished yet
1997 if node[ 0 ] == 'none' and not expectNoLeader:
1998 main.log.info( "Node has no leader, waiting 5 seconds to be " +
1999 "sure elections are complete." )
2000 time.sleep(5)
2001 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2002 # election still isn't done or there is a problem
2003 if node[ 0 ] == 'none':
2004 main.log.error( "No leader was elected on at least 1 node" )
2005 newLeaderResult = main.FALSE
2006 newAllCandidates.append( node )
2007 newLeaders.append( node[ 0 ] )
2008 newCandidates = newAllCandidates[ 0 ]
2009
2010 # Check that each node has the same leader. Defines newLeader
2011 if len( set( newLeaders ) ) != 1:
2012 newLeaderResult = main.FALSE
2013 main.log.error( "Nodes have different leaders: " +
2014 str( newLeaders ) )
2015 newLeader = None
2016 else:
2017 newLeader = newLeaders[ 0 ]
2018
2019 # Check that each node's candidate list is the same
2020 for candidates in newAllCandidates:
2021 if set( candidates ) != set( newCandidates ):
2022 newLeaderResult = main.FALSE
2023 main.error.log( "Discrepancy in candidate lists detected" )
2024
2025 # Check that the new leader is not the older leader, which was withdrawn
2026 if newLeader == oldLeader:
2027 newLeaderResult = main.FALSE
2028 main.log.error( "All nodes still see old leader: " + oldLeader +
2029 " as the current leader" )
2030
Jon Hall85794ff2015-07-08 14:12:30 -07002031 utilities.assert_equals(
2032 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002033 actual=newLeaderResult,
2034 onpass="Leadership election passed",
2035 onfail="Something went wrong with Leadership election" )
Jon Hall85794ff2015-07-08 14:12:30 -07002036
acsmars71adceb2015-08-31 15:09:26 -07002037 main.step( "Check that that new leader was the candidate of old leader")
2038 # candidates[ 2 ] should be come the top candidate after withdrawl
2039 correctCandidateResult = main.TRUE
2040 if expectNoLeader:
2041 if newLeader == 'none':
2042 main.log.info( "No leader expected. None found. Pass" )
2043 correctCandidateResult = main.TRUE
2044 else:
2045 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2046 correctCandidateResult = main.FALSE
2047 elif newLeader != oldCandidates[ 2 ]:
2048 correctCandidateResult = main.FALSE
2049 main.log.error( "Candidate " + newLeader + " was elected. " +
2050 oldCandidates[ 2 ] + " should have had priority." )
2051
2052 utilities.assert_equals(
2053 expect=main.TRUE,
2054 actual=correctCandidateResult,
2055 onpass="Correct Candidate Elected",
2056 onfail="Incorrect Candidate Elected" )
2057
Jon Hall85794ff2015-07-08 14:12:30 -07002058 main.step( "Run for election on old leader( just so everyone " +
2059 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002060 if oldLeaderCLI is not None:
2061 runResult = oldLeaderCLI.electionTestRun()
Jon Hall85794ff2015-07-08 14:12:30 -07002062 else:
acsmars71adceb2015-08-31 15:09:26 -07002063 main.log.error( "No old leader to re-elect" )
Jon Hall85794ff2015-07-08 14:12:30 -07002064 runResult = main.FALSE
2065 utilities.assert_equals(
2066 expect=main.TRUE,
2067 actual=runResult,
2068 onpass="App re-ran for election",
2069 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002070 main.step(
2071 "Check that oldLeader is a candidate, and leader if only 1 node" )
2072 # verify leader didn't just change
2073 positionResult = main.TRUE
2074 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
Jon Hall85794ff2015-07-08 14:12:30 -07002075
acsmars71adceb2015-08-31 15:09:26 -07002076 # Reset and reuse the new candidate and leaders lists
2077 newAllCandidates = []
2078 newCandidates = []
2079 newLeaders = []
2080 for cli in main.CLIs:
2081 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2082 if oldLeader not in node: # election might no have finished yet
2083 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2084 "be sure elections are complete" )
2085 time.sleep(5)
2086 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2087 if oldLeader not in node: # election still isn't done, errors
2088 main.log.error(
2089 "Old leader was not elected on at least one node" )
2090 positionResult = main.FALSE
2091 newAllCandidates.append( node )
2092 newLeaders.append( node[ 0 ] )
2093 newCandidates = newAllCandidates[ 0 ]
2094
2095 # Check that each node has the same leader. Defines newLeader
2096 if len( set( newLeaders ) ) != 1:
2097 positionResult = main.FALSE
2098 main.log.error( "Nodes have different leaders: " +
2099 str( newLeaders ) )
2100 newLeader = None
Jon Hall85794ff2015-07-08 14:12:30 -07002101 else:
acsmars71adceb2015-08-31 15:09:26 -07002102 newLeader = newLeaders[ 0 ]
2103
2104 # Check that each node's candidate list is the same
2105 for candidates in newAllCandidates:
2106 if set( candidates ) != set( newCandidates ):
2107 newLeaderResult = main.FALSE
2108 main.error.log( "Discrepancy in candidate lists detected" )
2109
2110 # Check that the re-elected node is last on the candidate List
2111 if oldLeader != newCandidates[ -1 ]:
2112 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
2113 str( newCandidates ) )
2114 positionResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07002115
2116 utilities.assert_equals(
2117 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002118 actual=positionResult,
Jon Hall85794ff2015-07-08 14:12:30 -07002119 onpass="Old leader successfully re-ran for election",
2120 onfail="Something went wrong with Leadership election after " +
2121 "the old leader re-ran for election" )
2122
2123 def CASE16( self, main ):
2124 """
2125 Install Distributed Primitives app
2126 """
Jon Halle1a3b752015-07-22 13:02:46 -07002127 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002128 assert main, "main not defined"
2129 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002130 assert main.CLIs, "main.CLIs not defined"
2131 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002132
2133 # Variables for the distributed primitives tests
2134 global pCounterName
2135 global iCounterName
2136 global pCounterValue
2137 global iCounterValue
2138 global onosSet
2139 global onosSetName
2140 pCounterName = "TestON-Partitions"
2141 iCounterName = "TestON-inMemory"
2142 pCounterValue = 0
2143 iCounterValue = 0
2144 onosSet = set([])
2145 onosSetName = "TestON-set"
2146
2147 description = "Install Primitives app"
2148 main.case( description )
2149 main.step( "Install Primitives app" )
2150 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002151 appResults = main.CLIs[0].activateApp( appName )
Jon Hall85794ff2015-07-08 14:12:30 -07002152 utilities.assert_equals( expect=main.TRUE,
2153 actual=appResults,
2154 onpass="Primitives app activated",
2155 onfail="Primitives app not activated" )
2156
2157 def CASE17( self, main ):
2158 """
2159 Check for basic functionality with distributed primitives
2160 """
Jon Hall85794ff2015-07-08 14:12:30 -07002161 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002162 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002163 assert main, "main not defined"
2164 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002165 assert main.CLIs, "main.CLIs not defined"
2166 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002167 assert pCounterName, "pCounterName not defined"
2168 assert iCounterName, "iCounterName not defined"
2169 assert onosSetName, "onosSetName not defined"
2170 # NOTE: assert fails if value is 0/None/Empty/False
2171 try:
2172 pCounterValue
2173 except NameError:
2174 main.log.error( "pCounterValue not defined, setting to 0" )
2175 pCounterValue = 0
2176 try:
2177 iCounterValue
2178 except NameError:
2179 main.log.error( "iCounterValue not defined, setting to 0" )
2180 iCounterValue = 0
2181 try:
2182 onosSet
2183 except NameError:
2184 main.log.error( "onosSet not defined, setting to empty Set" )
2185 onosSet = set([])
2186 # Variables for the distributed primitives tests. These are local only
2187 addValue = "a"
2188 addAllValue = "a b c d e f"
2189 retainValue = "c d e f"
2190
2191 description = "Check for basic functionality with distributed " +\
2192 "primitives"
2193 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07002194 main.caseExplanation = "Test the methods of the distributed " +\
2195 "primitives (counters and sets) throught the cli"
Jon Hall85794ff2015-07-08 14:12:30 -07002196 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07002197 # Partitioned counters
2198 main.step( "Increment then get a default counter on each node" )
Jon Hall85794ff2015-07-08 14:12:30 -07002199 pCounters = []
2200 threads = []
2201 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07002202 for i in range( main.numCtrls ):
2203 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2204 name="counterAddAndGet-" + str( i ),
Jon Hall85794ff2015-07-08 14:12:30 -07002205 args=[ pCounterName ] )
2206 pCounterValue += 1
2207 addedPValues.append( pCounterValue )
2208 threads.append( t )
2209 t.start()
2210
2211 for t in threads:
2212 t.join()
2213 pCounters.append( t.result )
2214 # Check that counter incremented numController times
2215 pCounterResults = True
2216 for i in addedPValues:
2217 tmpResult = i in pCounters
2218 pCounterResults = pCounterResults and tmpResult
2219 if not tmpResult:
2220 main.log.error( str( i ) + " is not in partitioned "
2221 "counter incremented results" )
2222 utilities.assert_equals( expect=True,
2223 actual=pCounterResults,
2224 onpass="Default counter incremented",
2225 onfail="Error incrementing default" +
2226 " counter" )
2227
Jon Halle1a3b752015-07-22 13:02:46 -07002228 main.step( "Get then Increment a default counter on each node" )
2229 pCounters = []
2230 threads = []
2231 addedPValues = []
2232 for i in range( main.numCtrls ):
2233 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2234 name="counterGetAndAdd-" + str( i ),
2235 args=[ pCounterName ] )
2236 addedPValues.append( pCounterValue )
2237 pCounterValue += 1
2238 threads.append( t )
2239 t.start()
2240
2241 for t in threads:
2242 t.join()
2243 pCounters.append( t.result )
2244 # Check that counter incremented numController times
2245 pCounterResults = True
2246 for i in addedPValues:
2247 tmpResult = i in pCounters
2248 pCounterResults = pCounterResults and tmpResult
2249 if not tmpResult:
2250 main.log.error( str( i ) + " is not in partitioned "
2251 "counter incremented results" )
2252 utilities.assert_equals( expect=True,
2253 actual=pCounterResults,
2254 onpass="Default counter incremented",
2255 onfail="Error incrementing default" +
2256 " counter" )
2257
2258 main.step( "Counters we added have the correct values" )
2259 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2260 utilities.assert_equals( expect=main.TRUE,
2261 actual=incrementCheck,
2262 onpass="Added counters are correct",
2263 onfail="Added counters are incorrect" )
2264
2265 main.step( "Add -8 to then get a default counter on each node" )
2266 pCounters = []
2267 threads = []
2268 addedPValues = []
2269 for i in range( main.numCtrls ):
2270 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2271 name="counterIncrement-" + str( i ),
2272 args=[ pCounterName ],
2273 kwargs={ "delta": -8 } )
2274 pCounterValue += -8
2275 addedPValues.append( pCounterValue )
2276 threads.append( t )
2277 t.start()
2278
2279 for t in threads:
2280 t.join()
2281 pCounters.append( t.result )
2282 # Check that counter incremented numController times
2283 pCounterResults = True
2284 for i in addedPValues:
2285 tmpResult = i in pCounters
2286 pCounterResults = pCounterResults and tmpResult
2287 if not tmpResult:
2288 main.log.error( str( i ) + " is not in partitioned "
2289 "counter incremented results" )
2290 utilities.assert_equals( expect=True,
2291 actual=pCounterResults,
2292 onpass="Default counter incremented",
2293 onfail="Error incrementing default" +
2294 " counter" )
2295
2296 main.step( "Add 5 to then get a default counter on each node" )
2297 pCounters = []
2298 threads = []
2299 addedPValues = []
2300 for i in range( main.numCtrls ):
2301 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2302 name="counterIncrement-" + str( i ),
2303 args=[ pCounterName ],
2304 kwargs={ "delta": 5 } )
2305 pCounterValue += 5
2306 addedPValues.append( pCounterValue )
2307 threads.append( t )
2308 t.start()
2309
2310 for t in threads:
2311 t.join()
2312 pCounters.append( t.result )
2313 # Check that counter incremented numController times
2314 pCounterResults = True
2315 for i in addedPValues:
2316 tmpResult = i in pCounters
2317 pCounterResults = pCounterResults and tmpResult
2318 if not tmpResult:
2319 main.log.error( str( i ) + " is not in partitioned "
2320 "counter incremented results" )
2321 utilities.assert_equals( expect=True,
2322 actual=pCounterResults,
2323 onpass="Default counter incremented",
2324 onfail="Error incrementing default" +
2325 " counter" )
2326
2327 main.step( "Get then add 5 to a default counter on each node" )
2328 pCounters = []
2329 threads = []
2330 addedPValues = []
2331 for i in range( main.numCtrls ):
2332 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2333 name="counterIncrement-" + str( i ),
2334 args=[ pCounterName ],
2335 kwargs={ "delta": 5 } )
2336 addedPValues.append( pCounterValue )
2337 pCounterValue += 5
2338 threads.append( t )
2339 t.start()
2340
2341 for t in threads:
2342 t.join()
2343 pCounters.append( t.result )
2344 # Check that counter incremented numController times
2345 pCounterResults = True
2346 for i in addedPValues:
2347 tmpResult = i in pCounters
2348 pCounterResults = pCounterResults and tmpResult
2349 if not tmpResult:
2350 main.log.error( str( i ) + " is not in partitioned "
2351 "counter incremented results" )
2352 utilities.assert_equals( expect=True,
2353 actual=pCounterResults,
2354 onpass="Default counter incremented",
2355 onfail="Error incrementing default" +
2356 " counter" )
2357
2358 main.step( "Counters we added have the correct values" )
2359 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2360 utilities.assert_equals( expect=main.TRUE,
2361 actual=incrementCheck,
2362 onpass="Added counters are correct",
2363 onfail="Added counters are incorrect" )
2364
2365 # In-Memory counters
2366 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall85794ff2015-07-08 14:12:30 -07002367 iCounters = []
2368 addedIValues = []
2369 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002370 for i in range( main.numCtrls ):
2371 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002372 name="icounterIncrement-" + str( i ),
2373 args=[ iCounterName ],
2374 kwargs={ "inMemory": True } )
2375 iCounterValue += 1
2376 addedIValues.append( iCounterValue )
2377 threads.append( t )
2378 t.start()
2379
2380 for t in threads:
2381 t.join()
2382 iCounters.append( t.result )
2383 # Check that counter incremented numController times
2384 iCounterResults = True
2385 for i in addedIValues:
2386 tmpResult = i in iCounters
2387 iCounterResults = iCounterResults and tmpResult
2388 if not tmpResult:
2389 main.log.error( str( i ) + " is not in the in-memory "
2390 "counter incremented results" )
2391 utilities.assert_equals( expect=True,
2392 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07002393 onpass="In-memory counter incremented",
2394 onfail="Error incrementing in-memory" +
Jon Hall85794ff2015-07-08 14:12:30 -07002395 " counter" )
2396
Jon Halle1a3b752015-07-22 13:02:46 -07002397 main.step( "Get then Increment a in-memory counter on each node" )
2398 iCounters = []
2399 threads = []
2400 addedIValues = []
2401 for i in range( main.numCtrls ):
2402 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2403 name="counterGetAndAdd-" + str( i ),
2404 args=[ iCounterName ],
2405 kwargs={ "inMemory": True } )
2406 addedIValues.append( iCounterValue )
2407 iCounterValue += 1
2408 threads.append( t )
2409 t.start()
2410
2411 for t in threads:
2412 t.join()
2413 iCounters.append( t.result )
2414 # Check that counter incremented numController times
2415 iCounterResults = True
2416 for i in addedIValues:
2417 tmpResult = i in iCounters
2418 iCounterResults = iCounterResults and tmpResult
2419 if not tmpResult:
2420 main.log.error( str( i ) + " is not in in-memory "
2421 "counter incremented results" )
2422 utilities.assert_equals( expect=True,
2423 actual=iCounterResults,
2424 onpass="In-memory counter incremented",
2425 onfail="Error incrementing in-memory" +
2426 " counter" )
2427
2428 main.step( "Counters we added have the correct values" )
2429 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2430 utilities.assert_equals( expect=main.TRUE,
2431 actual=incrementCheck,
2432 onpass="Added counters are correct",
2433 onfail="Added counters are incorrect" )
2434
2435 main.step( "Add -8 to then get a in-memory counter on each node" )
2436 iCounters = []
2437 threads = []
2438 addedIValues = []
2439 for i in range( main.numCtrls ):
2440 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2441 name="counterIncrement-" + str( i ),
2442 args=[ iCounterName ],
2443 kwargs={ "delta": -8, "inMemory": True } )
2444 iCounterValue += -8
2445 addedIValues.append( iCounterValue )
2446 threads.append( t )
2447 t.start()
2448
2449 for t in threads:
2450 t.join()
2451 iCounters.append( t.result )
2452 # Check that counter incremented numController times
2453 iCounterResults = True
2454 for i in addedIValues:
2455 tmpResult = i in iCounters
2456 iCounterResults = iCounterResults and tmpResult
2457 if not tmpResult:
2458 main.log.error( str( i ) + " is not in in-memory "
2459 "counter incremented results" )
2460 utilities.assert_equals( expect=True,
2461 actual=pCounterResults,
2462 onpass="In-memory counter incremented",
2463 onfail="Error incrementing in-memory" +
2464 " counter" )
2465
2466 main.step( "Add 5 to then get a in-memory counter on each node" )
2467 iCounters = []
2468 threads = []
2469 addedIValues = []
2470 for i in range( main.numCtrls ):
2471 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2472 name="counterIncrement-" + str( i ),
2473 args=[ iCounterName ],
2474 kwargs={ "delta": 5, "inMemory": True } )
2475 iCounterValue += 5
2476 addedIValues.append( iCounterValue )
2477 threads.append( t )
2478 t.start()
2479
2480 for t in threads:
2481 t.join()
2482 iCounters.append( t.result )
2483 # Check that counter incremented numController times
2484 iCounterResults = True
2485 for i in addedIValues:
2486 tmpResult = i in iCounters
2487 iCounterResults = iCounterResults and tmpResult
2488 if not tmpResult:
2489 main.log.error( str( i ) + " is not in in-memory "
2490 "counter incremented results" )
2491 utilities.assert_equals( expect=True,
2492 actual=pCounterResults,
2493 onpass="In-memory counter incremented",
2494 onfail="Error incrementing in-memory" +
2495 " counter" )
2496
2497 main.step( "Get then add 5 to a in-memory counter on each node" )
2498 iCounters = []
2499 threads = []
2500 addedIValues = []
2501 for i in range( main.numCtrls ):
2502 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2503 name="counterIncrement-" + str( i ),
2504 args=[ iCounterName ],
2505 kwargs={ "delta": 5, "inMemory": True } )
2506 addedIValues.append( iCounterValue )
2507 iCounterValue += 5
2508 threads.append( t )
2509 t.start()
2510
2511 for t in threads:
2512 t.join()
2513 iCounters.append( t.result )
2514 # Check that counter incremented numController times
2515 iCounterResults = True
2516 for i in addedIValues:
2517 tmpResult = i in iCounters
2518 iCounterResults = iCounterResults and tmpResult
2519 if not tmpResult:
2520 main.log.error( str( i ) + " is not in in-memory "
2521 "counter incremented results" )
2522 utilities.assert_equals( expect=True,
2523 actual=iCounterResults,
2524 onpass="In-memory counter incremented",
2525 onfail="Error incrementing in-memory" +
2526 " counter" )
2527
2528 main.step( "Counters we added have the correct values" )
2529 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2530 utilities.assert_equals( expect=main.TRUE,
2531 actual=incrementCheck,
2532 onpass="Added counters are correct",
2533 onfail="Added counters are incorrect" )
2534
Jon Hall85794ff2015-07-08 14:12:30 -07002535 main.step( "Check counters are consistant across nodes" )
2536 onosCounters = []
2537 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002538 for i in range( main.numCtrls ):
2539 t = main.Thread( target=main.CLIs[i].counters,
Jon Hall85794ff2015-07-08 14:12:30 -07002540 name="counters-" + str( i ) )
2541 threads.append( t )
2542 t.start()
2543 for t in threads:
2544 t.join()
2545 onosCounters.append( t.result )
2546 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
2547 if all( tmp ):
2548 main.log.info( "Counters are consistent across all nodes" )
2549 consistentCounterResults = main.TRUE
2550 else:
2551 main.log.error( "Counters are not consistent across all nodes" )
2552 consistentCounterResults = main.FALSE
2553 utilities.assert_equals( expect=main.TRUE,
2554 actual=consistentCounterResults,
2555 onpass="ONOS counters are consistent " +
2556 "across nodes",
2557 onfail="ONOS Counters are inconsistent " +
2558 "across nodes" )
2559
2560 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07002561 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2562 incrementCheck = incrementCheck and \
2563 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall85794ff2015-07-08 14:12:30 -07002564 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07002565 actual=incrementCheck,
Jon Hall85794ff2015-07-08 14:12:30 -07002566 onpass="Added counters are correct",
2567 onfail="Added counters are incorrect" )
Jon Halle1a3b752015-07-22 13:02:46 -07002568
Jon Hall85794ff2015-07-08 14:12:30 -07002569 # DISTRIBUTED SETS
2570 main.step( "Distributed Set get" )
2571 size = len( onosSet )
2572 getResponses = []
2573 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002574 for i in range( main.numCtrls ):
2575 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002576 name="setTestGet-" + str( i ),
2577 args=[ onosSetName ] )
2578 threads.append( t )
2579 t.start()
2580 for t in threads:
2581 t.join()
2582 getResponses.append( t.result )
2583
2584 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002585 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002586 if isinstance( getResponses[ i ], list):
2587 current = set( getResponses[ i ] )
2588 if len( current ) == len( getResponses[ i ] ):
2589 # no repeats
2590 if onosSet != current:
2591 main.log.error( "ONOS" + str( i + 1 ) +
2592 " has incorrect view" +
2593 " of set " + onosSetName + ":\n" +
2594 str( getResponses[ i ] ) )
2595 main.log.debug( "Expected: " + str( onosSet ) )
2596 main.log.debug( "Actual: " + str( current ) )
2597 getResults = main.FALSE
2598 else:
2599 # error, set is not a set
2600 main.log.error( "ONOS" + str( i + 1 ) +
2601 " has repeat elements in" +
2602 " set " + onosSetName + ":\n" +
2603 str( getResponses[ i ] ) )
2604 getResults = main.FALSE
2605 elif getResponses[ i ] == main.ERROR:
2606 getResults = main.FALSE
2607 utilities.assert_equals( expect=main.TRUE,
2608 actual=getResults,
2609 onpass="Set elements are correct",
2610 onfail="Set elements are incorrect" )
2611
2612 main.step( "Distributed Set size" )
2613 sizeResponses = []
2614 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002615 for i in range( main.numCtrls ):
2616 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002617 name="setTestSize-" + str( i ),
2618 args=[ onosSetName ] )
2619 threads.append( t )
2620 t.start()
2621 for t in threads:
2622 t.join()
2623 sizeResponses.append( t.result )
2624
2625 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002626 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002627 if size != sizeResponses[ i ]:
2628 sizeResults = main.FALSE
2629 main.log.error( "ONOS" + str( i + 1 ) +
2630 " expected a size of " + str( size ) +
2631 " for set " + onosSetName +
2632 " but got " + str( sizeResponses[ i ] ) )
2633 utilities.assert_equals( expect=main.TRUE,
2634 actual=sizeResults,
2635 onpass="Set sizes are correct",
2636 onfail="Set sizes are incorrect" )
2637
2638 main.step( "Distributed Set add()" )
2639 onosSet.add( addValue )
2640 addResponses = []
2641 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002642 for i in range( main.numCtrls ):
2643 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002644 name="setTestAdd-" + str( i ),
2645 args=[ onosSetName, addValue ] )
2646 threads.append( t )
2647 t.start()
2648 for t in threads:
2649 t.join()
2650 addResponses.append( t.result )
2651
2652 # main.TRUE = successfully changed the set
2653 # main.FALSE = action resulted in no change in set
2654 # main.ERROR - Some error in executing the function
2655 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002656 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002657 if addResponses[ i ] == main.TRUE:
2658 # All is well
2659 pass
2660 elif addResponses[ i ] == main.FALSE:
2661 # Already in set, probably fine
2662 pass
2663 elif addResponses[ i ] == main.ERROR:
2664 # Error in execution
2665 addResults = main.FALSE
2666 else:
2667 # unexpected result
2668 addResults = main.FALSE
2669 if addResults != main.TRUE:
2670 main.log.error( "Error executing set add" )
2671
2672 # Check if set is still correct
2673 size = len( onosSet )
2674 getResponses = []
2675 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002676 for i in range( main.numCtrls ):
2677 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002678 name="setTestGet-" + str( i ),
2679 args=[ onosSetName ] )
2680 threads.append( t )
2681 t.start()
2682 for t in threads:
2683 t.join()
2684 getResponses.append( t.result )
2685 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002686 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002687 if isinstance( getResponses[ i ], list):
2688 current = set( getResponses[ i ] )
2689 if len( current ) == len( getResponses[ i ] ):
2690 # no repeats
2691 if onosSet != current:
2692 main.log.error( "ONOS" + str( i + 1 ) +
2693 " has incorrect view" +
2694 " of set " + onosSetName + ":\n" +
2695 str( getResponses[ i ] ) )
2696 main.log.debug( "Expected: " + str( onosSet ) )
2697 main.log.debug( "Actual: " + str( current ) )
2698 getResults = main.FALSE
2699 else:
2700 # error, set is not a set
2701 main.log.error( "ONOS" + str( i + 1 ) +
2702 " has repeat elements in" +
2703 " set " + onosSetName + ":\n" +
2704 str( getResponses[ i ] ) )
2705 getResults = main.FALSE
2706 elif getResponses[ i ] == main.ERROR:
2707 getResults = main.FALSE
2708 sizeResponses = []
2709 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002710 for i in range( main.numCtrls ):
2711 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002712 name="setTestSize-" + str( i ),
2713 args=[ onosSetName ] )
2714 threads.append( t )
2715 t.start()
2716 for t in threads:
2717 t.join()
2718 sizeResponses.append( t.result )
2719 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002720 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002721 if size != sizeResponses[ i ]:
2722 sizeResults = main.FALSE
2723 main.log.error( "ONOS" + str( i + 1 ) +
2724 " expected a size of " + str( size ) +
2725 " for set " + onosSetName +
2726 " but got " + str( sizeResponses[ i ] ) )
2727 addResults = addResults and getResults and sizeResults
2728 utilities.assert_equals( expect=main.TRUE,
2729 actual=addResults,
2730 onpass="Set add correct",
2731 onfail="Set add was incorrect" )
2732
2733 main.step( "Distributed Set addAll()" )
2734 onosSet.update( addAllValue.split() )
2735 addResponses = []
2736 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002737 for i in range( main.numCtrls ):
2738 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002739 name="setTestAddAll-" + str( i ),
2740 args=[ onosSetName, addAllValue ] )
2741 threads.append( t )
2742 t.start()
2743 for t in threads:
2744 t.join()
2745 addResponses.append( t.result )
2746
2747 # main.TRUE = successfully changed the set
2748 # main.FALSE = action resulted in no change in set
2749 # main.ERROR - Some error in executing the function
2750 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002751 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002752 if addResponses[ i ] == main.TRUE:
2753 # All is well
2754 pass
2755 elif addResponses[ i ] == main.FALSE:
2756 # Already in set, probably fine
2757 pass
2758 elif addResponses[ i ] == main.ERROR:
2759 # Error in execution
2760 addAllResults = main.FALSE
2761 else:
2762 # unexpected result
2763 addAllResults = main.FALSE
2764 if addAllResults != main.TRUE:
2765 main.log.error( "Error executing set addAll" )
2766
2767 # Check if set is still correct
2768 size = len( onosSet )
2769 getResponses = []
2770 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002771 for i in range( main.numCtrls ):
2772 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002773 name="setTestGet-" + str( i ),
2774 args=[ onosSetName ] )
2775 threads.append( t )
2776 t.start()
2777 for t in threads:
2778 t.join()
2779 getResponses.append( t.result )
2780 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002781 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002782 if isinstance( getResponses[ i ], list):
2783 current = set( getResponses[ i ] )
2784 if len( current ) == len( getResponses[ i ] ):
2785 # no repeats
2786 if onosSet != current:
2787 main.log.error( "ONOS" + str( i + 1 ) +
2788 " has incorrect view" +
2789 " of set " + onosSetName + ":\n" +
2790 str( getResponses[ i ] ) )
2791 main.log.debug( "Expected: " + str( onosSet ) )
2792 main.log.debug( "Actual: " + str( current ) )
2793 getResults = main.FALSE
2794 else:
2795 # error, set is not a set
2796 main.log.error( "ONOS" + str( i + 1 ) +
2797 " has repeat elements in" +
2798 " set " + onosSetName + ":\n" +
2799 str( getResponses[ i ] ) )
2800 getResults = main.FALSE
2801 elif getResponses[ i ] == main.ERROR:
2802 getResults = main.FALSE
2803 sizeResponses = []
2804 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002805 for i in range( main.numCtrls ):
2806 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002807 name="setTestSize-" + str( i ),
2808 args=[ onosSetName ] )
2809 threads.append( t )
2810 t.start()
2811 for t in threads:
2812 t.join()
2813 sizeResponses.append( t.result )
2814 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002815 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002816 if size != sizeResponses[ i ]:
2817 sizeResults = main.FALSE
2818 main.log.error( "ONOS" + str( i + 1 ) +
2819 " expected a size of " + str( size ) +
2820 " for set " + onosSetName +
2821 " but got " + str( sizeResponses[ i ] ) )
2822 addAllResults = addAllResults and getResults and sizeResults
2823 utilities.assert_equals( expect=main.TRUE,
2824 actual=addAllResults,
2825 onpass="Set addAll correct",
2826 onfail="Set addAll was incorrect" )
2827
2828 main.step( "Distributed Set contains()" )
2829 containsResponses = []
2830 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002831 for i in range( main.numCtrls ):
2832 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002833 name="setContains-" + str( i ),
2834 args=[ onosSetName ],
2835 kwargs={ "values": addValue } )
2836 threads.append( t )
2837 t.start()
2838 for t in threads:
2839 t.join()
2840 # NOTE: This is the tuple
2841 containsResponses.append( t.result )
2842
2843 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002844 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002845 if containsResponses[ i ] == main.ERROR:
2846 containsResults = main.FALSE
2847 else:
2848 containsResults = containsResults and\
2849 containsResponses[ i ][ 1 ]
2850 utilities.assert_equals( expect=main.TRUE,
2851 actual=containsResults,
2852 onpass="Set contains is functional",
2853 onfail="Set contains failed" )
2854
2855 main.step( "Distributed Set containsAll()" )
2856 containsAllResponses = []
2857 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002858 for i in range( main.numCtrls ):
2859 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002860 name="setContainsAll-" + str( i ),
2861 args=[ onosSetName ],
2862 kwargs={ "values": addAllValue } )
2863 threads.append( t )
2864 t.start()
2865 for t in threads:
2866 t.join()
2867 # NOTE: This is the tuple
2868 containsAllResponses.append( t.result )
2869
2870 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002871 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002872 if containsResponses[ i ] == main.ERROR:
2873 containsResults = main.FALSE
2874 else:
2875 containsResults = containsResults and\
2876 containsResponses[ i ][ 1 ]
2877 utilities.assert_equals( expect=main.TRUE,
2878 actual=containsAllResults,
2879 onpass="Set containsAll is functional",
2880 onfail="Set containsAll failed" )
2881
2882 main.step( "Distributed Set remove()" )
2883 onosSet.remove( addValue )
2884 removeResponses = []
2885 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002886 for i in range( main.numCtrls ):
2887 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002888 name="setTestRemove-" + str( i ),
2889 args=[ onosSetName, addValue ] )
2890 threads.append( t )
2891 t.start()
2892 for t in threads:
2893 t.join()
2894 removeResponses.append( t.result )
2895
2896 # main.TRUE = successfully changed the set
2897 # main.FALSE = action resulted in no change in set
2898 # main.ERROR - Some error in executing the function
2899 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002900 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002901 if removeResponses[ i ] == main.TRUE:
2902 # All is well
2903 pass
2904 elif removeResponses[ i ] == main.FALSE:
2905 # not in set, probably fine
2906 pass
2907 elif removeResponses[ i ] == main.ERROR:
2908 # Error in execution
2909 removeResults = main.FALSE
2910 else:
2911 # unexpected result
2912 removeResults = main.FALSE
2913 if removeResults != main.TRUE:
2914 main.log.error( "Error executing set remove" )
2915
2916 # Check if set is still correct
2917 size = len( onosSet )
2918 getResponses = []
2919 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002920 for i in range( main.numCtrls ):
2921 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002922 name="setTestGet-" + str( i ),
2923 args=[ onosSetName ] )
2924 threads.append( t )
2925 t.start()
2926 for t in threads:
2927 t.join()
2928 getResponses.append( t.result )
2929 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002930 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002931 if isinstance( getResponses[ i ], list):
2932 current = set( getResponses[ i ] )
2933 if len( current ) == len( getResponses[ i ] ):
2934 # no repeats
2935 if onosSet != current:
2936 main.log.error( "ONOS" + str( i + 1 ) +
2937 " has incorrect view" +
2938 " of set " + onosSetName + ":\n" +
2939 str( getResponses[ i ] ) )
2940 main.log.debug( "Expected: " + str( onosSet ) )
2941 main.log.debug( "Actual: " + str( current ) )
2942 getResults = main.FALSE
2943 else:
2944 # error, set is not a set
2945 main.log.error( "ONOS" + str( i + 1 ) +
2946 " has repeat elements in" +
2947 " set " + onosSetName + ":\n" +
2948 str( getResponses[ i ] ) )
2949 getResults = main.FALSE
2950 elif getResponses[ i ] == main.ERROR:
2951 getResults = main.FALSE
2952 sizeResponses = []
2953 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002954 for i in range( main.numCtrls ):
2955 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002956 name="setTestSize-" + str( i ),
2957 args=[ onosSetName ] )
2958 threads.append( t )
2959 t.start()
2960 for t in threads:
2961 t.join()
2962 sizeResponses.append( t.result )
2963 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002964 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002965 if size != sizeResponses[ i ]:
2966 sizeResults = main.FALSE
2967 main.log.error( "ONOS" + str( i + 1 ) +
2968 " expected a size of " + str( size ) +
2969 " for set " + onosSetName +
2970 " but got " + str( sizeResponses[ i ] ) )
2971 removeResults = removeResults and getResults and sizeResults
2972 utilities.assert_equals( expect=main.TRUE,
2973 actual=removeResults,
2974 onpass="Set remove correct",
2975 onfail="Set remove was incorrect" )
2976
2977 main.step( "Distributed Set removeAll()" )
2978 onosSet.difference_update( addAllValue.split() )
2979 removeAllResponses = []
2980 threads = []
2981 try:
Jon Halle1a3b752015-07-22 13:02:46 -07002982 for i in range( main.numCtrls ):
2983 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002984 name="setTestRemoveAll-" + str( i ),
2985 args=[ onosSetName, addAllValue ] )
2986 threads.append( t )
2987 t.start()
2988 for t in threads:
2989 t.join()
2990 removeAllResponses.append( t.result )
2991 except Exception, e:
2992 main.log.exception(e)
2993
2994 # main.TRUE = successfully changed the set
2995 # main.FALSE = action resulted in no change in set
2996 # main.ERROR - Some error in executing the function
2997 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002998 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002999 if removeAllResponses[ i ] == main.TRUE:
3000 # All is well
3001 pass
3002 elif removeAllResponses[ i ] == main.FALSE:
3003 # not in set, probably fine
3004 pass
3005 elif removeAllResponses[ i ] == main.ERROR:
3006 # Error in execution
3007 removeAllResults = main.FALSE
3008 else:
3009 # unexpected result
3010 removeAllResults = main.FALSE
3011 if removeAllResults != main.TRUE:
3012 main.log.error( "Error executing set removeAll" )
3013
3014 # Check if set is still correct
3015 size = len( onosSet )
3016 getResponses = []
3017 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003018 for i in range( main.numCtrls ):
3019 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003020 name="setTestGet-" + str( i ),
3021 args=[ onosSetName ] )
3022 threads.append( t )
3023 t.start()
3024 for t in threads:
3025 t.join()
3026 getResponses.append( t.result )
3027 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003028 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003029 if isinstance( getResponses[ i ], list):
3030 current = set( getResponses[ i ] )
3031 if len( current ) == len( getResponses[ i ] ):
3032 # no repeats
3033 if onosSet != current:
3034 main.log.error( "ONOS" + str( i + 1 ) +
3035 " has incorrect view" +
3036 " of set " + onosSetName + ":\n" +
3037 str( getResponses[ i ] ) )
3038 main.log.debug( "Expected: " + str( onosSet ) )
3039 main.log.debug( "Actual: " + str( current ) )
3040 getResults = main.FALSE
3041 else:
3042 # error, set is not a set
3043 main.log.error( "ONOS" + str( i + 1 ) +
3044 " has repeat elements in" +
3045 " set " + onosSetName + ":\n" +
3046 str( getResponses[ i ] ) )
3047 getResults = main.FALSE
3048 elif getResponses[ i ] == main.ERROR:
3049 getResults = main.FALSE
3050 sizeResponses = []
3051 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003052 for i in range( main.numCtrls ):
3053 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003054 name="setTestSize-" + str( i ),
3055 args=[ onosSetName ] )
3056 threads.append( t )
3057 t.start()
3058 for t in threads:
3059 t.join()
3060 sizeResponses.append( t.result )
3061 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003062 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003063 if size != sizeResponses[ i ]:
3064 sizeResults = main.FALSE
3065 main.log.error( "ONOS" + str( i + 1 ) +
3066 " expected a size of " + str( size ) +
3067 " for set " + onosSetName +
3068 " but got " + str( sizeResponses[ i ] ) )
3069 removeAllResults = removeAllResults and getResults and sizeResults
3070 utilities.assert_equals( expect=main.TRUE,
3071 actual=removeAllResults,
3072 onpass="Set removeAll correct",
3073 onfail="Set removeAll was incorrect" )
3074
3075 main.step( "Distributed Set addAll()" )
3076 onosSet.update( addAllValue.split() )
3077 addResponses = []
3078 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003079 for i in range( main.numCtrls ):
3080 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003081 name="setTestAddAll-" + str( i ),
3082 args=[ onosSetName, addAllValue ] )
3083 threads.append( t )
3084 t.start()
3085 for t in threads:
3086 t.join()
3087 addResponses.append( t.result )
3088
3089 # main.TRUE = successfully changed the set
3090 # main.FALSE = action resulted in no change in set
3091 # main.ERROR - Some error in executing the function
3092 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003093 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003094 if addResponses[ i ] == main.TRUE:
3095 # All is well
3096 pass
3097 elif addResponses[ i ] == main.FALSE:
3098 # Already in set, probably fine
3099 pass
3100 elif addResponses[ i ] == main.ERROR:
3101 # Error in execution
3102 addAllResults = main.FALSE
3103 else:
3104 # unexpected result
3105 addAllResults = main.FALSE
3106 if addAllResults != main.TRUE:
3107 main.log.error( "Error executing set addAll" )
3108
3109 # Check if set is still correct
3110 size = len( onosSet )
3111 getResponses = []
3112 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003113 for i in range( main.numCtrls ):
3114 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003115 name="setTestGet-" + str( i ),
3116 args=[ onosSetName ] )
3117 threads.append( t )
3118 t.start()
3119 for t in threads:
3120 t.join()
3121 getResponses.append( t.result )
3122 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003123 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003124 if isinstance( getResponses[ i ], list):
3125 current = set( getResponses[ i ] )
3126 if len( current ) == len( getResponses[ i ] ):
3127 # no repeats
3128 if onosSet != current:
3129 main.log.error( "ONOS" + str( i + 1 ) +
3130 " has incorrect view" +
3131 " of set " + onosSetName + ":\n" +
3132 str( getResponses[ i ] ) )
3133 main.log.debug( "Expected: " + str( onosSet ) )
3134 main.log.debug( "Actual: " + str( current ) )
3135 getResults = main.FALSE
3136 else:
3137 # error, set is not a set
3138 main.log.error( "ONOS" + str( i + 1 ) +
3139 " has repeat elements in" +
3140 " set " + onosSetName + ":\n" +
3141 str( getResponses[ i ] ) )
3142 getResults = main.FALSE
3143 elif getResponses[ i ] == main.ERROR:
3144 getResults = main.FALSE
3145 sizeResponses = []
3146 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003147 for i in range( main.numCtrls ):
3148 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003149 name="setTestSize-" + str( i ),
3150 args=[ onosSetName ] )
3151 threads.append( t )
3152 t.start()
3153 for t in threads:
3154 t.join()
3155 sizeResponses.append( t.result )
3156 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003157 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003158 if size != sizeResponses[ i ]:
3159 sizeResults = main.FALSE
3160 main.log.error( "ONOS" + str( i + 1 ) +
3161 " expected a size of " + str( size ) +
3162 " for set " + onosSetName +
3163 " but got " + str( sizeResponses[ i ] ) )
3164 addAllResults = addAllResults and getResults and sizeResults
3165 utilities.assert_equals( expect=main.TRUE,
3166 actual=addAllResults,
3167 onpass="Set addAll correct",
3168 onfail="Set addAll was incorrect" )
3169
3170 main.step( "Distributed Set clear()" )
3171 onosSet.clear()
3172 clearResponses = []
3173 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003174 for i in range( main.numCtrls ):
3175 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003176 name="setTestClear-" + str( i ),
3177 args=[ onosSetName, " "], # Values doesn't matter
3178 kwargs={ "clear": True } )
3179 threads.append( t )
3180 t.start()
3181 for t in threads:
3182 t.join()
3183 clearResponses.append( t.result )
3184
3185 # main.TRUE = successfully changed the set
3186 # main.FALSE = action resulted in no change in set
3187 # main.ERROR - Some error in executing the function
3188 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003189 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003190 if clearResponses[ i ] == main.TRUE:
3191 # All is well
3192 pass
3193 elif clearResponses[ i ] == main.FALSE:
3194 # Nothing set, probably fine
3195 pass
3196 elif clearResponses[ i ] == main.ERROR:
3197 # Error in execution
3198 clearResults = main.FALSE
3199 else:
3200 # unexpected result
3201 clearResults = main.FALSE
3202 if clearResults != main.TRUE:
3203 main.log.error( "Error executing set clear" )
3204
3205 # Check if set is still correct
3206 size = len( onosSet )
3207 getResponses = []
3208 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003209 for i in range( main.numCtrls ):
3210 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003211 name="setTestGet-" + str( i ),
3212 args=[ onosSetName ] )
3213 threads.append( t )
3214 t.start()
3215 for t in threads:
3216 t.join()
3217 getResponses.append( t.result )
3218 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003219 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003220 if isinstance( getResponses[ i ], list):
3221 current = set( getResponses[ i ] )
3222 if len( current ) == len( getResponses[ i ] ):
3223 # no repeats
3224 if onosSet != current:
3225 main.log.error( "ONOS" + str( i + 1 ) +
3226 " has incorrect view" +
3227 " of set " + onosSetName + ":\n" +
3228 str( getResponses[ i ] ) )
3229 main.log.debug( "Expected: " + str( onosSet ) )
3230 main.log.debug( "Actual: " + str( current ) )
3231 getResults = main.FALSE
3232 else:
3233 # error, set is not a set
3234 main.log.error( "ONOS" + str( i + 1 ) +
3235 " has repeat elements in" +
3236 " set " + onosSetName + ":\n" +
3237 str( getResponses[ i ] ) )
3238 getResults = main.FALSE
3239 elif getResponses[ i ] == main.ERROR:
3240 getResults = main.FALSE
3241 sizeResponses = []
3242 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003243 for i in range( main.numCtrls ):
3244 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003245 name="setTestSize-" + str( i ),
3246 args=[ onosSetName ] )
3247 threads.append( t )
3248 t.start()
3249 for t in threads:
3250 t.join()
3251 sizeResponses.append( t.result )
3252 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003253 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003254 if size != sizeResponses[ i ]:
3255 sizeResults = main.FALSE
3256 main.log.error( "ONOS" + str( i + 1 ) +
3257 " expected a size of " + str( size ) +
3258 " for set " + onosSetName +
3259 " but got " + str( sizeResponses[ i ] ) )
3260 clearResults = clearResults and getResults and sizeResults
3261 utilities.assert_equals( expect=main.TRUE,
3262 actual=clearResults,
3263 onpass="Set clear correct",
3264 onfail="Set clear was incorrect" )
3265
3266 main.step( "Distributed Set addAll()" )
3267 onosSet.update( addAllValue.split() )
3268 addResponses = []
3269 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003270 for i in range( main.numCtrls ):
3271 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003272 name="setTestAddAll-" + str( i ),
3273 args=[ onosSetName, addAllValue ] )
3274 threads.append( t )
3275 t.start()
3276 for t in threads:
3277 t.join()
3278 addResponses.append( t.result )
3279
3280 # main.TRUE = successfully changed the set
3281 # main.FALSE = action resulted in no change in set
3282 # main.ERROR - Some error in executing the function
3283 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003284 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003285 if addResponses[ i ] == main.TRUE:
3286 # All is well
3287 pass
3288 elif addResponses[ i ] == main.FALSE:
3289 # Already in set, probably fine
3290 pass
3291 elif addResponses[ i ] == main.ERROR:
3292 # Error in execution
3293 addAllResults = main.FALSE
3294 else:
3295 # unexpected result
3296 addAllResults = main.FALSE
3297 if addAllResults != main.TRUE:
3298 main.log.error( "Error executing set addAll" )
3299
3300 # Check if set is still correct
3301 size = len( onosSet )
3302 getResponses = []
3303 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003304 for i in range( main.numCtrls ):
3305 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003306 name="setTestGet-" + str( i ),
3307 args=[ onosSetName ] )
3308 threads.append( t )
3309 t.start()
3310 for t in threads:
3311 t.join()
3312 getResponses.append( t.result )
3313 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003314 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003315 if isinstance( getResponses[ i ], list):
3316 current = set( getResponses[ i ] )
3317 if len( current ) == len( getResponses[ i ] ):
3318 # no repeats
3319 if onosSet != current:
3320 main.log.error( "ONOS" + str( i + 1 ) +
3321 " has incorrect view" +
3322 " of set " + onosSetName + ":\n" +
3323 str( getResponses[ i ] ) )
3324 main.log.debug( "Expected: " + str( onosSet ) )
3325 main.log.debug( "Actual: " + str( current ) )
3326 getResults = main.FALSE
3327 else:
3328 # error, set is not a set
3329 main.log.error( "ONOS" + str( i + 1 ) +
3330 " has repeat elements in" +
3331 " set " + onosSetName + ":\n" +
3332 str( getResponses[ i ] ) )
3333 getResults = main.FALSE
3334 elif getResponses[ i ] == main.ERROR:
3335 getResults = main.FALSE
3336 sizeResponses = []
3337 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003338 for i in range( main.numCtrls ):
3339 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003340 name="setTestSize-" + str( i ),
3341 args=[ onosSetName ] )
3342 threads.append( t )
3343 t.start()
3344 for t in threads:
3345 t.join()
3346 sizeResponses.append( t.result )
3347 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003348 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003349 if size != sizeResponses[ i ]:
3350 sizeResults = main.FALSE
3351 main.log.error( "ONOS" + str( i + 1 ) +
3352 " expected a size of " + str( size ) +
3353 " for set " + onosSetName +
3354 " but got " + str( sizeResponses[ i ] ) )
3355 addAllResults = addAllResults and getResults and sizeResults
3356 utilities.assert_equals( expect=main.TRUE,
3357 actual=addAllResults,
3358 onpass="Set addAll correct",
3359 onfail="Set addAll was incorrect" )
3360
3361 main.step( "Distributed Set retain()" )
3362 onosSet.intersection_update( retainValue.split() )
3363 retainResponses = []
3364 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003365 for i in range( main.numCtrls ):
3366 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003367 name="setTestRetain-" + str( i ),
3368 args=[ onosSetName, retainValue ],
3369 kwargs={ "retain": True } )
3370 threads.append( t )
3371 t.start()
3372 for t in threads:
3373 t.join()
3374 retainResponses.append( t.result )
3375
3376 # main.TRUE = successfully changed the set
3377 # main.FALSE = action resulted in no change in set
3378 # main.ERROR - Some error in executing the function
3379 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003380 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003381 if retainResponses[ i ] == main.TRUE:
3382 # All is well
3383 pass
3384 elif retainResponses[ i ] == main.FALSE:
3385 # Already in set, probably fine
3386 pass
3387 elif retainResponses[ i ] == main.ERROR:
3388 # Error in execution
3389 retainResults = main.FALSE
3390 else:
3391 # unexpected result
3392 retainResults = main.FALSE
3393 if retainResults != main.TRUE:
3394 main.log.error( "Error executing set retain" )
3395
3396 # Check if set is still correct
3397 size = len( onosSet )
3398 getResponses = []
3399 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003400 for i in range( main.numCtrls ):
3401 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003402 name="setTestGet-" + str( i ),
3403 args=[ onosSetName ] )
3404 threads.append( t )
3405 t.start()
3406 for t in threads:
3407 t.join()
3408 getResponses.append( t.result )
3409 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003410 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003411 if isinstance( getResponses[ i ], list):
3412 current = set( getResponses[ i ] )
3413 if len( current ) == len( getResponses[ i ] ):
3414 # no repeats
3415 if onosSet != current:
3416 main.log.error( "ONOS" + str( i + 1 ) +
3417 " has incorrect view" +
3418 " of set " + onosSetName + ":\n" +
3419 str( getResponses[ i ] ) )
3420 main.log.debug( "Expected: " + str( onosSet ) )
3421 main.log.debug( "Actual: " + str( current ) )
3422 getResults = main.FALSE
3423 else:
3424 # error, set is not a set
3425 main.log.error( "ONOS" + str( i + 1 ) +
3426 " has repeat elements in" +
3427 " set " + onosSetName + ":\n" +
3428 str( getResponses[ i ] ) )
3429 getResults = main.FALSE
3430 elif getResponses[ i ] == main.ERROR:
3431 getResults = main.FALSE
3432 sizeResponses = []
3433 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003434 for i in range( main.numCtrls ):
3435 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003436 name="setTestSize-" + str( i ),
3437 args=[ onosSetName ] )
3438 threads.append( t )
3439 t.start()
3440 for t in threads:
3441 t.join()
3442 sizeResponses.append( t.result )
3443 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003444 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003445 if size != sizeResponses[ i ]:
3446 sizeResults = main.FALSE
3447 main.log.error( "ONOS" + str( i + 1 ) +
3448 " expected a size of " +
3449 str( size ) + " for set " + onosSetName +
3450 " but got " + str( sizeResponses[ i ] ) )
3451 retainResults = retainResults and getResults and sizeResults
3452 utilities.assert_equals( expect=main.TRUE,
3453 actual=retainResults,
3454 onpass="Set retain correct",
3455 onfail="Set retain was incorrect" )
3456
Jon Hall2a5002c2015-08-21 16:49:11 -07003457 # Transactional maps
3458 main.step( "Partitioned Transactional maps put" )
3459 tMapValue = "Testing"
3460 numKeys = 100
3461 putResult = True
3462 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
3463 if len( putResponses ) == 100:
3464 for i in putResponses:
3465 if putResponses[ i ][ 'value' ] != tMapValue:
3466 putResult = False
3467 else:
3468 putResult = False
3469 if not putResult:
3470 main.log.debug( "Put response values: " + str( putResponses ) )
3471 utilities.assert_equals( expect=True,
3472 actual=putResult,
3473 onpass="Partitioned Transactional Map put successful",
3474 onfail="Partitioned Transactional Map put values are incorrect" )
3475
3476 main.step( "Partitioned Transactional maps get" )
3477 getCheck = True
3478 for n in range( 1, numKeys + 1 ):
3479 getResponses = []
3480 threads = []
3481 valueCheck = True
3482 for i in range( main.numCtrls ):
3483 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
3484 name="TMap-get-" + str( i ),
3485 args=[ "Key" + str ( n ) ] )
3486 threads.append( t )
3487 t.start()
3488 for t in threads:
3489 t.join()
3490 getResponses.append( t.result )
3491 for node in getResponses:
3492 if node != tMapValue:
3493 valueCheck = False
3494 if not valueCheck:
3495 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
3496 main.log.warn( getResponses )
3497 getCheck = getCheck and valueCheck
3498 utilities.assert_equals( expect=True,
3499 actual=getCheck,
3500 onpass="Partitioned Transactional Map get values were correct",
3501 onfail="Partitioned Transactional Map values incorrect" )
3502
3503 main.step( "In-memory Transactional maps put" )
3504 tMapValue = "Testing"
3505 numKeys = 100
3506 putResult = True
3507 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
3508 if len( putResponses ) == 100:
3509 for i in putResponses:
3510 if putResponses[ i ][ 'value' ] != tMapValue:
3511 putResult = False
3512 else:
3513 putResult = False
3514 if not putResult:
3515 main.log.debug( "Put response values: " + str( putResponses ) )
3516 utilities.assert_equals( expect=True,
3517 actual=putResult,
3518 onpass="In-Memory Transactional Map put successful",
3519 onfail="In-Memory Transactional Map put values are incorrect" )
3520
3521 main.step( "In-Memory Transactional maps get" )
3522 getCheck = True
3523 for n in range( 1, numKeys + 1 ):
3524 getResponses = []
3525 threads = []
3526 valueCheck = True
3527 for i in range( main.numCtrls ):
3528 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
3529 name="TMap-get-" + str( i ),
3530 args=[ "Key" + str ( n ) ],
3531 kwargs={ "inMemory": True } )
3532 threads.append( t )
3533 t.start()
3534 for t in threads:
3535 t.join()
3536 getResponses.append( t.result )
3537 for node in getResponses:
3538 if node != tMapValue:
3539 valueCheck = False
3540 if not valueCheck:
3541 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
3542 main.log.warn( getResponses )
3543 getCheck = getCheck and valueCheck
3544 utilities.assert_equals( expect=True,
3545 actual=getCheck,
3546 onpass="In-Memory Transactional Map get values were correct",
3547 onfail="In-Memory Transactional Map values incorrect" )