blob: a1652e9571b33f7408bc5ccea88b049144fd9e10 [file] [log] [blame]
Jon Hall85794ff2015-07-08 14:12:30 -07001"""
2Description: This test is to determine if a single
3 instance ONOS 'cluster' can handle a restart
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAsingleInstanceRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hall85794ff2015-07-08 14:12:30 -070051 main.log.info( "ONOS Single node cluster restart " +
52 "HA test - initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall85794ff2015-07-08 14:12:30 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall85794ff2015-07-08 14:12:30 -070070
Jon Halle1a3b752015-07-22 13:02:46 -070071 try:
72 fileName = "Counters"
73 path = main.params[ 'imports' ][ 'path' ]
74 main.Counters = imp.load_source( fileName,
75 path + fileName + ".py" )
76 except Exception as e:
77 main.log.exception( e )
78 main.cleanup()
79 main.exit()
80
81 main.CLIs = []
82 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -070083 ipList = []
84 for i in range( 1, int( main.ONOSbench.maxNodes ) + 1 ):
85 try:
Jon Halle1a3b752015-07-22 13:02:46 -070086 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
87 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
88 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -070089 except AttributeError:
90 break
Jon Hall85794ff2015-07-08 14:12:30 -070091
Jon Hall5cf14d52015-07-16 12:15:19 -070092 main.step( "Create cell file" )
93 cellAppString = main.params[ 'ENV' ][ 'appString' ]
94 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
95 main.Mininet1.ip_address,
96 cellAppString, ipList )
Jon Hall85794ff2015-07-08 14:12:30 -070097 main.step( "Applying cell variable to environment" )
98 cellResult = main.ONOSbench.setCell( cellName )
99 verifyResult = main.ONOSbench.verifyCell()
100
101 # FIXME:this is short term fix
102 main.log.info( "Removing raft logs" )
103 main.ONOSbench.onosRemoveRaftLogs()
104
105 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700106 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700107 main.ONOSbench.onosUninstall( node.ip_address )
108
109 # Make sure ONOS is DEAD
110 main.log.info( "Killing any ONOS processes" )
111 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700112 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700113 killed = main.ONOSbench.onosKill( node.ip_address )
114 killResults = killResults and killed
115
116 cleanInstallResult = main.TRUE
117 gitPullResult = main.TRUE
118
119 main.step( "Starting Mininet" )
120 # scp topo file to mininet
121 # TODO: move to params?
122 topoName = "obelisk.py"
123 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700124 main.ONOSbench.scp( main.Mininet1,
125 filePath + topoName,
126 main.Mininet1.home,
127 direction="to" )
Jon Hall85794ff2015-07-08 14:12:30 -0700128 mnResult = main.Mininet1.startNet( )
129 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
130 onpass="Mininet Started",
131 onfail="Error starting Mininet" )
132
133 main.step( "Git checkout and pull " + gitBranch )
134 if PULLCODE:
135 main.ONOSbench.gitCheckout( gitBranch )
136 gitPullResult = main.ONOSbench.gitPull()
137 # values of 1 or 3 are good
138 utilities.assert_lesser( expect=0, actual=gitPullResult,
139 onpass="Git pull successful",
140 onfail="Git pull failed" )
141 main.ONOSbench.getVersion( report=True )
142
143 main.step( "Using mvn clean install" )
144 cleanInstallResult = main.TRUE
145 if PULLCODE and gitPullResult == main.TRUE:
146 cleanInstallResult = main.ONOSbench.cleanInstall()
147 else:
148 main.log.warn( "Did not pull new code so skipping mvn " +
149 "clean install" )
150 utilities.assert_equals( expect=main.TRUE,
151 actual=cleanInstallResult,
152 onpass="MCI successful",
153 onfail="MCI failed" )
154 # GRAPHS
155 # NOTE: important params here:
156 # job = name of Jenkins job
157 # Plot Name = Plot-HA, only can be used if multiple plots
158 # index = The number of the graph under plot name
Jon Hall5cf14d52015-07-16 12:15:19 -0700159 job = "HAsingleInstanceRestart"
Jon Hall85794ff2015-07-08 14:12:30 -0700160 plotName = "Plot-HA"
161 graphs = '<ac:structured-macro ac:name="html">\n'
162 graphs += '<ac:plain-text-body><![CDATA[\n'
163 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
164 '/plot/' + plotName + '/getPlot?index=0' +\
165 '&width=500&height=300"' +\
166 'noborder="0" width="500" height="300" scrolling="yes" ' +\
167 'seamless="seamless"></iframe>\n'
168 graphs += ']]></ac:plain-text-body>\n'
169 graphs += '</ac:structured-macro>\n'
170 main.log.wiki(graphs)
171
Jon Halle1a3b752015-07-22 13:02:46 -0700172 main.CLIs = []
173 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700174 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700175 for i in range( 1, main.numCtrls + 1 ):
176 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
177 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
178 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700179
180 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, "SingleHA",
181 main.Mininet1.ip_address,
182 cellAppString, ipList[ 0 ] )
Jon Hall85794ff2015-07-08 14:12:30 -0700183 cellResult = main.ONOSbench.setCell( "SingleHA" )
184 verifyResult = main.ONOSbench.verifyCell()
185 main.step( "Creating ONOS package" )
186 packageResult = main.ONOSbench.onosPackage()
187 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
188 onpass="ONOS package successful",
189 onfail="ONOS package failed" )
190
191 main.step( "Installing ONOS package" )
192 onosInstallResult = main.ONOSbench.onosInstall(
Jon Halle1a3b752015-07-22 13:02:46 -0700193 options="-f", node=main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700194 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
195 onpass="ONOS install successful",
196 onfail="ONOS install failed" )
197
198 main.step( "Checking if ONOS is up yet" )
199 for i in range( 2 ):
Jon Halle1a3b752015-07-22 13:02:46 -0700200 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700201 if onos1Isup:
202 break
203 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
204 onpass="ONOS startup successful",
205 onfail="ONOS startup failed" )
206
207 main.log.step( "Starting ONOS CLI sessions" )
Jon Halle1a3b752015-07-22 13:02:46 -0700208 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700209 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
210 onpass="ONOS cli startup successful",
211 onfail="ONOS cli startup failed" )
212
213 if main.params[ 'tcpdump' ].lower() == "true":
214 main.step( "Start Packet Capture MN" )
215 main.Mininet2.startTcpdump(
216 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
217 + "-MN.pcap",
218 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
219 port=main.params[ 'MNtcpdump' ][ 'port' ] )
220
221 main.step( "App Ids check" )
222 appCheck = main.ONOScli1.appToIDCheck()
223 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700224 main.log.warn( main.CLIs[0].apps() )
225 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700226 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
227 onpass="App Ids seem to be correct",
228 onfail="Something is wrong with app Ids" )
229
230 if cliResults == main.FALSE:
231 main.log.error( "Failed to start ONOS, stopping test" )
232 main.cleanup()
233 main.exit()
234
235 def CASE2( self, main ):
236 """
237 Assign devices to controllers
238 """
239 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700240 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700241 assert main, "main not defined"
242 assert utilities.assert_equals, "utilities.assert_equals not defined"
243
244 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700245 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700246 "and check that an ONOS node becomes the " +\
247 "master of the device."
248 main.step( "Assign switches to controllers" )
249
250 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700251 for i in range( main.numCtrls ):
252 ipList.append( main.nodes[ i ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700253 swList = []
254 for i in range( 1, 29 ):
255 swList.append( "s" + str( i ) )
256 main.Mininet1.assignSwController( sw=swList, ip=ipList )
257
258 mastershipCheck = main.TRUE
259 for i in range( 1, 29 ):
260 response = main.Mininet1.getSwController( "s" + str( i ) )
261 try:
262 main.log.info( str( response ) )
263 except Exception:
264 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700265 if re.search( "tcp:" + main.nodes[0].ip_address, response ):
Jon Hall85794ff2015-07-08 14:12:30 -0700266 mastershipCheck = mastershipCheck and main.TRUE
267 else:
268 mastershipCheck = main.FALSE
269 if mastershipCheck == main.TRUE:
270 main.log.info( "Switch mastership assigned correctly" )
271 utilities.assert_equals(
272 expect=main.TRUE,
273 actual=mastershipCheck,
274 onpass="Switch mastership assigned correctly",
275 onfail="Switches not assigned correctly to controllers" )
276
277 def CASE21( self, main ):
278 """
279 Assign mastership to controllers
280 """
Jon Halle1a3b752015-07-22 13:02:46 -0700281 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700282 assert main, "main not defined"
283 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700284 assert main.CLIs, "main.CLIs not defined"
285 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700286
287 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700288 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700289 "device. Then manually assign" +\
290 " mastership to specific ONOS nodes using" +\
291 " 'device-role'"
292 main.step( "Assign mastership of switches to specific controllers" )
293 roleCall = main.TRUE
294 roleCheck = main.TRUE
295 try:
296 for i in range( 1, 29 ): # switches 1 through 28
Jon Halle1a3b752015-07-22 13:02:46 -0700297 ip = main.nodes[ 0 ].ip_address # ONOS1
Jon Hall85794ff2015-07-08 14:12:30 -0700298 # set up correct variables:
299 if i == 1:
300 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
301 elif i == 2:
302 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
303 elif i == 3:
304 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
305 elif i == 4:
306 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
307 elif i == 5:
308 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
309 elif i == 6:
310 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
311 elif i == 7:
312 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
313 elif i >= 8 and i <= 17:
314 dpid = '3' + str( i ).zfill( 3 )
315 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
316 elif i >= 18 and i <= 27:
317 dpid = '6' + str( i ).zfill( 3 )
318 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
319 elif i == 28:
320 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
321 else:
322 main.log.error( "You didn't write an else statement for " +
323 "switch s" + str( i ) )
324 # Assign switch
325 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
326 # TODO: make this controller dynamic
327 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
328 ip )
329 # Check assignment
330 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
331 if ip in master:
332 roleCheck = roleCheck and main.TRUE
333 else:
334 roleCheck = roleCheck and main.FALSE
335 main.log.error( "Error, controller " + ip + " is not" +
336 " master " + "of device " +
337 str( deviceId ) + ". Master is " +
338 repr( master ) + "." )
339 except ( AttributeError, AssertionError ):
340 main.log.exception( "Something is wrong with ONOS device view" )
341 main.log.info( main.ONOScli1.devices() )
342 utilities.assert_equals(
343 expect=main.TRUE,
344 actual=roleCall,
345 onpass="Re-assigned switch mastership to designated controller",
346 onfail="Something wrong with deviceRole calls" )
347
348 main.step( "Check mastership was correctly assigned" )
349 utilities.assert_equals(
350 expect=main.TRUE,
351 actual=roleCheck,
352 onpass="Switches were successfully reassigned to designated " +
353 "controller",
354 onfail="Switches were not successfully reassigned" )
355
356 def CASE3( self, main ):
357 """
358 Assign intents
359 """
360 import time
361 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700362 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700363 assert main, "main not defined"
364 assert utilities.assert_equals, "utilities.assert_equals not defined"
365 # NOTE: we must reinstall intents until we have a persistant intent
366 # datastore!
367 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700368 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700369 "assign predetermined host-to-host intents." +\
370 " After installation, check that the intent" +\
371 " is distributed to all nodes and the state" +\
372 " is INSTALLED"
373
374 # install onos-app-fwd
375 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700376 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700377 utilities.assert_equals( expect=main.TRUE, actual=installResults,
378 onpass="Install fwd successful",
379 onfail="Install fwd failed" )
380
381 main.step( "Check app ids" )
382 appCheck = main.ONOScli1.appToIDCheck()
383 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700384 main.log.warn( main.CLIs[0].apps() )
385 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700386 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
387 onpass="App Ids seem to be correct",
388 onfail="Something is wrong with app Ids" )
389
390 main.step( "Discovering Hosts( Via pingall for now )" )
391 # FIXME: Once we have a host discovery mechanism, use that instead
392 # REACTIVE FWD test
393 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700394 passMsg = "Reactive Pingall test passed"
395 time1 = time.time()
396 pingResult = main.Mininet1.pingall()
397 time2 = time.time()
398 if not pingResult:
399 main.log.warn("First pingall failed. Trying again...")
Jon Hall85794ff2015-07-08 14:12:30 -0700400 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700401 passMsg += " on the second try"
402 utilities.assert_equals(
403 expect=main.TRUE,
404 actual=pingResult,
405 onpass= passMsg,
406 onfail="Reactive Pingall failed, " +
407 "one or more ping pairs failed" )
408 main.log.info( "Time for pingall: %2f seconds" %
409 ( time2 - time1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700410 # timeout for fwd flows
411 time.sleep( 11 )
412 # uninstall onos-app-fwd
413 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700414 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700415 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
416 onpass="Uninstall fwd successful",
417 onfail="Uninstall fwd failed" )
418
419 main.step( "Check app ids" )
420 appCheck2 = main.ONOScli1.appToIDCheck()
421 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700422 main.log.warn( main.CLIs[0].apps() )
423 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700424 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
425 onpass="App Ids seem to be correct",
426 onfail="Something is wrong with app Ids" )
427
428 main.step( "Add host intents via cli" )
429 intentIds = []
430 # TODO: move the host numbers to params
431 # Maybe look at all the paths we ping?
432 intentAddResult = True
433 hostResult = main.TRUE
434 for i in range( 8, 18 ):
435 main.log.info( "Adding host intent between h" + str( i ) +
436 " and h" + str( i + 10 ) )
437 host1 = "00:00:00:00:00:" + \
438 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
439 host2 = "00:00:00:00:00:" + \
440 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
441 # NOTE: getHost can return None
442 host1Dict = main.ONOScli1.getHost( host1 )
443 host2Dict = main.ONOScli1.getHost( host2 )
444 host1Id = None
445 host2Id = None
446 if host1Dict and host2Dict:
447 host1Id = host1Dict.get( 'id', None )
448 host2Id = host2Dict.get( 'id', None )
449 if host1Id and host2Id:
450 tmpId = main.ONOScli1.addHostIntent( host1Id, host2Id )
451 if tmpId:
452 main.log.info( "Added intent with id: " + tmpId )
453 intentIds.append( tmpId )
454 else:
455 main.log.error( "addHostIntent returned: " +
456 repr( tmpId ) )
457 else:
458 main.log.error( "Error, getHost() failed for h" + str( i ) +
459 " and/or h" + str( i + 10 ) )
460 hosts = main.ONOScli1.hosts()
461 main.log.warn( "Hosts output: " )
462 try:
463 main.log.warn( json.dumps( json.loads( hosts ),
464 sort_keys=True,
465 indent=4,
466 separators=( ',', ': ' ) ) )
467 except ( ValueError, TypeError ):
468 main.log.warn( repr( hosts ) )
469 hostResult = main.FALSE
470 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
471 onpass="Found a host id for each host",
472 onfail="Error looking up host ids" )
473
474 intentStart = time.time()
475 onosIds = main.ONOScli1.getAllIntentsId()
476 main.log.info( "Submitted intents: " + str( intentIds ) )
477 main.log.info( "Intents in ONOS: " + str( onosIds ) )
478 for intent in intentIds:
479 if intent in onosIds:
480 pass # intent submitted is in onos
481 else:
482 intentAddResult = False
483 if intentAddResult:
484 intentStop = time.time()
485 else:
486 intentStop = None
487 # Print the intent states
488 intents = main.ONOScli1.intents()
489 intentStates = []
490 installedCheck = True
491 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
492 count = 0
493 try:
494 for intent in json.loads( intents ):
495 state = intent.get( 'state', None )
496 if "INSTALLED" not in state:
497 installedCheck = False
498 intentId = intent.get( 'id', None )
499 intentStates.append( ( intentId, state ) )
500 except ( ValueError, TypeError ):
501 main.log.exception( "Error parsing intents" )
502 # add submitted intents not in the store
503 tmplist = [ i for i, s in intentStates ]
504 missingIntents = False
505 for i in intentIds:
506 if i not in tmplist:
507 intentStates.append( ( i, " - " ) )
508 missingIntents = True
509 intentStates.sort()
510 for i, s in intentStates:
511 count += 1
512 main.log.info( "%-6s%-15s%-15s" %
513 ( str( count ), str( i ), str( s ) ) )
514 leaders = main.ONOScli1.leaders()
515 try:
516 missing = False
517 if leaders:
518 parsedLeaders = json.loads( leaders )
519 main.log.warn( json.dumps( parsedLeaders,
520 sort_keys=True,
521 indent=4,
522 separators=( ',', ': ' ) ) )
523 # check for all intent partitions
524 topics = []
525 for i in range( 14 ):
526 topics.append( "intent-partition-" + str( i ) )
527 main.log.debug( topics )
528 ONOStopics = [ j['topic'] for j in parsedLeaders ]
529 for topic in topics:
530 if topic not in ONOStopics:
531 main.log.error( "Error: " + topic +
532 " not in leaders" )
533 missing = True
534 else:
535 main.log.error( "leaders() returned None" )
536 except ( ValueError, TypeError ):
537 main.log.exception( "Error parsing leaders" )
538 main.log.error( repr( leaders ) )
539 # Check all nodes
540 if missing:
541 response = main.ONOScli1.leaders( jsonFormat=False)
542 main.log.warn( "ONOS1 leaders output: \n" +
543 str( response ) )
544
545 partitions = main.ONOScli1.partitions()
546 try:
547 if partitions :
548 parsedPartitions = json.loads( partitions )
549 main.log.warn( json.dumps( parsedPartitions,
550 sort_keys=True,
551 indent=4,
552 separators=( ',', ': ' ) ) )
553 # TODO check for a leader in all paritions
554 # TODO check for consistency among nodes
555 else:
556 main.log.error( "partitions() returned None" )
557 except ( ValueError, TypeError ):
558 main.log.exception( "Error parsing partitions" )
559 main.log.error( repr( partitions ) )
560 pendingMap = main.ONOScli1.pendingMap()
561 try:
562 if pendingMap :
563 parsedPending = json.loads( pendingMap )
564 main.log.warn( json.dumps( parsedPending,
565 sort_keys=True,
566 indent=4,
567 separators=( ',', ': ' ) ) )
568 # TODO check something here?
569 else:
570 main.log.error( "pendingMap() returned None" )
571 except ( ValueError, TypeError ):
572 main.log.exception( "Error parsing pending map" )
573 main.log.error( repr( pendingMap ) )
574
575 intentAddResult = bool( intentAddResult and not missingIntents and
576 installedCheck )
577 if not intentAddResult:
578 main.log.error( "Error in pushing host intents to ONOS" )
579
580 main.step( "Intent Anti-Entropy dispersion" )
581 for i in range(100):
582 correct = True
583 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700584 for cli in main.CLIs:
Jon Hall85794ff2015-07-08 14:12:30 -0700585 onosIds = []
586 ids = cli.getAllIntentsId()
587 onosIds.append( ids )
588 main.log.debug( "Intents in " + cli.name + ": " +
589 str( sorted( onosIds ) ) )
590 if sorted( ids ) != sorted( intentIds ):
591 main.log.warn( "Set of intent IDs doesn't match" )
592 correct = False
593 break
594 else:
595 intents = json.loads( cli.intents() )
596 for intent in intents:
597 if intent[ 'state' ] != "INSTALLED":
598 main.log.warn( "Intent " + intent[ 'id' ] +
599 " is " + intent[ 'state' ] )
600 correct = False
601 break
602 if correct:
603 break
604 else:
605 time.sleep(1)
606 if not intentStop:
607 intentStop = time.time()
608 global gossipTime
609 gossipTime = intentStop - intentStart
610 main.log.info( "It took about " + str( gossipTime ) +
611 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700612 gossipPeriod = int( main.params['timers']['gossip'] )
613 maxGossipTime = gossipPeriod * len( main.nodes )
Jon Hall85794ff2015-07-08 14:12:30 -0700614 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700615 expect=maxGossipTime, actual=gossipTime,
Jon Hall85794ff2015-07-08 14:12:30 -0700616 onpass="ECM anti-entropy for intents worked within " +
617 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700618 onfail="Intent ECM anti-entropy took too long. " +
619 "Expected time:{}, Actual time:{}".format( maxGossipTime,
620 gossipTime ) )
621 if gossipTime <= maxGossipTime:
Jon Hall85794ff2015-07-08 14:12:30 -0700622 intentAddResult = True
623
624 if not intentAddResult or "key" in pendingMap:
625 import time
626 installedCheck = True
627 main.log.info( "Sleeping 60 seconds to see if intents are found" )
628 time.sleep( 60 )
629 onosIds = main.ONOScli1.getAllIntentsId()
630 main.log.info( "Submitted intents: " + str( intentIds ) )
631 main.log.info( "Intents in ONOS: " + str( onosIds ) )
632 # Print the intent states
633 intents = main.ONOScli1.intents()
634 intentStates = []
635 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
636 count = 0
637 try:
638 for intent in json.loads( intents ):
639 # Iter through intents of a node
640 state = intent.get( 'state', None )
641 if "INSTALLED" not in state:
642 installedCheck = False
643 intentId = intent.get( 'id', None )
644 intentStates.append( ( intentId, state ) )
645 except ( ValueError, TypeError ):
646 main.log.exception( "Error parsing intents" )
647 # add submitted intents not in the store
648 tmplist = [ i for i, s in intentStates ]
649 for i in intentIds:
650 if i not in tmplist:
651 intentStates.append( ( i, " - " ) )
652 intentStates.sort()
653 for i, s in intentStates:
654 count += 1
655 main.log.info( "%-6s%-15s%-15s" %
656 ( str( count ), str( i ), str( s ) ) )
657 leaders = main.ONOScli1.leaders()
658 try:
659 missing = False
660 if leaders:
661 parsedLeaders = json.loads( leaders )
662 main.log.warn( json.dumps( parsedLeaders,
663 sort_keys=True,
664 indent=4,
665 separators=( ',', ': ' ) ) )
666 # check for all intent partitions
667 # check for election
668 topics = []
669 for i in range( 14 ):
670 topics.append( "intent-partition-" + str( i ) )
671 # FIXME: this should only be after we start the app
672 topics.append( "org.onosproject.election" )
673 main.log.debug( topics )
674 ONOStopics = [ j['topic'] for j in parsedLeaders ]
675 for topic in topics:
676 if topic not in ONOStopics:
677 main.log.error( "Error: " + topic +
678 " not in leaders" )
679 missing = True
680 else:
681 main.log.error( "leaders() returned None" )
682 except ( ValueError, TypeError ):
683 main.log.exception( "Error parsing leaders" )
684 main.log.error( repr( leaders ) )
685 # Check all nodes
686 if missing:
687 response = main.ONOScli1.leaders( jsonFormat=False)
688 main.log.warn( "ONOS1 leaders output: \n" +
689 str( response ) )
690 partitions = main.ONOScli1.partitions()
691 try:
692 if partitions :
693 parsedPartitions = json.loads( partitions )
694 main.log.warn( json.dumps( parsedPartitions,
695 sort_keys=True,
696 indent=4,
697 separators=( ',', ': ' ) ) )
698 # TODO check for a leader in all paritions
699 # TODO check for consistency among nodes
700 else:
701 main.log.error( "partitions() returned None" )
702 except ( ValueError, TypeError ):
703 main.log.exception( "Error parsing partitions" )
704 main.log.error( repr( partitions ) )
705 pendingMap = main.ONOScli1.pendingMap()
706 try:
707 if pendingMap :
708 parsedPending = json.loads( pendingMap )
709 main.log.warn( json.dumps( parsedPending,
710 sort_keys=True,
711 indent=4,
712 separators=( ',', ': ' ) ) )
713 # TODO check something here?
714 else:
715 main.log.error( "pendingMap() returned None" )
716 except ( ValueError, TypeError ):
717 main.log.exception( "Error parsing pending map" )
718 main.log.error( repr( pendingMap ) )
719
720 def CASE4( self, main ):
721 """
722 Ping across added host intents
723 """
724 import json
725 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700726 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700727 assert main, "main not defined"
728 assert utilities.assert_equals, "utilities.assert_equals not defined"
729 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700730 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700731 "functionality and check the state of " +\
732 "the intent"
733 main.step( "Ping across added host intents" )
734 PingResult = main.TRUE
735 for i in range( 8, 18 ):
736 ping = main.Mininet1.pingHost( src="h" + str( i ),
737 target="h" + str( i + 10 ) )
738 PingResult = PingResult and ping
739 if ping == main.FALSE:
740 main.log.warn( "Ping failed between h" + str( i ) +
741 " and h" + str( i + 10 ) )
742 elif ping == main.TRUE:
743 main.log.info( "Ping test passed!" )
744 # Don't set PingResult or you'd override failures
745 if PingResult == main.FALSE:
746 main.log.error(
747 "Intents have not been installed correctly, pings failed." )
748 # TODO: pretty print
749 main.log.warn( "ONOS1 intents: " )
750 try:
751 tmpIntents = main.ONOScli1.intents()
752 main.log.warn( json.dumps( json.loads( tmpIntents ),
753 sort_keys=True,
754 indent=4,
755 separators=( ',', ': ' ) ) )
756 except ( ValueError, TypeError ):
757 main.log.warn( repr( tmpIntents ) )
758 utilities.assert_equals(
759 expect=main.TRUE,
760 actual=PingResult,
761 onpass="Intents have been installed correctly and pings work",
762 onfail="Intents have not been installed correctly, pings failed." )
763
764 main.step( "Check Intent state" )
765 installedCheck = True
766 # Print the intent states
767 intents = main.ONOScli1.intents()
768 intentStates = []
769 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
770 count = 0
771 # Iter through intents of a node
772 try:
773 for intent in json.loads( intents ):
774 state = intent.get( 'state', None )
775 if "INSTALLED" not in state:
776 installedCheck = False
777 intentId = intent.get( 'id', None )
778 intentStates.append( ( intentId, state ) )
779 except ( ValueError, TypeError ):
780 main.log.exception( "Error parsing intents." )
781 # Print states
782 intentStates.sort()
783 for i, s in intentStates:
784 count += 1
785 main.log.info( "%-6s%-15s%-15s" %
786 ( str( count ), str( i ), str( s ) ) )
787 utilities.assert_equals( expect=True, actual=installedCheck,
788 onpass="Intents are all INSTALLED",
789 onfail="Intents are not all in " +
790 "INSTALLED state" )
791
792 main.step( "Check leadership of topics" )
793 leaders = main.ONOScli1.leaders()
794 topicCheck = main.TRUE
795 try:
796 if leaders:
797 parsedLeaders = json.loads( leaders )
798 main.log.warn( json.dumps( parsedLeaders,
799 sort_keys=True,
800 indent=4,
801 separators=( ',', ': ' ) ) )
802 # check for all intent partitions
803 # check for election
804 # TODO: Look at Devices as topics now that it uses this system
805 topics = []
806 for i in range( 14 ):
807 topics.append( "intent-partition-" + str( i ) )
808 # FIXME: this should only be after we start the app
809 # FIXME: topics.append( "org.onosproject.election" )
810 # Print leaders output
811 main.log.debug( topics )
812 ONOStopics = [ j['topic'] for j in parsedLeaders ]
813 for topic in topics:
814 if topic not in ONOStopics:
815 main.log.error( "Error: " + topic +
816 " not in leaders" )
817 topicCheck = main.FALSE
818 else:
819 main.log.error( "leaders() returned None" )
820 topicCheck = main.FALSE
821 except ( ValueError, TypeError ):
822 topicCheck = main.FALSE
823 main.log.exception( "Error parsing leaders" )
824 main.log.error( repr( leaders ) )
825 # TODO: Check for a leader of these topics
826 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
827 onpass="intent Partitions is in leaders",
828 onfail="Some topics were lost " )
829 # Print partitions
830 partitions = main.ONOScli1.partitions()
831 try:
832 if partitions :
833 parsedPartitions = json.loads( partitions )
834 main.log.warn( json.dumps( parsedPartitions,
835 sort_keys=True,
836 indent=4,
837 separators=( ',', ': ' ) ) )
838 # TODO check for a leader in all paritions
839 # TODO check for consistency among nodes
840 else:
841 main.log.error( "partitions() returned None" )
842 except ( ValueError, TypeError ):
843 main.log.exception( "Error parsing partitions" )
844 main.log.error( repr( partitions ) )
845 # Print Pending Map
846 pendingMap = main.ONOScli1.pendingMap()
847 try:
848 if pendingMap :
849 parsedPending = json.loads( pendingMap )
850 main.log.warn( json.dumps( parsedPending,
851 sort_keys=True,
852 indent=4,
853 separators=( ',', ': ' ) ) )
854 # TODO check something here?
855 else:
856 main.log.error( "pendingMap() returned None" )
857 except ( ValueError, TypeError ):
858 main.log.exception( "Error parsing pending map" )
859 main.log.error( repr( pendingMap ) )
860
861 if not installedCheck:
862 main.log.info( "Waiting 60 seconds to see if the state of " +
863 "intents change" )
864 time.sleep( 60 )
865 # Print the intent states
866 intents = main.ONOScli1.intents()
867 intentStates = []
868 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
869 count = 0
870 # Iter through intents of a node
871 try:
872 for intent in json.loads( intents ):
873 state = intent.get( 'state', None )
874 if "INSTALLED" not in state:
875 installedCheck = False
876 intentId = intent.get( 'id', None )
877 intentStates.append( ( intentId, state ) )
878 except ( ValueError, TypeError ):
879 main.log.exception( "Error parsing intents." )
880 intentStates.sort()
881 for i, s in intentStates:
882 count += 1
883 main.log.info( "%-6s%-15s%-15s" %
884 ( str( count ), str( i ), str( s ) ) )
885 leaders = main.ONOScli1.leaders()
886 try:
887 missing = False
888 if leaders:
889 parsedLeaders = json.loads( leaders )
890 main.log.warn( json.dumps( parsedLeaders,
891 sort_keys=True,
892 indent=4,
893 separators=( ',', ': ' ) ) )
894 # check for all intent partitions
895 # check for election
896 topics = []
897 for i in range( 14 ):
898 topics.append( "intent-partition-" + str( i ) )
899 # FIXME: this should only be after we start the app
900 topics.append( "org.onosproject.election" )
901 main.log.debug( topics )
902 ONOStopics = [ j['topic'] for j in parsedLeaders ]
903 for topic in topics:
904 if topic not in ONOStopics:
905 main.log.error( "Error: " + topic +
906 " not in leaders" )
907 missing = True
908 else:
909 main.log.error( "leaders() returned None" )
910 except ( ValueError, TypeError ):
911 main.log.exception( "Error parsing leaders" )
912 main.log.error( repr( leaders ) )
913 if missing:
914 response = main.ONOScli1.leaders( jsonFormat=False)
915 main.log.warn( "ONOS1 leaders output: \n" +
916 str( response ) )
917 partitions = main.ONOScli1.partitions()
918 try:
919 if partitions :
920 parsedPartitions = json.loads( partitions )
921 main.log.warn( json.dumps( parsedPartitions,
922 sort_keys=True,
923 indent=4,
924 separators=( ',', ': ' ) ) )
925 # TODO check for a leader in all paritions
926 # TODO check for consistency among nodes
927 else:
928 main.log.error( "partitions() returned None" )
929 except ( ValueError, TypeError ):
930 main.log.exception( "Error parsing partitions" )
931 main.log.error( repr( partitions ) )
932 pendingMap = main.ONOScli1.pendingMap()
933 try:
934 if pendingMap :
935 parsedPending = json.loads( pendingMap )
936 main.log.warn( json.dumps( parsedPending,
937 sort_keys=True,
938 indent=4,
939 separators=( ',', ': ' ) ) )
940 # TODO check something here?
941 else:
942 main.log.error( "pendingMap() returned None" )
943 except ( ValueError, TypeError ):
944 main.log.exception( "Error parsing pending map" )
945 main.log.error( repr( pendingMap ) )
946 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -0700947 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700948 main.step( "Wait a minute then ping again" )
949 # the wait is above
950 PingResult = main.TRUE
951 for i in range( 8, 18 ):
952 ping = main.Mininet1.pingHost( src="h" + str( i ),
953 target="h" + str( i + 10 ) )
954 PingResult = PingResult and ping
955 if ping == main.FALSE:
956 main.log.warn( "Ping failed between h" + str( i ) +
957 " and h" + str( i + 10 ) )
958 elif ping == main.TRUE:
959 main.log.info( "Ping test passed!" )
960 # Don't set PingResult or you'd override failures
961 if PingResult == main.FALSE:
962 main.log.error(
963 "Intents have not been installed correctly, pings failed." )
964 # TODO: pretty print
965 main.log.warn( "ONOS1 intents: " )
966 try:
967 tmpIntents = main.ONOScli1.intents()
968 main.log.warn( json.dumps( json.loads( tmpIntents ),
969 sort_keys=True,
970 indent=4,
971 separators=( ',', ': ' ) ) )
972 except ( ValueError, TypeError ):
973 main.log.warn( repr( tmpIntents ) )
974 utilities.assert_equals(
975 expect=main.TRUE,
976 actual=PingResult,
977 onpass="Intents have been installed correctly and pings work",
978 onfail="Intents have not been installed correctly, pings failed." )
979
980 def CASE5( self, main ):
981 """
982 Reading state of ONOS
983 """
984 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700985 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700986 assert main, "main not defined"
987 assert utilities.assert_equals, "utilities.assert_equals not defined"
988
989 main.case( "Setting up and gathering data for current state" )
990 # The general idea for this test case is to pull the state of
991 # ( intents,flows, topology,... ) from each ONOS node
992 # We can then compare them with each other and also with past states
993
994 main.step( "Check that each switch has a master" )
995 global mastershipState
996 mastershipState = '[]'
997
998 # Assert that each device has a master
999 rolesNotNull = main.ONOScli1.rolesNotNull()
1000 utilities.assert_equals(
1001 expect=main.TRUE,
1002 actual=rolesNotNull,
1003 onpass="Each device has a master",
1004 onfail="Some devices don't have a master assigned" )
1005
1006 main.step( "Get the Mastership of each switch" )
1007 ONOS1Mastership = main.ONOScli1.roles()
1008 # TODO: Make this a meaningful check
1009 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1010 main.log.error( "Error in getting ONOS roles" )
1011 main.log.warn(
1012 "ONOS1 mastership response: " +
1013 repr( ONOS1Mastership ) )
1014 consistentMastership = main.FALSE
1015 else:
1016 mastershipState = ONOS1Mastership
1017 consistentMastership = main.TRUE
1018
1019 main.step( "Get the intents from each controller" )
1020 global intentState
1021 intentState = []
1022 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1023 intentCheck = main.FALSE
1024 if "Error" in ONOS1Intents or not ONOS1Intents:
1025 main.log.error( "Error in getting ONOS intents" )
1026 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1027 else:
1028 intentCheck = main.TRUE
1029
1030 main.step( "Get the flows from each controller" )
1031 global flowState
1032 flowState = []
1033 flowCheck = main.FALSE
1034 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
1035 if "Error" in ONOS1Flows or not ONOS1Flows:
1036 main.log.error( "Error in getting ONOS flows" )
1037 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
1038 else:
1039 # TODO: Do a better check, maybe compare flows on switches?
1040 flowState = ONOS1Flows
1041 flowCheck = main.TRUE
1042
1043 main.step( "Get the OF Table entries" )
1044 global flows
1045 flows = []
1046 for i in range( 1, 29 ):
Jon Hall9043c902015-07-30 14:23:44 -07001047 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001048 if flowCheck == main.FALSE:
1049 for table in flows:
1050 main.log.warn( table )
1051 # TODO: Compare switch flow tables with ONOS flow tables
1052
1053 main.step( "Collecting topology information from ONOS" )
1054 devices = []
1055 devices.append( main.ONOScli1.devices() )
1056 hosts = []
1057 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1058 ports = []
1059 ports.append( main.ONOScli1.ports() )
1060 links = []
1061 links.append( main.ONOScli1.links() )
1062 clusters = []
1063 clusters.append( main.ONOScli1.clusters() )
1064
1065 main.step( "Each host has an IP address" )
1066 ipResult = main.TRUE
1067 for controller in range( 0, len( hosts ) ):
1068 controllerStr = str( controller + 1 )
1069 for host in hosts[ controller ]:
1070 if host is None or host.get( 'ipAddresses', [] ) == []:
1071 main.log.error(
1072 "DEBUG:Error with host ips on controller" +
1073 controllerStr + ": " + str( host ) )
1074 ipResult = main.FALSE
1075 utilities.assert_equals(
1076 expect=main.TRUE,
1077 actual=ipResult,
1078 onpass="The ips of the hosts aren't empty",
1079 onfail="The ip of at least one host is missing" )
1080
1081 # there should always only be one cluster
1082 main.step( "There is only one dataplane cluster" )
1083 try:
1084 numClusters = len( json.loads( clusters[ 0 ] ) )
1085 except ( ValueError, TypeError ):
1086 main.log.exception( "Error parsing clusters[0]: " +
1087 repr( clusters[ 0 ] ) )
1088 clusterResults = main.FALSE
1089 if numClusters == 1:
1090 clusterResults = main.TRUE
1091 utilities.assert_equals(
1092 expect=1,
1093 actual=numClusters,
1094 onpass="ONOS shows 1 SCC",
1095 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1096
1097 main.step( "Comparing ONOS topology to MN" )
1098 devicesResults = main.TRUE
1099 linksResults = main.TRUE
1100 hostsResults = main.TRUE
1101 mnSwitches = main.Mininet1.getSwitches()
1102 mnLinks = main.Mininet1.getLinks()
1103 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001104 for controller in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07001105 controllerStr = str( controller + 1 )
1106 if devices[ controller ] and ports[ controller ] and\
1107 "Error" not in devices[ controller ] and\
1108 "Error" not in ports[ controller ]:
1109
1110 currentDevicesResult = main.Mininet1.compareSwitches(
1111 mnSwitches,
1112 json.loads( devices[ controller ] ),
1113 json.loads( ports[ controller ] ) )
1114 else:
1115 currentDevicesResult = main.FALSE
1116 utilities.assert_equals( expect=main.TRUE,
1117 actual=currentDevicesResult,
1118 onpass="ONOS" + controllerStr +
1119 " Switches view is correct",
1120 onfail="ONOS" + controllerStr +
1121 " Switches view is incorrect" )
1122 if links[ controller ] and "Error" not in links[ controller ]:
1123 currentLinksResult = main.Mininet1.compareLinks(
1124 mnSwitches, mnLinks,
1125 json.loads( links[ controller ] ) )
1126 else:
1127 currentLinksResult = main.FALSE
1128 utilities.assert_equals( expect=main.TRUE,
1129 actual=currentLinksResult,
1130 onpass="ONOS" + controllerStr +
1131 " links view is correct",
1132 onfail="ONOS" + controllerStr +
1133 " links view is incorrect" )
1134
1135 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1136 currentHostsResult = main.Mininet1.compareHosts(
1137 mnHosts,
1138 hosts[ controller ] )
1139 else:
1140 currentHostsResult = main.FALSE
1141 utilities.assert_equals( expect=main.TRUE,
1142 actual=currentHostsResult,
1143 onpass="ONOS" + controllerStr +
1144 " hosts exist in Mininet",
1145 onfail="ONOS" + controllerStr +
1146 " hosts don't match Mininet" )
1147
1148 devicesResults = devicesResults and currentDevicesResult
1149 linksResults = linksResults and currentLinksResult
1150 hostsResults = hostsResults and currentHostsResult
1151
1152 main.step( "Device information is correct" )
1153 utilities.assert_equals(
1154 expect=main.TRUE,
1155 actual=devicesResults,
1156 onpass="Device information is correct",
1157 onfail="Device information is incorrect" )
1158
1159 main.step( "Links are correct" )
1160 utilities.assert_equals(
1161 expect=main.TRUE,
1162 actual=linksResults,
1163 onpass="Link are correct",
1164 onfail="Links are incorrect" )
1165
1166 main.step( "Hosts are correct" )
1167 utilities.assert_equals(
1168 expect=main.TRUE,
1169 actual=hostsResults,
1170 onpass="Hosts are correct",
1171 onfail="Hosts are incorrect" )
1172
1173 def CASE6( self, main ):
1174 """
1175 The Failure case.
1176 """
1177 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001178 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001179 assert main, "main not defined"
1180 assert utilities.assert_equals, "utilities.assert_equals not defined"
1181
1182 # Reset non-persistent variables
1183 try:
1184 iCounterValue = 0
1185 except NameError:
1186 main.log.error( "iCounterValue not defined, setting to 0" )
1187 iCounterValue = 0
1188
1189 main.case( "Restart ONOS node" )
Jon Hall783bbf92015-07-23 14:33:19 -07001190 main.caseExplanation = "Killing ONOS process and restart cli " +\
Jon Hall85794ff2015-07-08 14:12:30 -07001191 "sessions once onos is up."
Jon Hall96091e62015-09-21 17:34:17 -07001192
1193 main.step( "Checking ONOS Logs for errors" )
1194 for node in main.nodes:
1195 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1196 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1197
Jon Hall85794ff2015-07-08 14:12:30 -07001198 main.step( "Killing ONOS processes" )
Jon Halle1a3b752015-07-22 13:02:46 -07001199 killResult = main.ONOSbench.onosKill( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001200 start = time.time()
1201 utilities.assert_equals( expect=main.TRUE, actual=killResult,
1202 onpass="ONOS Killed",
1203 onfail="Error killing ONOS" )
1204
1205 main.step( "Checking if ONOS is up yet" )
1206 count = 0
1207 while count < 10:
Jon Halle1a3b752015-07-22 13:02:46 -07001208 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001209 if onos1Isup == main.TRUE:
1210 elapsed = time.time() - start
1211 break
1212 else:
1213 count = count + 1
1214 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
1215 onpass="ONOS is back up",
1216 onfail="ONOS failed to start" )
1217
1218 main.log.step( "Starting ONOS CLI sessions" )
Jon Halle1a3b752015-07-22 13:02:46 -07001219 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001220 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1221 onpass="ONOS cli startup successful",
1222 onfail="ONOS cli startup failed" )
1223
1224 if elapsed:
1225 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
1226 str( elapsed ) )
1227 main.restartTime = elapsed
1228 else:
1229 main.restartTime = -1
1230 time.sleep( 5 )
1231 # rerun on election apps
1232 main.ONOScli1.electionTestRun()
1233
1234 def CASE7( self, main ):
1235 """
1236 Check state after ONOS failure
1237 """
1238 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001239 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001240 assert main, "main not defined"
1241 assert utilities.assert_equals, "utilities.assert_equals not defined"
1242 main.case( "Running ONOS Constant State Tests" )
1243 main.step( "Check that each switch has a master" )
1244 # Assert that each device has a master
1245 rolesNotNull = main.ONOScli1.rolesNotNull()
1246 utilities.assert_equals(
1247 expect=main.TRUE,
1248 actual=rolesNotNull,
1249 onpass="Each device has a master",
1250 onfail="Some devices don't have a master assigned" )
1251
1252 main.step( "Check if switch roles are consistent across all nodes" )
1253 ONOS1Mastership = main.ONOScli1.roles()
1254 # FIXME: Refactor this whole case for single instance
1255 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1256 main.log.error( "Error in getting ONOS mastership" )
1257 main.log.warn( "ONOS1 mastership response: " +
1258 repr( ONOS1Mastership ) )
1259 consistentMastership = main.FALSE
1260 else:
1261 consistentMastership = main.TRUE
1262 utilities.assert_equals(
1263 expect=main.TRUE,
1264 actual=consistentMastership,
1265 onpass="Switch roles are consistent across all ONOS nodes",
1266 onfail="ONOS nodes have different views of switch roles" )
1267
1268 description2 = "Compare switch roles from before failure"
1269 main.step( description2 )
1270
1271 currentJson = json.loads( ONOS1Mastership )
1272 oldJson = json.loads( mastershipState )
1273 mastershipCheck = main.TRUE
1274 for i in range( 1, 29 ):
1275 switchDPID = str(
1276 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1277
1278 current = [ switch[ 'master' ] for switch in currentJson
1279 if switchDPID in switch[ 'id' ] ]
1280 old = [ switch[ 'master' ] for switch in oldJson
1281 if switchDPID in switch[ 'id' ] ]
1282 if current == old:
1283 mastershipCheck = mastershipCheck and main.TRUE
1284 else:
1285 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1286 mastershipCheck = main.FALSE
1287 utilities.assert_equals(
1288 expect=main.TRUE,
1289 actual=mastershipCheck,
1290 onpass="Mastership of Switches was not changed",
1291 onfail="Mastership of some switches changed" )
1292 mastershipCheck = mastershipCheck and consistentMastership
1293
1294 main.step( "Get the intents and compare across all nodes" )
1295 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1296 intentCheck = main.FALSE
1297 if "Error" in ONOS1Intents or not ONOS1Intents:
1298 main.log.error( "Error in getting ONOS intents" )
1299 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1300 else:
1301 intentCheck = main.TRUE
1302 utilities.assert_equals(
1303 expect=main.TRUE,
1304 actual=intentCheck,
1305 onpass="Intents are consistent across all ONOS nodes",
1306 onfail="ONOS nodes have different views of intents" )
1307 # Print the intent states
1308 intents = []
1309 intents.append( ONOS1Intents )
1310 intentStates = []
1311 for node in intents: # Iter through ONOS nodes
1312 nodeStates = []
1313 # Iter through intents of a node
1314 for intent in json.loads( node ):
1315 nodeStates.append( intent[ 'state' ] )
1316 intentStates.append( nodeStates )
1317 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1318 main.log.info( dict( out ) )
1319
1320 # NOTE: Store has no durability, so intents are lost across system
1321 # restarts
1322 """
1323 main.step( "Compare current intents with intents before the failure" )
1324 # NOTE: this requires case 5 to pass for intentState to be set.
1325 # maybe we should stop the test if that fails?
1326 sameIntents = main.FALSE
1327 if intentState and intentState == ONOSIntents[ 0 ]:
1328 sameIntents = main.TRUE
1329 main.log.info( "Intents are consistent with before failure" )
1330 # TODO: possibly the states have changed? we may need to figure out
1331 # what the acceptable states are
1332 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1333 sameIntents = main.TRUE
1334 try:
1335 before = json.loads( intentState )
1336 after = json.loads( ONOSIntents[ 0 ] )
1337 for intent in before:
1338 if intent not in after:
1339 sameIntents = main.FALSE
1340 main.log.debug( "Intent is not currently in ONOS " +
1341 "(at least in the same form):" )
1342 main.log.debug( json.dumps( intent ) )
1343 except ( ValueError, TypeError ):
1344 main.log.exception( "Exception printing intents" )
1345 main.log.debug( repr( ONOSIntents[0] ) )
1346 main.log.debug( repr( intentState ) )
1347 if sameIntents == main.FALSE:
1348 try:
1349 main.log.debug( "ONOS intents before: " )
1350 main.log.debug( json.dumps( json.loads( intentState ),
1351 sort_keys=True, indent=4,
1352 separators=( ',', ': ' ) ) )
1353 main.log.debug( "Current ONOS intents: " )
1354 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1355 sort_keys=True, indent=4,
1356 separators=( ',', ': ' ) ) )
1357 except ( ValueError, TypeError ):
1358 main.log.exception( "Exception printing intents" )
1359 main.log.debug( repr( ONOSIntents[0] ) )
1360 main.log.debug( repr( intentState ) )
1361 utilities.assert_equals(
1362 expect=main.TRUE,
1363 actual=sameIntents,
1364 onpass="Intents are consistent with before failure",
1365 onfail="The Intents changed during failure" )
1366 intentCheck = intentCheck and sameIntents
1367 """
1368 main.step( "Get the OF Table entries and compare to before " +
1369 "component failure" )
1370 FlowTables = main.TRUE
1371 flows2 = []
1372 for i in range( 28 ):
1373 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall9043c902015-07-30 14:23:44 -07001374 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001375 flows2.append( tmpFlows )
Jon Hall9043c902015-07-30 14:23:44 -07001376 tempResult = main.Mininet1.flowComp(
Jon Hall85794ff2015-07-08 14:12:30 -07001377 flow1=flows[ i ],
1378 flow2=tmpFlows )
1379 FlowTables = FlowTables and tempResult
1380 if FlowTables == main.FALSE:
1381 main.log.info( "Differences in flow table for switch: s" +
1382 str( i + 1 ) )
1383 utilities.assert_equals(
1384 expect=main.TRUE,
1385 actual=FlowTables,
1386 onpass="No changes were found in the flow tables",
1387 onfail="Changes were found in the flow tables" )
1388
1389 main.step( "Leadership Election is still functional" )
1390 # Test of LeadershipElection
1391
Jon Halle1a3b752015-07-22 13:02:46 -07001392 leader = main.nodes[0].ip_address
Jon Hall85794ff2015-07-08 14:12:30 -07001393 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001394 for controller in range( 1, main.numCtrls + 1 ):
Jon Hall85794ff2015-07-08 14:12:30 -07001395 # loop through ONOScli handlers
1396 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
1397 leaderN = node.electionTestLeader()
1398 # verify leader is ONOS1
1399 # NOTE even though we restarted ONOS, it is the only one so onos 1
1400 # must be leader
1401 if leaderN == leader:
1402 # all is well
1403 pass
1404 elif leaderN == main.FALSE:
1405 # error in response
1406 main.log.error( "Something is wrong with " +
1407 "electionTestLeader function, check the" +
1408 " error logs" )
1409 leaderResult = main.FALSE
1410 elif leader != leaderN:
1411 leaderResult = main.FALSE
1412 main.log.error( "ONOS" + str( controller ) + " sees " +
1413 str( leaderN ) +
1414 " as the leader of the election app. " +
1415 "Leader should be " + str( leader ) )
1416 utilities.assert_equals(
1417 expect=main.TRUE,
1418 actual=leaderResult,
1419 onpass="Leadership election passed",
1420 onfail="Something went wrong with Leadership election" )
1421
1422 def CASE8( self, main ):
1423 """
1424 Compare topo
1425 """
1426 import json
1427 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001428 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001429 assert main, "main not defined"
1430 assert utilities.assert_equals, "utilities.assert_equals not defined"
1431
1432 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07001433 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall85794ff2015-07-08 14:12:30 -07001434 " and ONOS"
1435
1436 main.step( "Comparing ONOS topology to MN" )
Jon Hall85794ff2015-07-08 14:12:30 -07001437 topoResult = main.FALSE
1438 elapsed = 0
1439 count = 0
1440 main.step( "Collecting topology information from ONOS" )
1441 startTime = time.time()
1442 # Give time for Gossip to work
1443 while topoResult == main.FALSE and elapsed < 60:
Jon Hall96091e62015-09-21 17:34:17 -07001444 devicesResults = main.TRUE
1445 linksResults = main.TRUE
1446 hostsResults = main.TRUE
1447 hostAttachmentResults = True
Jon Hall85794ff2015-07-08 14:12:30 -07001448 count += 1
1449 cliStart = time.time()
1450 devices = []
1451 devices.append( main.ONOScli1.devices() )
1452 hosts = []
1453 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1454 ipResult = main.TRUE
1455 for controller in range( 0, len( hosts ) ):
1456 controllerStr = str( controller + 1 )
1457 for host in hosts[ controller ]:
1458 if host is None or host.get( 'ipAddresses', [] ) == []:
1459 main.log.error(
1460 "DEBUG:Error with host ips on controller" +
1461 controllerStr + ": " + str( host ) )
1462 ipResult = main.FALSE
1463 ports = []
1464 ports.append( main.ONOScli1.ports() )
1465 links = []
1466 links.append( main.ONOScli1.links() )
1467 clusters = []
1468 clusters.append( main.ONOScli1.clusters() )
1469
1470 elapsed = time.time() - startTime
1471 cliTime = time.time() - cliStart
1472 print "CLI time: " + str( cliTime )
1473
1474 mnSwitches = main.Mininet1.getSwitches()
1475 mnLinks = main.Mininet1.getLinks()
1476 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001477 for controller in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07001478 controllerStr = str( controller + 1 )
1479 if devices[ controller ] and ports[ controller ] and\
1480 "Error" not in devices[ controller ] and\
1481 "Error" not in ports[ controller ]:
1482
1483 currentDevicesResult = main.Mininet1.compareSwitches(
1484 mnSwitches,
1485 json.loads( devices[ controller ] ),
1486 json.loads( ports[ controller ] ) )
1487 else:
1488 currentDevicesResult = main.FALSE
1489 utilities.assert_equals( expect=main.TRUE,
1490 actual=currentDevicesResult,
1491 onpass="ONOS" + controllerStr +
1492 " Switches view is correct",
1493 onfail="ONOS" + controllerStr +
1494 " Switches view is incorrect" )
1495
1496 if links[ controller ] and "Error" not in links[ controller ]:
1497 currentLinksResult = main.Mininet1.compareLinks(
1498 mnSwitches, mnLinks,
1499 json.loads( links[ controller ] ) )
1500 else:
1501 currentLinksResult = main.FALSE
1502 utilities.assert_equals( expect=main.TRUE,
1503 actual=currentLinksResult,
1504 onpass="ONOS" + controllerStr +
1505 " links view is correct",
1506 onfail="ONOS" + controllerStr +
1507 " links view is incorrect" )
1508
1509 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1510 currentHostsResult = main.Mininet1.compareHosts(
1511 mnHosts,
1512 hosts[ controller ] )
1513 else:
1514 currentHostsResult = main.FALSE
1515 utilities.assert_equals( expect=main.TRUE,
1516 actual=currentHostsResult,
1517 onpass="ONOS" + controllerStr +
1518 " hosts exist in Mininet",
1519 onfail="ONOS" + controllerStr +
1520 " hosts don't match Mininet" )
1521 # CHECKING HOST ATTACHMENT POINTS
1522 hostAttachment = True
1523 zeroHosts = False
1524 # FIXME: topo-HA/obelisk specific mappings:
1525 # key is mac and value is dpid
1526 mappings = {}
1527 for i in range( 1, 29 ): # hosts 1 through 28
1528 # set up correct variables:
1529 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
1530 if i == 1:
1531 deviceId = "1000".zfill(16)
1532 elif i == 2:
1533 deviceId = "2000".zfill(16)
1534 elif i == 3:
1535 deviceId = "3000".zfill(16)
1536 elif i == 4:
1537 deviceId = "3004".zfill(16)
1538 elif i == 5:
1539 deviceId = "5000".zfill(16)
1540 elif i == 6:
1541 deviceId = "6000".zfill(16)
1542 elif i == 7:
1543 deviceId = "6007".zfill(16)
1544 elif i >= 8 and i <= 17:
1545 dpid = '3' + str( i ).zfill( 3 )
1546 deviceId = dpid.zfill(16)
1547 elif i >= 18 and i <= 27:
1548 dpid = '6' + str( i ).zfill( 3 )
1549 deviceId = dpid.zfill(16)
1550 elif i == 28:
1551 deviceId = "2800".zfill(16)
1552 mappings[ macId ] = deviceId
1553 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1554 if hosts[ controller ] == []:
1555 main.log.warn( "There are no hosts discovered" )
1556 zeroHosts = True
1557 else:
1558 for host in hosts[ controller ]:
1559 mac = None
1560 location = None
1561 device = None
1562 port = None
1563 try:
1564 mac = host.get( 'mac' )
1565 assert mac, "mac field could not be found for this host object"
1566
1567 location = host.get( 'location' )
1568 assert location, "location field could not be found for this host object"
1569
1570 # Trim the protocol identifier off deviceId
1571 device = str( location.get( 'elementId' ) ).split(':')[1]
1572 assert device, "elementId field could not be found for this host location object"
1573
1574 port = location.get( 'port' )
1575 assert port, "port field could not be found for this host location object"
1576
1577 # Now check if this matches where they should be
1578 if mac and device and port:
1579 if str( port ) != "1":
1580 main.log.error( "The attachment port is incorrect for " +
1581 "host " + str( mac ) +
1582 ". Expected: 1 Actual: " + str( port) )
1583 hostAttachment = False
1584 if device != mappings[ str( mac ) ]:
1585 main.log.error( "The attachment device is incorrect for " +
1586 "host " + str( mac ) +
1587 ". Expected: " + mappings[ str( mac ) ] +
1588 " Actual: " + device )
1589 hostAttachment = False
1590 else:
1591 hostAttachment = False
1592 except AssertionError:
1593 main.log.exception( "Json object not as expected" )
1594 main.log.error( repr( host ) )
1595 hostAttachment = False
1596 else:
1597 main.log.error( "No hosts json output or \"Error\"" +
1598 " in output. hosts = " +
1599 repr( hosts[ controller ] ) )
1600 if zeroHosts is False:
1601 hostAttachment = True
1602
Jon Hall85794ff2015-07-08 14:12:30 -07001603 devicesResults = devicesResults and currentDevicesResult
1604 linksResults = linksResults and currentLinksResult
1605 hostsResults = hostsResults and currentHostsResult
1606 hostAttachmentResults = hostAttachmentResults and\
1607 hostAttachment
1608
1609 # "consistent" results don't make sense for single instance
1610 # there should always only be one cluster
1611 numClusters = len( json.loads( clusters[ 0 ] ) )
1612 clusterResults = main.FALSE
1613 if numClusters == 1:
1614 clusterResults = main.TRUE
1615 utilities.assert_equals(
1616 expect=1,
1617 actual=numClusters,
1618 onpass="ONOS shows 1 SCC",
1619 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1620
1621 topoResult = ( devicesResults and linksResults
1622 and hostsResults and ipResult and clusterResults and
1623 hostAttachmentResults )
1624
1625 topoResult = topoResult and int( count <= 2 )
1626 note = "note it takes about " + str( int( cliTime ) ) + \
1627 " seconds for the test to make all the cli calls to fetch " +\
1628 "the topology from each ONOS instance"
1629 main.log.info(
1630 "Very crass estimate for topology discovery/convergence( " +
1631 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1632 str( count ) + " tries" )
1633 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1634 onpass="Topology Check Test successful",
1635 onfail="Topology Check Test NOT successful" )
1636
1637 def CASE9( self, main ):
1638 """
1639 Link s3-s28 down
1640 """
1641 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001642 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001643 assert main, "main not defined"
1644 assert utilities.assert_equals, "utilities.assert_equals not defined"
1645 # NOTE: You should probably run a topology check after this
1646
1647 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1648
1649 description = "Turn off a link to ensure that Link Discovery " +\
1650 "is working properly"
1651 main.case( description )
1652
1653 main.step( "Kill Link between s3 and s28" )
1654 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
1655 main.log.info( "Waiting " + str( linkSleep ) +
1656 " seconds for link down to be discovered" )
1657 time.sleep( linkSleep )
1658 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
1659 onpass="Link down successful",
1660 onfail="Failed to bring link down" )
1661 # TODO do some sort of check here
1662
1663 def CASE10( self, main ):
1664 """
1665 Link s3-s28 up
1666 """
1667 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001668 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001669 assert main, "main not defined"
1670 assert utilities.assert_equals, "utilities.assert_equals not defined"
1671 # NOTE: You should probably run a topology check after this
1672
1673 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1674
1675 description = "Restore a link to ensure that Link Discovery is " + \
1676 "working properly"
1677 main.case( description )
1678
1679 main.step( "Bring link between s3 and s28 back up" )
1680 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
1681 main.log.info( "Waiting " + str( linkSleep ) +
1682 " seconds for link up to be discovered" )
1683 time.sleep( linkSleep )
1684 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
1685 onpass="Link up successful",
1686 onfail="Failed to bring link up" )
1687 # TODO do some sort of check here
1688
1689 def CASE11( self, main ):
1690 """
1691 Switch Down
1692 """
1693 # NOTE: You should probably run a topology check after this
1694 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001695 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001696 assert main, "main not defined"
1697 assert utilities.assert_equals, "utilities.assert_equals not defined"
1698
1699 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1700
1701 description = "Killing a switch to ensure it is discovered correctly"
1702 main.case( description )
1703 switch = main.params[ 'kill' ][ 'switch' ]
1704 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1705
1706 # TODO: Make this switch parameterizable
1707 main.step( "Kill " + switch )
1708 main.log.info( "Deleting " + switch )
1709 main.Mininet1.delSwitch( switch )
1710 main.log.info( "Waiting " + str( switchSleep ) +
1711 " seconds for switch down to be discovered" )
1712 time.sleep( switchSleep )
1713 device = main.ONOScli1.getDevice( dpid=switchDPID )
1714 # Peek at the deleted switch
1715 main.log.warn( str( device ) )
1716 result = main.FALSE
1717 if device and device[ 'available' ] is False:
1718 result = main.TRUE
1719 utilities.assert_equals( expect=main.TRUE, actual=result,
1720 onpass="Kill switch successful",
1721 onfail="Failed to kill switch?" )
1722
1723 def CASE12( self, main ):
1724 """
1725 Switch Up
1726 """
1727 # NOTE: You should probably run a topology check after this
1728 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001729 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001730 assert main, "main not defined"
1731 assert utilities.assert_equals, "utilities.assert_equals not defined"
1732
1733 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1734 switch = main.params[ 'kill' ][ 'switch' ]
1735 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1736 links = main.params[ 'kill' ][ 'links' ].split()
1737 description = "Adding a switch to ensure it is discovered correctly"
1738 main.case( description )
1739
1740 main.step( "Add back " + switch )
1741 main.Mininet1.addSwitch( switch, dpid=switchDPID )
1742 for peer in links:
1743 main.Mininet1.addLink( switch, peer )
1744 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07001745 for i in range( main.numCtrls ):
1746 ipList.append( main.nodes[ i ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001747 main.Mininet1.assignSwController( sw=switch, ip=ipList )
1748 main.log.info( "Waiting " + str( switchSleep ) +
1749 " seconds for switch up to be discovered" )
1750 time.sleep( switchSleep )
1751 device = main.ONOScli1.getDevice( dpid=switchDPID )
1752 # Peek at the deleted switch
1753 main.log.warn( str( device ) )
1754 result = main.FALSE
1755 if device and device[ 'available' ]:
1756 result = main.TRUE
1757 utilities.assert_equals( expect=main.TRUE, actual=result,
1758 onpass="add switch successful",
1759 onfail="Failed to add switch?" )
1760
1761 def CASE13( self, main ):
1762 """
1763 Clean up
1764 """
1765 import os
1766 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001767 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001768 assert main, "main not defined"
1769 assert utilities.assert_equals, "utilities.assert_equals not defined"
1770 # printing colors to terminal
1771 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
1772 'blue': '\033[94m', 'green': '\033[92m',
1773 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
1774 main.case( "Test Cleanup" )
1775 main.step( "Killing tcpdumps" )
1776 main.Mininet2.stopTcpdump()
1777
1778 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07001779 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall85794ff2015-07-08 14:12:30 -07001780 main.step( "Copying MN pcap and ONOS log files to test station" )
1781 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
1782 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07001783 # NOTE: MN Pcap file is being saved to logdir.
1784 # We scp this file as MN and TestON aren't necessarily the same vm
1785
1786 # FIXME: To be replaced with a Jenkin's post script
Jon Hall85794ff2015-07-08 14:12:30 -07001787 # TODO: Load these from params
1788 # NOTE: must end in /
1789 logFolder = "/opt/onos/log/"
1790 logFiles = [ "karaf.log", "karaf.log.1" ]
1791 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07001792 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07001793 for node in main.nodes:
1794 dstName = main.logdir + "/" + node.name + "-" + f
1795 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
1796 logFolder + f, dstName )
Jon Hall85794ff2015-07-08 14:12:30 -07001797 # std*.log's
1798 # NOTE: must end in /
1799 logFolder = "/opt/onos/var/"
1800 logFiles = [ "stderr.log", "stdout.log" ]
1801 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07001802 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07001803 for node in main.nodes:
1804 dstName = main.logdir + "/" + node.name + "-" + f
1805 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
1806 logFolder + f, dstName )
1807 else:
1808 main.log.debug( "skipping saving log files" )
Jon Hall85794ff2015-07-08 14:12:30 -07001809
Jon Hall85794ff2015-07-08 14:12:30 -07001810 main.step( "Stopping Mininet" )
1811 mnResult = main.Mininet1.stopNet()
1812 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
1813 onpass="Mininet stopped",
1814 onfail="MN cleanup NOT successful" )
1815
1816 main.step( "Checking ONOS Logs for errors" )
Jon Hall96091e62015-09-21 17:34:17 -07001817 for node in main.nodes:
1818 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1819 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001820
1821 try:
1822 timerLog = open( main.logdir + "/Timers.csv", 'w')
1823 # Overwrite with empty line and close
1824 labels = "Gossip Intents, Restart"
1825 data = str( gossipTime ) + ", " + str( main.restartTime )
1826 timerLog.write( labels + "\n" + data )
1827 timerLog.close()
1828 except NameError, e:
1829 main.log.exception(e)
1830
1831 def CASE14( self, main ):
1832 """
1833 start election app on all onos nodes
1834 """
Jon Halle1a3b752015-07-22 13:02:46 -07001835 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001836 assert main, "main not defined"
1837 assert utilities.assert_equals, "utilities.assert_equals not defined"
1838
1839 main.case("Start Leadership Election app")
1840 main.step( "Install leadership election app" )
1841 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
1842 utilities.assert_equals(
1843 expect=main.TRUE,
1844 actual=appResult,
1845 onpass="Election app installed",
1846 onfail="Something went wrong with installing Leadership election" )
1847
1848 main.step( "Run for election on each node" )
1849 leaderResult = main.ONOScli1.electionTestRun()
1850 # check for leader
1851 leader = main.ONOScli1.electionTestLeader()
1852 # verify leader is ONOS1
Jon Halle1a3b752015-07-22 13:02:46 -07001853 if leader == main.nodes[0].ip_address:
Jon Hall85794ff2015-07-08 14:12:30 -07001854 # all is well
1855 pass
1856 elif leader is None:
1857 # No leader elected
1858 main.log.error( "No leader was elected" )
1859 leaderResult = main.FALSE
1860 elif leader == main.FALSE:
1861 # error in response
1862 # TODO: add check for "Command not found:" in the driver, this
1863 # means the app isn't loaded
1864 main.log.error( "Something is wrong with electionTestLeader" +
1865 " function, check the error logs" )
1866 leaderResult = main.FALSE
1867 else:
1868 # error in response
1869 main.log.error(
1870 "Unexpected response from electionTestLeader function:'" +
1871 str( leader ) +
1872 "'" )
1873 leaderResult = main.FALSE
1874 utilities.assert_equals(
1875 expect=main.TRUE,
1876 actual=leaderResult,
1877 onpass="Successfully ran for leadership",
1878 onfail="Failed to run for leadership" )
1879
1880 def CASE15( self, main ):
1881 """
1882 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07001883 15.1 Run election on each node
1884 15.2 Check that each node has the same leaders and candidates
1885 15.3 Find current leader and withdraw
1886 15.4 Check that a new node was elected leader
1887 15.5 Check that that new leader was the candidate of old leader
1888 15.6 Run for election on old leader
1889 15.7 Check that oldLeader is a candidate, and leader if only 1 node
1890 15.8 Make sure that the old leader was added to the candidate list
1891
1892 old and new variable prefixes refer to data from before vs after
1893 withdrawl and later before withdrawl vs after re-election
Jon Hall85794ff2015-07-08 14:12:30 -07001894 """
acsmars71adceb2015-08-31 15:09:26 -07001895 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001896 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001897 assert main, "main not defined"
1898 assert utilities.assert_equals, "utilities.assert_equals not defined"
acsmars71adceb2015-08-31 15:09:26 -07001899 assert main.CLIs, "main.CLIs not defined"
1900 assert main.nodes, "main.nodes not defined"
1901
Jon Hall85794ff2015-07-08 14:12:30 -07001902 description = "Check that Leadership Election is still functional"
1903 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07001904 # NOTE: Need to re-run since being a canidate is not persistant
1905 # TODO: add check for "Command not found:" in the driver, this
1906 # means the election test app isn't loaded
acsmars2c2fcdd2015-08-25 17:14:13 -07001907
acsmars71adceb2015-08-31 15:09:26 -07001908 oldLeaders = [] # leaders by node before withdrawl from candidates
1909 newLeaders = [] # leaders by node after withdrawl from candidates
1910 oldAllCandidates = [] # list of lists of each nodes' candidates before
1911 newAllCandidates = [] # list of lists of each nodes' candidates after
1912 oldCandidates = [] # list of candidates from node 0 before withdrawl
1913 newCandidates = [] # list of candidates from node 0 after withdrawl
1914 oldLeader = '' # the old leader from oldLeaders, None if not same
1915 newLeader = '' # the new leaders fron newLoeaders, None if not same
1916 oldLeaderCLI = None # the CLI of the old leader used for re-electing
1917 expectNoLeader = False # True when there is only one leader
1918 if main.numCtrls == 1:
1919 expectNoLeader = True
acsmars2c2fcdd2015-08-25 17:14:13 -07001920
acsmars71adceb2015-08-31 15:09:26 -07001921 main.step( "Run for election on each node" )
1922 electionResult = main.TRUE
1923
1924 for cli in main.CLIs: # run test election on each node
1925 if cli.electionTestRun() == main.FALSE:
1926 electionResult = main.FALSE
1927
1928 utilities.assert_equals(
1929 expect=main.TRUE,
1930 actual=electionResult,
1931 onpass="All nodes successfully ran for leadership",
1932 onfail="At least one node failed to run for leadership" )
1933
acsmars3a72bde2015-09-02 14:16:22 -07001934 if electionResult == main.FALSE:
1935 main.log.error(
1936 "Skipping Test Case because Election Test App isn't loaded" )
1937 main.skipCase()
1938
acsmars71adceb2015-08-31 15:09:26 -07001939 main.step( "Check that each node shows the same leader and candidates" )
1940 sameResult = main.TRUE
1941 failMessage = "Nodes have different leaders"
1942 for cli in main.CLIs:
1943 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
1944 oldAllCandidates.append( node )
1945 oldLeaders.append( node[ 0 ] )
1946 oldCandidates = oldAllCandidates[ 0 ]
1947
1948 # Check that each node has the same leader. Defines oldLeader
1949 if len( set( oldLeaders ) ) != 1:
1950 sameResult = main.FALSE
1951 main.log.error( "More than one leader present:" + str( oldLeaders ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001952 oldLeader = None
1953 else:
acsmars71adceb2015-08-31 15:09:26 -07001954 oldLeader = oldLeaders[ 0 ]
1955
1956 # Check that each node's candidate list is the same
acsmars29233db2015-11-04 11:15:00 -08001957 candidateDiscrepancy = False # Boolean of candidate mismatches
acsmars71adceb2015-08-31 15:09:26 -07001958 for candidates in oldAllCandidates:
1959 if set( candidates ) != set( oldCandidates ):
1960 sameResult = main.FALSE
acsmars29233db2015-11-04 11:15:00 -08001961 candidateDiscrepancy = True
1962
1963 if candidateDiscrepancy:
1964 failMessage += " and candidates"
acsmars71adceb2015-08-31 15:09:26 -07001965
1966 utilities.assert_equals(
1967 expect=main.TRUE,
1968 actual=sameResult,
1969 onpass="Leadership is consistent for the election topic",
1970 onfail=failMessage )
1971
1972 main.step( "Find current leader and withdraw" )
1973 withdrawResult = main.TRUE
1974 # do some sanity checking on leader before using it
1975 if oldLeader is None:
1976 main.log.error( "Leadership isn't consistent." )
1977 withdrawResult = main.FALSE
1978 # Get the CLI of the oldLeader
1979 for i in range( len( main.CLIs ) ):
1980 if oldLeader == main.nodes[ i ].ip_address:
1981 oldLeaderCLI = main.CLIs[ i ]
1982 break
1983 else: # FOR/ELSE statement
1984 main.log.error( "Leader election, could not find current leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07001985 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07001986 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall85794ff2015-07-08 14:12:30 -07001987 utilities.assert_equals(
1988 expect=main.TRUE,
1989 actual=withdrawResult,
1990 onpass="Node was withdrawn from election",
1991 onfail="Node was not withdrawn from election" )
1992
acsmars71adceb2015-08-31 15:09:26 -07001993 main.step( "Check that a new node was elected leader" )
1994
1995 # FIXME: use threads
1996 newLeaderResult = main.TRUE
1997 failMessage = "Nodes have different leaders"
1998
1999 # Get new leaders and candidates
2000 for cli in main.CLIs:
2001 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2002 # elections might no have finished yet
2003 if node[ 0 ] == 'none' and not expectNoLeader:
2004 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2005 "sure elections are complete." )
2006 time.sleep(5)
2007 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2008 # election still isn't done or there is a problem
2009 if node[ 0 ] == 'none':
2010 main.log.error( "No leader was elected on at least 1 node" )
2011 newLeaderResult = main.FALSE
2012 newAllCandidates.append( node )
2013 newLeaders.append( node[ 0 ] )
2014 newCandidates = newAllCandidates[ 0 ]
2015
2016 # Check that each node has the same leader. Defines newLeader
2017 if len( set( newLeaders ) ) != 1:
2018 newLeaderResult = main.FALSE
2019 main.log.error( "Nodes have different leaders: " +
2020 str( newLeaders ) )
2021 newLeader = None
2022 else:
2023 newLeader = newLeaders[ 0 ]
2024
2025 # Check that each node's candidate list is the same
2026 for candidates in newAllCandidates:
2027 if set( candidates ) != set( newCandidates ):
2028 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002029 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002030
2031 # Check that the new leader is not the older leader, which was withdrawn
2032 if newLeader == oldLeader:
2033 newLeaderResult = main.FALSE
2034 main.log.error( "All nodes still see old leader: " + oldLeader +
2035 " as the current leader" )
2036
Jon Hall85794ff2015-07-08 14:12:30 -07002037 utilities.assert_equals(
2038 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002039 actual=newLeaderResult,
2040 onpass="Leadership election passed",
2041 onfail="Something went wrong with Leadership election" )
Jon Hall85794ff2015-07-08 14:12:30 -07002042
acsmars71adceb2015-08-31 15:09:26 -07002043 main.step( "Check that that new leader was the candidate of old leader")
2044 # candidates[ 2 ] should be come the top candidate after withdrawl
2045 correctCandidateResult = main.TRUE
2046 if expectNoLeader:
2047 if newLeader == 'none':
2048 main.log.info( "No leader expected. None found. Pass" )
2049 correctCandidateResult = main.TRUE
2050 else:
2051 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2052 correctCandidateResult = main.FALSE
2053 elif newLeader != oldCandidates[ 2 ]:
2054 correctCandidateResult = main.FALSE
2055 main.log.error( "Candidate " + newLeader + " was elected. " +
2056 oldCandidates[ 2 ] + " should have had priority." )
2057
2058 utilities.assert_equals(
2059 expect=main.TRUE,
2060 actual=correctCandidateResult,
2061 onpass="Correct Candidate Elected",
2062 onfail="Incorrect Candidate Elected" )
2063
Jon Hall85794ff2015-07-08 14:12:30 -07002064 main.step( "Run for election on old leader( just so everyone " +
2065 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002066 if oldLeaderCLI is not None:
2067 runResult = oldLeaderCLI.electionTestRun()
Jon Hall85794ff2015-07-08 14:12:30 -07002068 else:
acsmars71adceb2015-08-31 15:09:26 -07002069 main.log.error( "No old leader to re-elect" )
Jon Hall85794ff2015-07-08 14:12:30 -07002070 runResult = main.FALSE
2071 utilities.assert_equals(
2072 expect=main.TRUE,
2073 actual=runResult,
2074 onpass="App re-ran for election",
2075 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002076 main.step(
2077 "Check that oldLeader is a candidate, and leader if only 1 node" )
2078 # verify leader didn't just change
2079 positionResult = main.TRUE
2080 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
Jon Hall85794ff2015-07-08 14:12:30 -07002081
acsmars71adceb2015-08-31 15:09:26 -07002082 # Reset and reuse the new candidate and leaders lists
2083 newAllCandidates = []
2084 newCandidates = []
2085 newLeaders = []
2086 for cli in main.CLIs:
2087 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2088 if oldLeader not in node: # election might no have finished yet
2089 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2090 "be sure elections are complete" )
2091 time.sleep(5)
2092 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2093 if oldLeader not in node: # election still isn't done, errors
2094 main.log.error(
2095 "Old leader was not elected on at least one node" )
2096 positionResult = main.FALSE
2097 newAllCandidates.append( node )
2098 newLeaders.append( node[ 0 ] )
2099 newCandidates = newAllCandidates[ 0 ]
2100
2101 # Check that each node has the same leader. Defines newLeader
2102 if len( set( newLeaders ) ) != 1:
2103 positionResult = main.FALSE
2104 main.log.error( "Nodes have different leaders: " +
2105 str( newLeaders ) )
2106 newLeader = None
Jon Hall85794ff2015-07-08 14:12:30 -07002107 else:
acsmars71adceb2015-08-31 15:09:26 -07002108 newLeader = newLeaders[ 0 ]
2109
2110 # Check that each node's candidate list is the same
2111 for candidates in newAllCandidates:
2112 if set( candidates ) != set( newCandidates ):
2113 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002114 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002115
2116 # Check that the re-elected node is last on the candidate List
2117 if oldLeader != newCandidates[ -1 ]:
2118 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
2119 str( newCandidates ) )
2120 positionResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07002121
2122 utilities.assert_equals(
2123 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002124 actual=positionResult,
Jon Hall85794ff2015-07-08 14:12:30 -07002125 onpass="Old leader successfully re-ran for election",
2126 onfail="Something went wrong with Leadership election after " +
2127 "the old leader re-ran for election" )
2128
2129 def CASE16( self, main ):
2130 """
2131 Install Distributed Primitives app
2132 """
Jon Halle1a3b752015-07-22 13:02:46 -07002133 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002134 assert main, "main not defined"
2135 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002136 assert main.CLIs, "main.CLIs not defined"
2137 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002138
2139 # Variables for the distributed primitives tests
2140 global pCounterName
2141 global iCounterName
2142 global pCounterValue
2143 global iCounterValue
2144 global onosSet
2145 global onosSetName
2146 pCounterName = "TestON-Partitions"
2147 iCounterName = "TestON-inMemory"
2148 pCounterValue = 0
2149 iCounterValue = 0
2150 onosSet = set([])
2151 onosSetName = "TestON-set"
2152
2153 description = "Install Primitives app"
2154 main.case( description )
2155 main.step( "Install Primitives app" )
2156 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002157 appResults = main.CLIs[0].activateApp( appName )
Jon Hall85794ff2015-07-08 14:12:30 -07002158 utilities.assert_equals( expect=main.TRUE,
2159 actual=appResults,
2160 onpass="Primitives app activated",
2161 onfail="Primitives app not activated" )
2162
2163 def CASE17( self, main ):
2164 """
2165 Check for basic functionality with distributed primitives
2166 """
Jon Hall85794ff2015-07-08 14:12:30 -07002167 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002168 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002169 assert main, "main not defined"
2170 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002171 assert main.CLIs, "main.CLIs not defined"
2172 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002173 assert pCounterName, "pCounterName not defined"
2174 assert iCounterName, "iCounterName not defined"
2175 assert onosSetName, "onosSetName not defined"
2176 # NOTE: assert fails if value is 0/None/Empty/False
2177 try:
2178 pCounterValue
2179 except NameError:
2180 main.log.error( "pCounterValue not defined, setting to 0" )
2181 pCounterValue = 0
2182 try:
2183 iCounterValue
2184 except NameError:
2185 main.log.error( "iCounterValue not defined, setting to 0" )
2186 iCounterValue = 0
2187 try:
2188 onosSet
2189 except NameError:
2190 main.log.error( "onosSet not defined, setting to empty Set" )
2191 onosSet = set([])
2192 # Variables for the distributed primitives tests. These are local only
2193 addValue = "a"
2194 addAllValue = "a b c d e f"
2195 retainValue = "c d e f"
2196
2197 description = "Check for basic functionality with distributed " +\
2198 "primitives"
2199 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07002200 main.caseExplanation = "Test the methods of the distributed " +\
2201 "primitives (counters and sets) throught the cli"
Jon Hall85794ff2015-07-08 14:12:30 -07002202 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07002203 # Partitioned counters
2204 main.step( "Increment then get a default counter on each node" )
Jon Hall85794ff2015-07-08 14:12:30 -07002205 pCounters = []
2206 threads = []
2207 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07002208 for i in range( main.numCtrls ):
2209 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2210 name="counterAddAndGet-" + str( i ),
Jon Hall85794ff2015-07-08 14:12:30 -07002211 args=[ pCounterName ] )
2212 pCounterValue += 1
2213 addedPValues.append( pCounterValue )
2214 threads.append( t )
2215 t.start()
2216
2217 for t in threads:
2218 t.join()
2219 pCounters.append( t.result )
2220 # Check that counter incremented numController times
2221 pCounterResults = True
2222 for i in addedPValues:
2223 tmpResult = i in pCounters
2224 pCounterResults = pCounterResults and tmpResult
2225 if not tmpResult:
2226 main.log.error( str( i ) + " is not in partitioned "
2227 "counter incremented results" )
2228 utilities.assert_equals( expect=True,
2229 actual=pCounterResults,
2230 onpass="Default counter incremented",
2231 onfail="Error incrementing default" +
2232 " counter" )
2233
Jon Halle1a3b752015-07-22 13:02:46 -07002234 main.step( "Get then Increment a default counter on each node" )
2235 pCounters = []
2236 threads = []
2237 addedPValues = []
2238 for i in range( main.numCtrls ):
2239 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2240 name="counterGetAndAdd-" + str( i ),
2241 args=[ pCounterName ] )
2242 addedPValues.append( pCounterValue )
2243 pCounterValue += 1
2244 threads.append( t )
2245 t.start()
2246
2247 for t in threads:
2248 t.join()
2249 pCounters.append( t.result )
2250 # Check that counter incremented numController times
2251 pCounterResults = True
2252 for i in addedPValues:
2253 tmpResult = i in pCounters
2254 pCounterResults = pCounterResults and tmpResult
2255 if not tmpResult:
2256 main.log.error( str( i ) + " is not in partitioned "
2257 "counter incremented results" )
2258 utilities.assert_equals( expect=True,
2259 actual=pCounterResults,
2260 onpass="Default counter incremented",
2261 onfail="Error incrementing default" +
2262 " counter" )
2263
2264 main.step( "Counters we added have the correct values" )
2265 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2266 utilities.assert_equals( expect=main.TRUE,
2267 actual=incrementCheck,
2268 onpass="Added counters are correct",
2269 onfail="Added counters are incorrect" )
2270
2271 main.step( "Add -8 to then get a default counter on each node" )
2272 pCounters = []
2273 threads = []
2274 addedPValues = []
2275 for i in range( main.numCtrls ):
2276 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2277 name="counterIncrement-" + str( i ),
2278 args=[ pCounterName ],
2279 kwargs={ "delta": -8 } )
2280 pCounterValue += -8
2281 addedPValues.append( pCounterValue )
2282 threads.append( t )
2283 t.start()
2284
2285 for t in threads:
2286 t.join()
2287 pCounters.append( t.result )
2288 # Check that counter incremented numController times
2289 pCounterResults = True
2290 for i in addedPValues:
2291 tmpResult = i in pCounters
2292 pCounterResults = pCounterResults and tmpResult
2293 if not tmpResult:
2294 main.log.error( str( i ) + " is not in partitioned "
2295 "counter incremented results" )
2296 utilities.assert_equals( expect=True,
2297 actual=pCounterResults,
2298 onpass="Default counter incremented",
2299 onfail="Error incrementing default" +
2300 " counter" )
2301
2302 main.step( "Add 5 to then get a default counter on each node" )
2303 pCounters = []
2304 threads = []
2305 addedPValues = []
2306 for i in range( main.numCtrls ):
2307 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2308 name="counterIncrement-" + str( i ),
2309 args=[ pCounterName ],
2310 kwargs={ "delta": 5 } )
2311 pCounterValue += 5
2312 addedPValues.append( pCounterValue )
2313 threads.append( t )
2314 t.start()
2315
2316 for t in threads:
2317 t.join()
2318 pCounters.append( t.result )
2319 # Check that counter incremented numController times
2320 pCounterResults = True
2321 for i in addedPValues:
2322 tmpResult = i in pCounters
2323 pCounterResults = pCounterResults and tmpResult
2324 if not tmpResult:
2325 main.log.error( str( i ) + " is not in partitioned "
2326 "counter incremented results" )
2327 utilities.assert_equals( expect=True,
2328 actual=pCounterResults,
2329 onpass="Default counter incremented",
2330 onfail="Error incrementing default" +
2331 " counter" )
2332
2333 main.step( "Get then add 5 to a default counter on each node" )
2334 pCounters = []
2335 threads = []
2336 addedPValues = []
2337 for i in range( main.numCtrls ):
2338 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2339 name="counterIncrement-" + str( i ),
2340 args=[ pCounterName ],
2341 kwargs={ "delta": 5 } )
2342 addedPValues.append( pCounterValue )
2343 pCounterValue += 5
2344 threads.append( t )
2345 t.start()
2346
2347 for t in threads:
2348 t.join()
2349 pCounters.append( t.result )
2350 # Check that counter incremented numController times
2351 pCounterResults = True
2352 for i in addedPValues:
2353 tmpResult = i in pCounters
2354 pCounterResults = pCounterResults and tmpResult
2355 if not tmpResult:
2356 main.log.error( str( i ) + " is not in partitioned "
2357 "counter incremented results" )
2358 utilities.assert_equals( expect=True,
2359 actual=pCounterResults,
2360 onpass="Default counter incremented",
2361 onfail="Error incrementing default" +
2362 " counter" )
2363
2364 main.step( "Counters we added have the correct values" )
2365 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2366 utilities.assert_equals( expect=main.TRUE,
2367 actual=incrementCheck,
2368 onpass="Added counters are correct",
2369 onfail="Added counters are incorrect" )
2370
2371 # In-Memory counters
2372 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall85794ff2015-07-08 14:12:30 -07002373 iCounters = []
2374 addedIValues = []
2375 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002376 for i in range( main.numCtrls ):
2377 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002378 name="icounterIncrement-" + str( i ),
2379 args=[ iCounterName ],
2380 kwargs={ "inMemory": True } )
2381 iCounterValue += 1
2382 addedIValues.append( iCounterValue )
2383 threads.append( t )
2384 t.start()
2385
2386 for t in threads:
2387 t.join()
2388 iCounters.append( t.result )
2389 # Check that counter incremented numController times
2390 iCounterResults = True
2391 for i in addedIValues:
2392 tmpResult = i in iCounters
2393 iCounterResults = iCounterResults and tmpResult
2394 if not tmpResult:
2395 main.log.error( str( i ) + " is not in the in-memory "
2396 "counter incremented results" )
2397 utilities.assert_equals( expect=True,
2398 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07002399 onpass="In-memory counter incremented",
2400 onfail="Error incrementing in-memory" +
Jon Hall85794ff2015-07-08 14:12:30 -07002401 " counter" )
2402
Jon Halle1a3b752015-07-22 13:02:46 -07002403 main.step( "Get then Increment a in-memory counter on each node" )
2404 iCounters = []
2405 threads = []
2406 addedIValues = []
2407 for i in range( main.numCtrls ):
2408 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2409 name="counterGetAndAdd-" + str( i ),
2410 args=[ iCounterName ],
2411 kwargs={ "inMemory": True } )
2412 addedIValues.append( iCounterValue )
2413 iCounterValue += 1
2414 threads.append( t )
2415 t.start()
2416
2417 for t in threads:
2418 t.join()
2419 iCounters.append( t.result )
2420 # Check that counter incremented numController times
2421 iCounterResults = True
2422 for i in addedIValues:
2423 tmpResult = i in iCounters
2424 iCounterResults = iCounterResults and tmpResult
2425 if not tmpResult:
2426 main.log.error( str( i ) + " is not in in-memory "
2427 "counter incremented results" )
2428 utilities.assert_equals( expect=True,
2429 actual=iCounterResults,
2430 onpass="In-memory counter incremented",
2431 onfail="Error incrementing in-memory" +
2432 " counter" )
2433
2434 main.step( "Counters we added have the correct values" )
2435 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2436 utilities.assert_equals( expect=main.TRUE,
2437 actual=incrementCheck,
2438 onpass="Added counters are correct",
2439 onfail="Added counters are incorrect" )
2440
2441 main.step( "Add -8 to then get a in-memory counter on each node" )
2442 iCounters = []
2443 threads = []
2444 addedIValues = []
2445 for i in range( main.numCtrls ):
2446 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2447 name="counterIncrement-" + str( i ),
2448 args=[ iCounterName ],
2449 kwargs={ "delta": -8, "inMemory": True } )
2450 iCounterValue += -8
2451 addedIValues.append( iCounterValue )
2452 threads.append( t )
2453 t.start()
2454
2455 for t in threads:
2456 t.join()
2457 iCounters.append( t.result )
2458 # Check that counter incremented numController times
2459 iCounterResults = True
2460 for i in addedIValues:
2461 tmpResult = i in iCounters
2462 iCounterResults = iCounterResults and tmpResult
2463 if not tmpResult:
2464 main.log.error( str( i ) + " is not in in-memory "
2465 "counter incremented results" )
2466 utilities.assert_equals( expect=True,
2467 actual=pCounterResults,
2468 onpass="In-memory counter incremented",
2469 onfail="Error incrementing in-memory" +
2470 " counter" )
2471
2472 main.step( "Add 5 to then get a in-memory counter on each node" )
2473 iCounters = []
2474 threads = []
2475 addedIValues = []
2476 for i in range( main.numCtrls ):
2477 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2478 name="counterIncrement-" + str( i ),
2479 args=[ iCounterName ],
2480 kwargs={ "delta": 5, "inMemory": True } )
2481 iCounterValue += 5
2482 addedIValues.append( iCounterValue )
2483 threads.append( t )
2484 t.start()
2485
2486 for t in threads:
2487 t.join()
2488 iCounters.append( t.result )
2489 # Check that counter incremented numController times
2490 iCounterResults = True
2491 for i in addedIValues:
2492 tmpResult = i in iCounters
2493 iCounterResults = iCounterResults and tmpResult
2494 if not tmpResult:
2495 main.log.error( str( i ) + " is not in in-memory "
2496 "counter incremented results" )
2497 utilities.assert_equals( expect=True,
2498 actual=pCounterResults,
2499 onpass="In-memory counter incremented",
2500 onfail="Error incrementing in-memory" +
2501 " counter" )
2502
2503 main.step( "Get then add 5 to a in-memory counter on each node" )
2504 iCounters = []
2505 threads = []
2506 addedIValues = []
2507 for i in range( main.numCtrls ):
2508 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2509 name="counterIncrement-" + str( i ),
2510 args=[ iCounterName ],
2511 kwargs={ "delta": 5, "inMemory": True } )
2512 addedIValues.append( iCounterValue )
2513 iCounterValue += 5
2514 threads.append( t )
2515 t.start()
2516
2517 for t in threads:
2518 t.join()
2519 iCounters.append( t.result )
2520 # Check that counter incremented numController times
2521 iCounterResults = True
2522 for i in addedIValues:
2523 tmpResult = i in iCounters
2524 iCounterResults = iCounterResults and tmpResult
2525 if not tmpResult:
2526 main.log.error( str( i ) + " is not in in-memory "
2527 "counter incremented results" )
2528 utilities.assert_equals( expect=True,
2529 actual=iCounterResults,
2530 onpass="In-memory counter incremented",
2531 onfail="Error incrementing in-memory" +
2532 " counter" )
2533
2534 main.step( "Counters we added have the correct values" )
2535 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2536 utilities.assert_equals( expect=main.TRUE,
2537 actual=incrementCheck,
2538 onpass="Added counters are correct",
2539 onfail="Added counters are incorrect" )
2540
Jon Hall85794ff2015-07-08 14:12:30 -07002541 main.step( "Check counters are consistant across nodes" )
Jon Hall57b50432015-10-22 10:20:10 -07002542 onosCounters, consistentCounterResults = main.Counters.consistentCheck()
Jon Hall85794ff2015-07-08 14:12:30 -07002543 utilities.assert_equals( expect=main.TRUE,
2544 actual=consistentCounterResults,
2545 onpass="ONOS counters are consistent " +
2546 "across nodes",
2547 onfail="ONOS Counters are inconsistent " +
2548 "across nodes" )
2549
2550 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07002551 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2552 incrementCheck = incrementCheck and \
2553 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall85794ff2015-07-08 14:12:30 -07002554 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07002555 actual=incrementCheck,
Jon Hall85794ff2015-07-08 14:12:30 -07002556 onpass="Added counters are correct",
2557 onfail="Added counters are incorrect" )
2558 # DISTRIBUTED SETS
2559 main.step( "Distributed Set get" )
2560 size = len( onosSet )
2561 getResponses = []
2562 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002563 for i in range( main.numCtrls ):
2564 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002565 name="setTestGet-" + str( i ),
2566 args=[ onosSetName ] )
2567 threads.append( t )
2568 t.start()
2569 for t in threads:
2570 t.join()
2571 getResponses.append( t.result )
2572
2573 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002574 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002575 if isinstance( getResponses[ i ], list):
2576 current = set( getResponses[ i ] )
2577 if len( current ) == len( getResponses[ i ] ):
2578 # no repeats
2579 if onosSet != current:
2580 main.log.error( "ONOS" + str( i + 1 ) +
2581 " has incorrect view" +
2582 " of set " + onosSetName + ":\n" +
2583 str( getResponses[ i ] ) )
2584 main.log.debug( "Expected: " + str( onosSet ) )
2585 main.log.debug( "Actual: " + str( current ) )
2586 getResults = main.FALSE
2587 else:
2588 # error, set is not a set
2589 main.log.error( "ONOS" + str( i + 1 ) +
2590 " has repeat elements in" +
2591 " set " + onosSetName + ":\n" +
2592 str( getResponses[ i ] ) )
2593 getResults = main.FALSE
2594 elif getResponses[ i ] == main.ERROR:
2595 getResults = main.FALSE
2596 utilities.assert_equals( expect=main.TRUE,
2597 actual=getResults,
2598 onpass="Set elements are correct",
2599 onfail="Set elements are incorrect" )
2600
2601 main.step( "Distributed Set size" )
2602 sizeResponses = []
2603 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002604 for i in range( main.numCtrls ):
2605 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002606 name="setTestSize-" + str( i ),
2607 args=[ onosSetName ] )
2608 threads.append( t )
2609 t.start()
2610 for t in threads:
2611 t.join()
2612 sizeResponses.append( t.result )
2613
2614 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002615 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002616 if size != sizeResponses[ i ]:
2617 sizeResults = main.FALSE
2618 main.log.error( "ONOS" + str( i + 1 ) +
2619 " expected a size of " + str( size ) +
2620 " for set " + onosSetName +
2621 " but got " + str( sizeResponses[ i ] ) )
2622 utilities.assert_equals( expect=main.TRUE,
2623 actual=sizeResults,
2624 onpass="Set sizes are correct",
2625 onfail="Set sizes are incorrect" )
2626
2627 main.step( "Distributed Set add()" )
2628 onosSet.add( addValue )
2629 addResponses = []
2630 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002631 for i in range( main.numCtrls ):
2632 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002633 name="setTestAdd-" + str( i ),
2634 args=[ onosSetName, addValue ] )
2635 threads.append( t )
2636 t.start()
2637 for t in threads:
2638 t.join()
2639 addResponses.append( t.result )
2640
2641 # main.TRUE = successfully changed the set
2642 # main.FALSE = action resulted in no change in set
2643 # main.ERROR - Some error in executing the function
2644 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002645 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002646 if addResponses[ i ] == main.TRUE:
2647 # All is well
2648 pass
2649 elif addResponses[ i ] == main.FALSE:
2650 # Already in set, probably fine
2651 pass
2652 elif addResponses[ i ] == main.ERROR:
2653 # Error in execution
2654 addResults = main.FALSE
2655 else:
2656 # unexpected result
2657 addResults = main.FALSE
2658 if addResults != main.TRUE:
2659 main.log.error( "Error executing set add" )
2660
2661 # Check if set is still correct
2662 size = len( onosSet )
2663 getResponses = []
2664 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002665 for i in range( main.numCtrls ):
2666 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002667 name="setTestGet-" + str( i ),
2668 args=[ onosSetName ] )
2669 threads.append( t )
2670 t.start()
2671 for t in threads:
2672 t.join()
2673 getResponses.append( t.result )
2674 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002675 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002676 if isinstance( getResponses[ i ], list):
2677 current = set( getResponses[ i ] )
2678 if len( current ) == len( getResponses[ i ] ):
2679 # no repeats
2680 if onosSet != current:
2681 main.log.error( "ONOS" + str( i + 1 ) +
2682 " has incorrect view" +
2683 " of set " + onosSetName + ":\n" +
2684 str( getResponses[ i ] ) )
2685 main.log.debug( "Expected: " + str( onosSet ) )
2686 main.log.debug( "Actual: " + str( current ) )
2687 getResults = main.FALSE
2688 else:
2689 # error, set is not a set
2690 main.log.error( "ONOS" + str( i + 1 ) +
2691 " has repeat elements in" +
2692 " set " + onosSetName + ":\n" +
2693 str( getResponses[ i ] ) )
2694 getResults = main.FALSE
2695 elif getResponses[ i ] == main.ERROR:
2696 getResults = main.FALSE
2697 sizeResponses = []
2698 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002699 for i in range( main.numCtrls ):
2700 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002701 name="setTestSize-" + str( i ),
2702 args=[ onosSetName ] )
2703 threads.append( t )
2704 t.start()
2705 for t in threads:
2706 t.join()
2707 sizeResponses.append( t.result )
2708 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002709 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002710 if size != sizeResponses[ i ]:
2711 sizeResults = main.FALSE
2712 main.log.error( "ONOS" + str( i + 1 ) +
2713 " expected a size of " + str( size ) +
2714 " for set " + onosSetName +
2715 " but got " + str( sizeResponses[ i ] ) )
2716 addResults = addResults and getResults and sizeResults
2717 utilities.assert_equals( expect=main.TRUE,
2718 actual=addResults,
2719 onpass="Set add correct",
2720 onfail="Set add was incorrect" )
2721
2722 main.step( "Distributed Set addAll()" )
2723 onosSet.update( addAllValue.split() )
2724 addResponses = []
2725 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002726 for i in range( main.numCtrls ):
2727 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002728 name="setTestAddAll-" + str( i ),
2729 args=[ onosSetName, addAllValue ] )
2730 threads.append( t )
2731 t.start()
2732 for t in threads:
2733 t.join()
2734 addResponses.append( t.result )
2735
2736 # main.TRUE = successfully changed the set
2737 # main.FALSE = action resulted in no change in set
2738 # main.ERROR - Some error in executing the function
2739 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002740 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002741 if addResponses[ i ] == main.TRUE:
2742 # All is well
2743 pass
2744 elif addResponses[ i ] == main.FALSE:
2745 # Already in set, probably fine
2746 pass
2747 elif addResponses[ i ] == main.ERROR:
2748 # Error in execution
2749 addAllResults = main.FALSE
2750 else:
2751 # unexpected result
2752 addAllResults = main.FALSE
2753 if addAllResults != main.TRUE:
2754 main.log.error( "Error executing set addAll" )
2755
2756 # Check if set is still correct
2757 size = len( onosSet )
2758 getResponses = []
2759 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002760 for i in range( main.numCtrls ):
2761 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002762 name="setTestGet-" + str( i ),
2763 args=[ onosSetName ] )
2764 threads.append( t )
2765 t.start()
2766 for t in threads:
2767 t.join()
2768 getResponses.append( t.result )
2769 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002770 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002771 if isinstance( getResponses[ i ], list):
2772 current = set( getResponses[ i ] )
2773 if len( current ) == len( getResponses[ i ] ):
2774 # no repeats
2775 if onosSet != current:
2776 main.log.error( "ONOS" + str( i + 1 ) +
2777 " has incorrect view" +
2778 " of set " + onosSetName + ":\n" +
2779 str( getResponses[ i ] ) )
2780 main.log.debug( "Expected: " + str( onosSet ) )
2781 main.log.debug( "Actual: " + str( current ) )
2782 getResults = main.FALSE
2783 else:
2784 # error, set is not a set
2785 main.log.error( "ONOS" + str( i + 1 ) +
2786 " has repeat elements in" +
2787 " set " + onosSetName + ":\n" +
2788 str( getResponses[ i ] ) )
2789 getResults = main.FALSE
2790 elif getResponses[ i ] == main.ERROR:
2791 getResults = main.FALSE
2792 sizeResponses = []
2793 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002794 for i in range( main.numCtrls ):
2795 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002796 name="setTestSize-" + str( i ),
2797 args=[ onosSetName ] )
2798 threads.append( t )
2799 t.start()
2800 for t in threads:
2801 t.join()
2802 sizeResponses.append( t.result )
2803 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002804 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002805 if size != sizeResponses[ i ]:
2806 sizeResults = main.FALSE
2807 main.log.error( "ONOS" + str( i + 1 ) +
2808 " expected a size of " + str( size ) +
2809 " for set " + onosSetName +
2810 " but got " + str( sizeResponses[ i ] ) )
2811 addAllResults = addAllResults and getResults and sizeResults
2812 utilities.assert_equals( expect=main.TRUE,
2813 actual=addAllResults,
2814 onpass="Set addAll correct",
2815 onfail="Set addAll was incorrect" )
2816
2817 main.step( "Distributed Set contains()" )
2818 containsResponses = []
2819 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002820 for i in range( main.numCtrls ):
2821 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002822 name="setContains-" + str( i ),
2823 args=[ onosSetName ],
2824 kwargs={ "values": addValue } )
2825 threads.append( t )
2826 t.start()
2827 for t in threads:
2828 t.join()
2829 # NOTE: This is the tuple
2830 containsResponses.append( t.result )
2831
2832 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002833 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002834 if containsResponses[ i ] == main.ERROR:
2835 containsResults = main.FALSE
2836 else:
2837 containsResults = containsResults and\
2838 containsResponses[ i ][ 1 ]
2839 utilities.assert_equals( expect=main.TRUE,
2840 actual=containsResults,
2841 onpass="Set contains is functional",
2842 onfail="Set contains failed" )
2843
2844 main.step( "Distributed Set containsAll()" )
2845 containsAllResponses = []
2846 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002847 for i in range( main.numCtrls ):
2848 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002849 name="setContainsAll-" + str( i ),
2850 args=[ onosSetName ],
2851 kwargs={ "values": addAllValue } )
2852 threads.append( t )
2853 t.start()
2854 for t in threads:
2855 t.join()
2856 # NOTE: This is the tuple
2857 containsAllResponses.append( t.result )
2858
2859 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002860 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002861 if containsResponses[ i ] == main.ERROR:
2862 containsResults = main.FALSE
2863 else:
2864 containsResults = containsResults and\
2865 containsResponses[ i ][ 1 ]
2866 utilities.assert_equals( expect=main.TRUE,
2867 actual=containsAllResults,
2868 onpass="Set containsAll is functional",
2869 onfail="Set containsAll failed" )
2870
2871 main.step( "Distributed Set remove()" )
2872 onosSet.remove( addValue )
2873 removeResponses = []
2874 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002875 for i in range( main.numCtrls ):
2876 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002877 name="setTestRemove-" + str( i ),
2878 args=[ onosSetName, addValue ] )
2879 threads.append( t )
2880 t.start()
2881 for t in threads:
2882 t.join()
2883 removeResponses.append( t.result )
2884
2885 # main.TRUE = successfully changed the set
2886 # main.FALSE = action resulted in no change in set
2887 # main.ERROR - Some error in executing the function
2888 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002889 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002890 if removeResponses[ i ] == main.TRUE:
2891 # All is well
2892 pass
2893 elif removeResponses[ i ] == main.FALSE:
2894 # not in set, probably fine
2895 pass
2896 elif removeResponses[ i ] == main.ERROR:
2897 # Error in execution
2898 removeResults = main.FALSE
2899 else:
2900 # unexpected result
2901 removeResults = main.FALSE
2902 if removeResults != main.TRUE:
2903 main.log.error( "Error executing set remove" )
2904
2905 # Check if set is still correct
2906 size = len( onosSet )
2907 getResponses = []
2908 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002909 for i in range( main.numCtrls ):
2910 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002911 name="setTestGet-" + str( i ),
2912 args=[ onosSetName ] )
2913 threads.append( t )
2914 t.start()
2915 for t in threads:
2916 t.join()
2917 getResponses.append( t.result )
2918 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002919 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002920 if isinstance( getResponses[ i ], list):
2921 current = set( getResponses[ i ] )
2922 if len( current ) == len( getResponses[ i ] ):
2923 # no repeats
2924 if onosSet != current:
2925 main.log.error( "ONOS" + str( i + 1 ) +
2926 " has incorrect view" +
2927 " of set " + onosSetName + ":\n" +
2928 str( getResponses[ i ] ) )
2929 main.log.debug( "Expected: " + str( onosSet ) )
2930 main.log.debug( "Actual: " + str( current ) )
2931 getResults = main.FALSE
2932 else:
2933 # error, set is not a set
2934 main.log.error( "ONOS" + str( i + 1 ) +
2935 " has repeat elements in" +
2936 " set " + onosSetName + ":\n" +
2937 str( getResponses[ i ] ) )
2938 getResults = main.FALSE
2939 elif getResponses[ i ] == main.ERROR:
2940 getResults = main.FALSE
2941 sizeResponses = []
2942 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002943 for i in range( main.numCtrls ):
2944 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002945 name="setTestSize-" + str( i ),
2946 args=[ onosSetName ] )
2947 threads.append( t )
2948 t.start()
2949 for t in threads:
2950 t.join()
2951 sizeResponses.append( t.result )
2952 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002953 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002954 if size != sizeResponses[ i ]:
2955 sizeResults = main.FALSE
2956 main.log.error( "ONOS" + str( i + 1 ) +
2957 " expected a size of " + str( size ) +
2958 " for set " + onosSetName +
2959 " but got " + str( sizeResponses[ i ] ) )
2960 removeResults = removeResults and getResults and sizeResults
2961 utilities.assert_equals( expect=main.TRUE,
2962 actual=removeResults,
2963 onpass="Set remove correct",
2964 onfail="Set remove was incorrect" )
2965
2966 main.step( "Distributed Set removeAll()" )
2967 onosSet.difference_update( addAllValue.split() )
2968 removeAllResponses = []
2969 threads = []
2970 try:
Jon Halle1a3b752015-07-22 13:02:46 -07002971 for i in range( main.numCtrls ):
2972 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002973 name="setTestRemoveAll-" + str( i ),
2974 args=[ onosSetName, addAllValue ] )
2975 threads.append( t )
2976 t.start()
2977 for t in threads:
2978 t.join()
2979 removeAllResponses.append( t.result )
2980 except Exception, e:
2981 main.log.exception(e)
2982
2983 # main.TRUE = successfully changed the set
2984 # main.FALSE = action resulted in no change in set
2985 # main.ERROR - Some error in executing the function
2986 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002987 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002988 if removeAllResponses[ i ] == main.TRUE:
2989 # All is well
2990 pass
2991 elif removeAllResponses[ i ] == main.FALSE:
2992 # not in set, probably fine
2993 pass
2994 elif removeAllResponses[ i ] == main.ERROR:
2995 # Error in execution
2996 removeAllResults = main.FALSE
2997 else:
2998 # unexpected result
2999 removeAllResults = main.FALSE
3000 if removeAllResults != main.TRUE:
3001 main.log.error( "Error executing set removeAll" )
3002
3003 # Check if set is still correct
3004 size = len( onosSet )
3005 getResponses = []
3006 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003007 for i in range( main.numCtrls ):
3008 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003009 name="setTestGet-" + str( i ),
3010 args=[ onosSetName ] )
3011 threads.append( t )
3012 t.start()
3013 for t in threads:
3014 t.join()
3015 getResponses.append( t.result )
3016 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003017 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003018 if isinstance( getResponses[ i ], list):
3019 current = set( getResponses[ i ] )
3020 if len( current ) == len( getResponses[ i ] ):
3021 # no repeats
3022 if onosSet != current:
3023 main.log.error( "ONOS" + str( i + 1 ) +
3024 " has incorrect view" +
3025 " of set " + onosSetName + ":\n" +
3026 str( getResponses[ i ] ) )
3027 main.log.debug( "Expected: " + str( onosSet ) )
3028 main.log.debug( "Actual: " + str( current ) )
3029 getResults = main.FALSE
3030 else:
3031 # error, set is not a set
3032 main.log.error( "ONOS" + str( i + 1 ) +
3033 " has repeat elements in" +
3034 " set " + onosSetName + ":\n" +
3035 str( getResponses[ i ] ) )
3036 getResults = main.FALSE
3037 elif getResponses[ i ] == main.ERROR:
3038 getResults = main.FALSE
3039 sizeResponses = []
3040 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003041 for i in range( main.numCtrls ):
3042 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003043 name="setTestSize-" + str( i ),
3044 args=[ onosSetName ] )
3045 threads.append( t )
3046 t.start()
3047 for t in threads:
3048 t.join()
3049 sizeResponses.append( t.result )
3050 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003051 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003052 if size != sizeResponses[ i ]:
3053 sizeResults = main.FALSE
3054 main.log.error( "ONOS" + str( i + 1 ) +
3055 " expected a size of " + str( size ) +
3056 " for set " + onosSetName +
3057 " but got " + str( sizeResponses[ i ] ) )
3058 removeAllResults = removeAllResults and getResults and sizeResults
3059 utilities.assert_equals( expect=main.TRUE,
3060 actual=removeAllResults,
3061 onpass="Set removeAll correct",
3062 onfail="Set removeAll was incorrect" )
3063
3064 main.step( "Distributed Set addAll()" )
3065 onosSet.update( addAllValue.split() )
3066 addResponses = []
3067 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003068 for i in range( main.numCtrls ):
3069 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003070 name="setTestAddAll-" + str( i ),
3071 args=[ onosSetName, addAllValue ] )
3072 threads.append( t )
3073 t.start()
3074 for t in threads:
3075 t.join()
3076 addResponses.append( t.result )
3077
3078 # main.TRUE = successfully changed the set
3079 # main.FALSE = action resulted in no change in set
3080 # main.ERROR - Some error in executing the function
3081 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003082 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003083 if addResponses[ i ] == main.TRUE:
3084 # All is well
3085 pass
3086 elif addResponses[ i ] == main.FALSE:
3087 # Already in set, probably fine
3088 pass
3089 elif addResponses[ i ] == main.ERROR:
3090 # Error in execution
3091 addAllResults = main.FALSE
3092 else:
3093 # unexpected result
3094 addAllResults = main.FALSE
3095 if addAllResults != main.TRUE:
3096 main.log.error( "Error executing set addAll" )
3097
3098 # Check if set is still correct
3099 size = len( onosSet )
3100 getResponses = []
3101 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003102 for i in range( main.numCtrls ):
3103 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003104 name="setTestGet-" + str( i ),
3105 args=[ onosSetName ] )
3106 threads.append( t )
3107 t.start()
3108 for t in threads:
3109 t.join()
3110 getResponses.append( t.result )
3111 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003112 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003113 if isinstance( getResponses[ i ], list):
3114 current = set( getResponses[ i ] )
3115 if len( current ) == len( getResponses[ i ] ):
3116 # no repeats
3117 if onosSet != current:
3118 main.log.error( "ONOS" + str( i + 1 ) +
3119 " has incorrect view" +
3120 " of set " + onosSetName + ":\n" +
3121 str( getResponses[ i ] ) )
3122 main.log.debug( "Expected: " + str( onosSet ) )
3123 main.log.debug( "Actual: " + str( current ) )
3124 getResults = main.FALSE
3125 else:
3126 # error, set is not a set
3127 main.log.error( "ONOS" + str( i + 1 ) +
3128 " has repeat elements in" +
3129 " set " + onosSetName + ":\n" +
3130 str( getResponses[ i ] ) )
3131 getResults = main.FALSE
3132 elif getResponses[ i ] == main.ERROR:
3133 getResults = main.FALSE
3134 sizeResponses = []
3135 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003136 for i in range( main.numCtrls ):
3137 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003138 name="setTestSize-" + str( i ),
3139 args=[ onosSetName ] )
3140 threads.append( t )
3141 t.start()
3142 for t in threads:
3143 t.join()
3144 sizeResponses.append( t.result )
3145 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003146 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003147 if size != sizeResponses[ i ]:
3148 sizeResults = main.FALSE
3149 main.log.error( "ONOS" + str( i + 1 ) +
3150 " expected a size of " + str( size ) +
3151 " for set " + onosSetName +
3152 " but got " + str( sizeResponses[ i ] ) )
3153 addAllResults = addAllResults and getResults and sizeResults
3154 utilities.assert_equals( expect=main.TRUE,
3155 actual=addAllResults,
3156 onpass="Set addAll correct",
3157 onfail="Set addAll was incorrect" )
3158
3159 main.step( "Distributed Set clear()" )
3160 onosSet.clear()
3161 clearResponses = []
3162 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003163 for i in range( main.numCtrls ):
3164 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003165 name="setTestClear-" + str( i ),
3166 args=[ onosSetName, " "], # Values doesn't matter
3167 kwargs={ "clear": True } )
3168 threads.append( t )
3169 t.start()
3170 for t in threads:
3171 t.join()
3172 clearResponses.append( t.result )
3173
3174 # main.TRUE = successfully changed the set
3175 # main.FALSE = action resulted in no change in set
3176 # main.ERROR - Some error in executing the function
3177 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003178 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003179 if clearResponses[ i ] == main.TRUE:
3180 # All is well
3181 pass
3182 elif clearResponses[ i ] == main.FALSE:
3183 # Nothing set, probably fine
3184 pass
3185 elif clearResponses[ i ] == main.ERROR:
3186 # Error in execution
3187 clearResults = main.FALSE
3188 else:
3189 # unexpected result
3190 clearResults = main.FALSE
3191 if clearResults != main.TRUE:
3192 main.log.error( "Error executing set clear" )
3193
3194 # Check if set is still correct
3195 size = len( onosSet )
3196 getResponses = []
3197 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003198 for i in range( main.numCtrls ):
3199 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003200 name="setTestGet-" + str( i ),
3201 args=[ onosSetName ] )
3202 threads.append( t )
3203 t.start()
3204 for t in threads:
3205 t.join()
3206 getResponses.append( t.result )
3207 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003208 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003209 if isinstance( getResponses[ i ], list):
3210 current = set( getResponses[ i ] )
3211 if len( current ) == len( getResponses[ i ] ):
3212 # no repeats
3213 if onosSet != current:
3214 main.log.error( "ONOS" + str( i + 1 ) +
3215 " has incorrect view" +
3216 " of set " + onosSetName + ":\n" +
3217 str( getResponses[ i ] ) )
3218 main.log.debug( "Expected: " + str( onosSet ) )
3219 main.log.debug( "Actual: " + str( current ) )
3220 getResults = main.FALSE
3221 else:
3222 # error, set is not a set
3223 main.log.error( "ONOS" + str( i + 1 ) +
3224 " has repeat elements in" +
3225 " set " + onosSetName + ":\n" +
3226 str( getResponses[ i ] ) )
3227 getResults = main.FALSE
3228 elif getResponses[ i ] == main.ERROR:
3229 getResults = main.FALSE
3230 sizeResponses = []
3231 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003232 for i in range( main.numCtrls ):
3233 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003234 name="setTestSize-" + str( i ),
3235 args=[ onosSetName ] )
3236 threads.append( t )
3237 t.start()
3238 for t in threads:
3239 t.join()
3240 sizeResponses.append( t.result )
3241 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003242 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003243 if size != sizeResponses[ i ]:
3244 sizeResults = main.FALSE
3245 main.log.error( "ONOS" + str( i + 1 ) +
3246 " expected a size of " + str( size ) +
3247 " for set " + onosSetName +
3248 " but got " + str( sizeResponses[ i ] ) )
3249 clearResults = clearResults and getResults and sizeResults
3250 utilities.assert_equals( expect=main.TRUE,
3251 actual=clearResults,
3252 onpass="Set clear correct",
3253 onfail="Set clear was incorrect" )
3254
3255 main.step( "Distributed Set addAll()" )
3256 onosSet.update( addAllValue.split() )
3257 addResponses = []
3258 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003259 for i in range( main.numCtrls ):
3260 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003261 name="setTestAddAll-" + str( i ),
3262 args=[ onosSetName, addAllValue ] )
3263 threads.append( t )
3264 t.start()
3265 for t in threads:
3266 t.join()
3267 addResponses.append( t.result )
3268
3269 # main.TRUE = successfully changed the set
3270 # main.FALSE = action resulted in no change in set
3271 # main.ERROR - Some error in executing the function
3272 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003273 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003274 if addResponses[ i ] == main.TRUE:
3275 # All is well
3276 pass
3277 elif addResponses[ i ] == main.FALSE:
3278 # Already in set, probably fine
3279 pass
3280 elif addResponses[ i ] == main.ERROR:
3281 # Error in execution
3282 addAllResults = main.FALSE
3283 else:
3284 # unexpected result
3285 addAllResults = main.FALSE
3286 if addAllResults != main.TRUE:
3287 main.log.error( "Error executing set addAll" )
3288
3289 # Check if set is still correct
3290 size = len( onosSet )
3291 getResponses = []
3292 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003293 for i in range( main.numCtrls ):
3294 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003295 name="setTestGet-" + str( i ),
3296 args=[ onosSetName ] )
3297 threads.append( t )
3298 t.start()
3299 for t in threads:
3300 t.join()
3301 getResponses.append( t.result )
3302 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003303 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003304 if isinstance( getResponses[ i ], list):
3305 current = set( getResponses[ i ] )
3306 if len( current ) == len( getResponses[ i ] ):
3307 # no repeats
3308 if onosSet != current:
3309 main.log.error( "ONOS" + str( i + 1 ) +
3310 " has incorrect view" +
3311 " of set " + onosSetName + ":\n" +
3312 str( getResponses[ i ] ) )
3313 main.log.debug( "Expected: " + str( onosSet ) )
3314 main.log.debug( "Actual: " + str( current ) )
3315 getResults = main.FALSE
3316 else:
3317 # error, set is not a set
3318 main.log.error( "ONOS" + str( i + 1 ) +
3319 " has repeat elements in" +
3320 " set " + onosSetName + ":\n" +
3321 str( getResponses[ i ] ) )
3322 getResults = main.FALSE
3323 elif getResponses[ i ] == main.ERROR:
3324 getResults = main.FALSE
3325 sizeResponses = []
3326 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003327 for i in range( main.numCtrls ):
3328 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003329 name="setTestSize-" + str( i ),
3330 args=[ onosSetName ] )
3331 threads.append( t )
3332 t.start()
3333 for t in threads:
3334 t.join()
3335 sizeResponses.append( t.result )
3336 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003337 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003338 if size != sizeResponses[ i ]:
3339 sizeResults = main.FALSE
3340 main.log.error( "ONOS" + str( i + 1 ) +
3341 " expected a size of " + str( size ) +
3342 " for set " + onosSetName +
3343 " but got " + str( sizeResponses[ i ] ) )
3344 addAllResults = addAllResults and getResults and sizeResults
3345 utilities.assert_equals( expect=main.TRUE,
3346 actual=addAllResults,
3347 onpass="Set addAll correct",
3348 onfail="Set addAll was incorrect" )
3349
3350 main.step( "Distributed Set retain()" )
3351 onosSet.intersection_update( retainValue.split() )
3352 retainResponses = []
3353 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003354 for i in range( main.numCtrls ):
3355 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003356 name="setTestRetain-" + str( i ),
3357 args=[ onosSetName, retainValue ],
3358 kwargs={ "retain": True } )
3359 threads.append( t )
3360 t.start()
3361 for t in threads:
3362 t.join()
3363 retainResponses.append( t.result )
3364
3365 # main.TRUE = successfully changed the set
3366 # main.FALSE = action resulted in no change in set
3367 # main.ERROR - Some error in executing the function
3368 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003369 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003370 if retainResponses[ i ] == main.TRUE:
3371 # All is well
3372 pass
3373 elif retainResponses[ i ] == main.FALSE:
3374 # Already in set, probably fine
3375 pass
3376 elif retainResponses[ i ] == main.ERROR:
3377 # Error in execution
3378 retainResults = main.FALSE
3379 else:
3380 # unexpected result
3381 retainResults = main.FALSE
3382 if retainResults != main.TRUE:
3383 main.log.error( "Error executing set retain" )
3384
3385 # Check if set is still correct
3386 size = len( onosSet )
3387 getResponses = []
3388 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003389 for i in range( main.numCtrls ):
3390 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003391 name="setTestGet-" + str( i ),
3392 args=[ onosSetName ] )
3393 threads.append( t )
3394 t.start()
3395 for t in threads:
3396 t.join()
3397 getResponses.append( t.result )
3398 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003399 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003400 if isinstance( getResponses[ i ], list):
3401 current = set( getResponses[ i ] )
3402 if len( current ) == len( getResponses[ i ] ):
3403 # no repeats
3404 if onosSet != current:
3405 main.log.error( "ONOS" + str( i + 1 ) +
3406 " has incorrect view" +
3407 " of set " + onosSetName + ":\n" +
3408 str( getResponses[ i ] ) )
3409 main.log.debug( "Expected: " + str( onosSet ) )
3410 main.log.debug( "Actual: " + str( current ) )
3411 getResults = main.FALSE
3412 else:
3413 # error, set is not a set
3414 main.log.error( "ONOS" + str( i + 1 ) +
3415 " has repeat elements in" +
3416 " set " + onosSetName + ":\n" +
3417 str( getResponses[ i ] ) )
3418 getResults = main.FALSE
3419 elif getResponses[ i ] == main.ERROR:
3420 getResults = main.FALSE
3421 sizeResponses = []
3422 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003423 for i in range( main.numCtrls ):
3424 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003425 name="setTestSize-" + str( i ),
3426 args=[ onosSetName ] )
3427 threads.append( t )
3428 t.start()
3429 for t in threads:
3430 t.join()
3431 sizeResponses.append( t.result )
3432 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003433 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003434 if size != sizeResponses[ i ]:
3435 sizeResults = main.FALSE
3436 main.log.error( "ONOS" + str( i + 1 ) +
3437 " expected a size of " +
3438 str( size ) + " for set " + onosSetName +
3439 " but got " + str( sizeResponses[ i ] ) )
3440 retainResults = retainResults and getResults and sizeResults
3441 utilities.assert_equals( expect=main.TRUE,
3442 actual=retainResults,
3443 onpass="Set retain correct",
3444 onfail="Set retain was incorrect" )
3445
Jon Hall2a5002c2015-08-21 16:49:11 -07003446 # Transactional maps
3447 main.step( "Partitioned Transactional maps put" )
3448 tMapValue = "Testing"
3449 numKeys = 100
3450 putResult = True
3451 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
3452 if len( putResponses ) == 100:
3453 for i in putResponses:
3454 if putResponses[ i ][ 'value' ] != tMapValue:
3455 putResult = False
3456 else:
3457 putResult = False
3458 if not putResult:
3459 main.log.debug( "Put response values: " + str( putResponses ) )
3460 utilities.assert_equals( expect=True,
3461 actual=putResult,
3462 onpass="Partitioned Transactional Map put successful",
3463 onfail="Partitioned Transactional Map put values are incorrect" )
3464
3465 main.step( "Partitioned Transactional maps get" )
3466 getCheck = True
3467 for n in range( 1, numKeys + 1 ):
3468 getResponses = []
3469 threads = []
3470 valueCheck = True
3471 for i in range( main.numCtrls ):
3472 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
3473 name="TMap-get-" + str( i ),
3474 args=[ "Key" + str ( n ) ] )
3475 threads.append( t )
3476 t.start()
3477 for t in threads:
3478 t.join()
3479 getResponses.append( t.result )
3480 for node in getResponses:
3481 if node != tMapValue:
3482 valueCheck = False
3483 if not valueCheck:
3484 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
3485 main.log.warn( getResponses )
3486 getCheck = getCheck and valueCheck
3487 utilities.assert_equals( expect=True,
3488 actual=getCheck,
3489 onpass="Partitioned Transactional Map get values were correct",
3490 onfail="Partitioned Transactional Map values incorrect" )
3491
3492 main.step( "In-memory Transactional maps put" )
3493 tMapValue = "Testing"
3494 numKeys = 100
3495 putResult = True
3496 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
3497 if len( putResponses ) == 100:
3498 for i in putResponses:
3499 if putResponses[ i ][ 'value' ] != tMapValue:
3500 putResult = False
3501 else:
3502 putResult = False
3503 if not putResult:
3504 main.log.debug( "Put response values: " + str( putResponses ) )
3505 utilities.assert_equals( expect=True,
3506 actual=putResult,
3507 onpass="In-Memory Transactional Map put successful",
3508 onfail="In-Memory Transactional Map put values are incorrect" )
3509
3510 main.step( "In-Memory Transactional maps get" )
3511 getCheck = True
3512 for n in range( 1, numKeys + 1 ):
3513 getResponses = []
3514 threads = []
3515 valueCheck = True
3516 for i in range( main.numCtrls ):
3517 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
3518 name="TMap-get-" + str( i ),
3519 args=[ "Key" + str ( n ) ],
3520 kwargs={ "inMemory": True } )
3521 threads.append( t )
3522 t.start()
3523 for t in threads:
3524 t.join()
3525 getResponses.append( t.result )
3526 for node in getResponses:
3527 if node != tMapValue:
3528 valueCheck = False
3529 if not valueCheck:
3530 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
3531 main.log.warn( getResponses )
3532 getCheck = getCheck and valueCheck
3533 utilities.assert_equals( expect=True,
3534 actual=getCheck,
3535 onpass="In-Memory Transactional Map get values were correct",
3536 onfail="In-Memory Transactional Map values incorrect" )