blob: 3ab50722891292e02eeca41e75086de1ecf43d79 [file] [log] [blame]
Jon Hall85794ff2015-07-08 14:12:30 -07001"""
2Description: This test is to determine if a single
3 instance ONOS 'cluster' can handle a restart
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAsingleInstanceRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080051 import time
Jon Hall85794ff2015-07-08 14:12:30 -070052 main.log.info( "ONOS Single node cluster restart " +
53 "HA test - initialization" )
54 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070055 main.caseExplanation = "Setup the test environment including " +\
Jon Hall85794ff2015-07-08 14:12:30 -070056 "installing ONOS, starting Mininet and ONOS" +\
57 "cli sessions."
58 # TODO: save all the timers and output them for plotting
59
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall85794ff2015-07-08 14:12:30 -070071
Jon Halle1a3b752015-07-22 13:02:46 -070072 try:
73 fileName = "Counters"
74 path = main.params[ 'imports' ][ 'path' ]
75 main.Counters = imp.load_source( fileName,
76 path + fileName + ".py" )
77 except Exception as e:
78 main.log.exception( e )
79 main.cleanup()
80 main.exit()
81
82 main.CLIs = []
83 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -070084 ipList = []
85 for i in range( 1, int( main.ONOSbench.maxNodes ) + 1 ):
86 try:
Jon Halle1a3b752015-07-22 13:02:46 -070087 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
88 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
89 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -070090 except AttributeError:
91 break
Jon Hall85794ff2015-07-08 14:12:30 -070092
Jon Hall5cf14d52015-07-16 12:15:19 -070093 main.step( "Create cell file" )
94 cellAppString = main.params[ 'ENV' ][ 'appString' ]
95 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
96 main.Mininet1.ip_address,
97 cellAppString, ipList )
Jon Hall85794ff2015-07-08 14:12:30 -070098 main.step( "Applying cell variable to environment" )
99 cellResult = main.ONOSbench.setCell( cellName )
100 verifyResult = main.ONOSbench.verifyCell()
101
102 # FIXME:this is short term fix
103 main.log.info( "Removing raft logs" )
104 main.ONOSbench.onosRemoveRaftLogs()
105
106 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700107 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700108 main.ONOSbench.onosUninstall( node.ip_address )
109
110 # Make sure ONOS is DEAD
111 main.log.info( "Killing any ONOS processes" )
112 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700113 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700114 killed = main.ONOSbench.onosKill( node.ip_address )
115 killResults = killResults and killed
116
117 cleanInstallResult = main.TRUE
118 gitPullResult = main.TRUE
119
120 main.step( "Starting Mininet" )
121 # scp topo file to mininet
122 # TODO: move to params?
123 topoName = "obelisk.py"
124 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700125 main.ONOSbench.scp( main.Mininet1,
126 filePath + topoName,
127 main.Mininet1.home,
128 direction="to" )
Jon Hall85794ff2015-07-08 14:12:30 -0700129 mnResult = main.Mininet1.startNet( )
130 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
131 onpass="Mininet Started",
132 onfail="Error starting Mininet" )
133
134 main.step( "Git checkout and pull " + gitBranch )
135 if PULLCODE:
136 main.ONOSbench.gitCheckout( gitBranch )
137 gitPullResult = main.ONOSbench.gitPull()
138 # values of 1 or 3 are good
139 utilities.assert_lesser( expect=0, actual=gitPullResult,
140 onpass="Git pull successful",
141 onfail="Git pull failed" )
142 main.ONOSbench.getVersion( report=True )
143
144 main.step( "Using mvn clean install" )
145 cleanInstallResult = main.TRUE
146 if PULLCODE and gitPullResult == main.TRUE:
147 cleanInstallResult = main.ONOSbench.cleanInstall()
148 else:
149 main.log.warn( "Did not pull new code so skipping mvn " +
150 "clean install" )
151 utilities.assert_equals( expect=main.TRUE,
152 actual=cleanInstallResult,
153 onpass="MCI successful",
154 onfail="MCI failed" )
155 # GRAPHS
156 # NOTE: important params here:
157 # job = name of Jenkins job
158 # Plot Name = Plot-HA, only can be used if multiple plots
159 # index = The number of the graph under plot name
Jon Hall5cf14d52015-07-16 12:15:19 -0700160 job = "HAsingleInstanceRestart"
Jon Hall85794ff2015-07-08 14:12:30 -0700161 plotName = "Plot-HA"
Jon Hallff566d52016-01-15 14:45:36 -0800162 index = "1"
Jon Hall85794ff2015-07-08 14:12:30 -0700163 graphs = '<ac:structured-macro ac:name="html">\n'
164 graphs += '<ac:plain-text-body><![CDATA[\n'
165 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800166 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall85794ff2015-07-08 14:12:30 -0700167 '&width=500&height=300"' +\
168 'noborder="0" width="500" height="300" scrolling="yes" ' +\
169 'seamless="seamless"></iframe>\n'
170 graphs += ']]></ac:plain-text-body>\n'
171 graphs += '</ac:structured-macro>\n'
172 main.log.wiki(graphs)
173
Jon Halle1a3b752015-07-22 13:02:46 -0700174 main.CLIs = []
175 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700176 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700177 for i in range( 1, main.numCtrls + 1 ):
178 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
179 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
180 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700181
182 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, "SingleHA",
183 main.Mininet1.ip_address,
184 cellAppString, ipList[ 0 ] )
Jon Hall85794ff2015-07-08 14:12:30 -0700185 cellResult = main.ONOSbench.setCell( "SingleHA" )
186 verifyResult = main.ONOSbench.verifyCell()
187 main.step( "Creating ONOS package" )
188 packageResult = main.ONOSbench.onosPackage()
189 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
190 onpass="ONOS package successful",
191 onfail="ONOS package failed" )
192
193 main.step( "Installing ONOS package" )
194 onosInstallResult = main.ONOSbench.onosInstall(
Jon Halle1a3b752015-07-22 13:02:46 -0700195 options="-f", node=main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700196 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
197 onpass="ONOS install successful",
198 onfail="ONOS install failed" )
199
200 main.step( "Checking if ONOS is up yet" )
201 for i in range( 2 ):
Jon Halle1a3b752015-07-22 13:02:46 -0700202 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700203 if onos1Isup:
204 break
205 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
206 onpass="ONOS startup successful",
207 onfail="ONOS startup failed" )
208
209 main.log.step( "Starting ONOS CLI sessions" )
Jon Halle1a3b752015-07-22 13:02:46 -0700210 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700211 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
212 onpass="ONOS cli startup successful",
213 onfail="ONOS cli startup failed" )
214
215 if main.params[ 'tcpdump' ].lower() == "true":
216 main.step( "Start Packet Capture MN" )
217 main.Mininet2.startTcpdump(
218 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
219 + "-MN.pcap",
220 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
221 port=main.params[ 'MNtcpdump' ][ 'port' ] )
222
223 main.step( "App Ids check" )
Jon Hallf3d16e72015-12-16 17:45:08 -0800224 time.sleep(60)
Jon Hall85794ff2015-07-08 14:12:30 -0700225 appCheck = main.ONOScli1.appToIDCheck()
226 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700227 main.log.warn( main.CLIs[0].apps() )
228 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700229 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
230 onpass="App Ids seem to be correct",
231 onfail="Something is wrong with app Ids" )
232
233 if cliResults == main.FALSE:
234 main.log.error( "Failed to start ONOS, stopping test" )
235 main.cleanup()
236 main.exit()
237
238 def CASE2( self, main ):
239 """
240 Assign devices to controllers
241 """
242 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700243 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700244 assert main, "main not defined"
245 assert utilities.assert_equals, "utilities.assert_equals not defined"
246
247 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700248 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700249 "and check that an ONOS node becomes the " +\
250 "master of the device."
251 main.step( "Assign switches to controllers" )
252
253 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700254 for i in range( main.numCtrls ):
255 ipList.append( main.nodes[ i ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700256 swList = []
257 for i in range( 1, 29 ):
258 swList.append( "s" + str( i ) )
259 main.Mininet1.assignSwController( sw=swList, ip=ipList )
260
261 mastershipCheck = main.TRUE
262 for i in range( 1, 29 ):
263 response = main.Mininet1.getSwController( "s" + str( i ) )
264 try:
265 main.log.info( str( response ) )
266 except Exception:
267 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700268 if re.search( "tcp:" + main.nodes[0].ip_address, response ):
Jon Hall85794ff2015-07-08 14:12:30 -0700269 mastershipCheck = mastershipCheck and main.TRUE
270 else:
271 mastershipCheck = main.FALSE
272 if mastershipCheck == main.TRUE:
273 main.log.info( "Switch mastership assigned correctly" )
274 utilities.assert_equals(
275 expect=main.TRUE,
276 actual=mastershipCheck,
277 onpass="Switch mastership assigned correctly",
278 onfail="Switches not assigned correctly to controllers" )
279
280 def CASE21( self, main ):
281 """
282 Assign mastership to controllers
283 """
Jon Halle1a3b752015-07-22 13:02:46 -0700284 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700285 assert main, "main not defined"
286 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700287 assert main.CLIs, "main.CLIs not defined"
288 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700289
290 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700291 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700292 "device. Then manually assign" +\
293 " mastership to specific ONOS nodes using" +\
294 " 'device-role'"
295 main.step( "Assign mastership of switches to specific controllers" )
296 roleCall = main.TRUE
297 roleCheck = main.TRUE
298 try:
299 for i in range( 1, 29 ): # switches 1 through 28
Jon Halle1a3b752015-07-22 13:02:46 -0700300 ip = main.nodes[ 0 ].ip_address # ONOS1
Jon Hall85794ff2015-07-08 14:12:30 -0700301 # set up correct variables:
302 if i == 1:
303 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
304 elif i == 2:
305 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
306 elif i == 3:
307 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
308 elif i == 4:
309 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
310 elif i == 5:
311 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
312 elif i == 6:
313 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
314 elif i == 7:
315 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
316 elif i >= 8 and i <= 17:
317 dpid = '3' + str( i ).zfill( 3 )
318 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
319 elif i >= 18 and i <= 27:
320 dpid = '6' + str( i ).zfill( 3 )
321 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
322 elif i == 28:
323 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
324 else:
325 main.log.error( "You didn't write an else statement for " +
326 "switch s" + str( i ) )
327 # Assign switch
328 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
329 # TODO: make this controller dynamic
330 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
331 ip )
332 # Check assignment
333 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
334 if ip in master:
335 roleCheck = roleCheck and main.TRUE
336 else:
337 roleCheck = roleCheck and main.FALSE
338 main.log.error( "Error, controller " + ip + " is not" +
339 " master " + "of device " +
340 str( deviceId ) + ". Master is " +
341 repr( master ) + "." )
342 except ( AttributeError, AssertionError ):
343 main.log.exception( "Something is wrong with ONOS device view" )
344 main.log.info( main.ONOScli1.devices() )
345 utilities.assert_equals(
346 expect=main.TRUE,
347 actual=roleCall,
348 onpass="Re-assigned switch mastership to designated controller",
349 onfail="Something wrong with deviceRole calls" )
350
351 main.step( "Check mastership was correctly assigned" )
352 utilities.assert_equals(
353 expect=main.TRUE,
354 actual=roleCheck,
355 onpass="Switches were successfully reassigned to designated " +
356 "controller",
357 onfail="Switches were not successfully reassigned" )
358
359 def CASE3( self, main ):
360 """
361 Assign intents
362 """
363 import time
364 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700365 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700366 assert main, "main not defined"
367 assert utilities.assert_equals, "utilities.assert_equals not defined"
368 # NOTE: we must reinstall intents until we have a persistant intent
369 # datastore!
370 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700371 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700372 "assign predetermined host-to-host intents." +\
373 " After installation, check that the intent" +\
374 " is distributed to all nodes and the state" +\
375 " is INSTALLED"
376
377 # install onos-app-fwd
378 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700379 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700380 utilities.assert_equals( expect=main.TRUE, actual=installResults,
381 onpass="Install fwd successful",
382 onfail="Install fwd failed" )
383
384 main.step( "Check app ids" )
385 appCheck = main.ONOScli1.appToIDCheck()
386 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700387 main.log.warn( main.CLIs[0].apps() )
388 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700389 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
390 onpass="App Ids seem to be correct",
391 onfail="Something is wrong with app Ids" )
392
393 main.step( "Discovering Hosts( Via pingall for now )" )
394 # FIXME: Once we have a host discovery mechanism, use that instead
395 # REACTIVE FWD test
396 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700397 passMsg = "Reactive Pingall test passed"
398 time1 = time.time()
399 pingResult = main.Mininet1.pingall()
400 time2 = time.time()
401 if not pingResult:
402 main.log.warn("First pingall failed. Trying again...")
Jon Hall85794ff2015-07-08 14:12:30 -0700403 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700404 passMsg += " on the second try"
405 utilities.assert_equals(
406 expect=main.TRUE,
407 actual=pingResult,
408 onpass= passMsg,
409 onfail="Reactive Pingall failed, " +
410 "one or more ping pairs failed" )
411 main.log.info( "Time for pingall: %2f seconds" %
412 ( time2 - time1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700413 # timeout for fwd flows
414 time.sleep( 11 )
415 # uninstall onos-app-fwd
416 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700417 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700418 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
419 onpass="Uninstall fwd successful",
420 onfail="Uninstall fwd failed" )
421
422 main.step( "Check app ids" )
423 appCheck2 = main.ONOScli1.appToIDCheck()
424 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700425 main.log.warn( main.CLIs[0].apps() )
426 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700427 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
428 onpass="App Ids seem to be correct",
429 onfail="Something is wrong with app Ids" )
430
431 main.step( "Add host intents via cli" )
432 intentIds = []
433 # TODO: move the host numbers to params
434 # Maybe look at all the paths we ping?
435 intentAddResult = True
436 hostResult = main.TRUE
437 for i in range( 8, 18 ):
438 main.log.info( "Adding host intent between h" + str( i ) +
439 " and h" + str( i + 10 ) )
440 host1 = "00:00:00:00:00:" + \
441 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
442 host2 = "00:00:00:00:00:" + \
443 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
444 # NOTE: getHost can return None
445 host1Dict = main.ONOScli1.getHost( host1 )
446 host2Dict = main.ONOScli1.getHost( host2 )
447 host1Id = None
448 host2Id = None
449 if host1Dict and host2Dict:
450 host1Id = host1Dict.get( 'id', None )
451 host2Id = host2Dict.get( 'id', None )
452 if host1Id and host2Id:
453 tmpId = main.ONOScli1.addHostIntent( host1Id, host2Id )
454 if tmpId:
455 main.log.info( "Added intent with id: " + tmpId )
456 intentIds.append( tmpId )
457 else:
458 main.log.error( "addHostIntent returned: " +
459 repr( tmpId ) )
460 else:
461 main.log.error( "Error, getHost() failed for h" + str( i ) +
462 " and/or h" + str( i + 10 ) )
463 hosts = main.ONOScli1.hosts()
464 main.log.warn( "Hosts output: " )
465 try:
466 main.log.warn( json.dumps( json.loads( hosts ),
467 sort_keys=True,
468 indent=4,
469 separators=( ',', ': ' ) ) )
470 except ( ValueError, TypeError ):
471 main.log.warn( repr( hosts ) )
472 hostResult = main.FALSE
473 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
474 onpass="Found a host id for each host",
475 onfail="Error looking up host ids" )
476
477 intentStart = time.time()
478 onosIds = main.ONOScli1.getAllIntentsId()
479 main.log.info( "Submitted intents: " + str( intentIds ) )
480 main.log.info( "Intents in ONOS: " + str( onosIds ) )
481 for intent in intentIds:
482 if intent in onosIds:
483 pass # intent submitted is in onos
484 else:
485 intentAddResult = False
486 if intentAddResult:
487 intentStop = time.time()
488 else:
489 intentStop = None
490 # Print the intent states
491 intents = main.ONOScli1.intents()
492 intentStates = []
493 installedCheck = True
494 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
495 count = 0
496 try:
497 for intent in json.loads( intents ):
498 state = intent.get( 'state', None )
499 if "INSTALLED" not in state:
500 installedCheck = False
501 intentId = intent.get( 'id', None )
502 intentStates.append( ( intentId, state ) )
503 except ( ValueError, TypeError ):
504 main.log.exception( "Error parsing intents" )
505 # add submitted intents not in the store
506 tmplist = [ i for i, s in intentStates ]
507 missingIntents = False
508 for i in intentIds:
509 if i not in tmplist:
510 intentStates.append( ( i, " - " ) )
511 missingIntents = True
512 intentStates.sort()
513 for i, s in intentStates:
514 count += 1
515 main.log.info( "%-6s%-15s%-15s" %
516 ( str( count ), str( i ), str( s ) ) )
517 leaders = main.ONOScli1.leaders()
518 try:
519 missing = False
520 if leaders:
521 parsedLeaders = json.loads( leaders )
522 main.log.warn( json.dumps( parsedLeaders,
523 sort_keys=True,
524 indent=4,
525 separators=( ',', ': ' ) ) )
526 # check for all intent partitions
527 topics = []
528 for i in range( 14 ):
529 topics.append( "intent-partition-" + str( i ) )
530 main.log.debug( topics )
531 ONOStopics = [ j['topic'] for j in parsedLeaders ]
532 for topic in topics:
533 if topic not in ONOStopics:
534 main.log.error( "Error: " + topic +
535 " not in leaders" )
536 missing = True
537 else:
538 main.log.error( "leaders() returned None" )
539 except ( ValueError, TypeError ):
540 main.log.exception( "Error parsing leaders" )
541 main.log.error( repr( leaders ) )
542 # Check all nodes
543 if missing:
544 response = main.ONOScli1.leaders( jsonFormat=False)
545 main.log.warn( "ONOS1 leaders output: \n" +
546 str( response ) )
547
548 partitions = main.ONOScli1.partitions()
549 try:
550 if partitions :
551 parsedPartitions = json.loads( partitions )
552 main.log.warn( json.dumps( parsedPartitions,
553 sort_keys=True,
554 indent=4,
555 separators=( ',', ': ' ) ) )
556 # TODO check for a leader in all paritions
557 # TODO check for consistency among nodes
558 else:
559 main.log.error( "partitions() returned None" )
560 except ( ValueError, TypeError ):
561 main.log.exception( "Error parsing partitions" )
562 main.log.error( repr( partitions ) )
563 pendingMap = main.ONOScli1.pendingMap()
564 try:
565 if pendingMap :
566 parsedPending = json.loads( pendingMap )
567 main.log.warn( json.dumps( parsedPending,
568 sort_keys=True,
569 indent=4,
570 separators=( ',', ': ' ) ) )
571 # TODO check something here?
572 else:
573 main.log.error( "pendingMap() returned None" )
574 except ( ValueError, TypeError ):
575 main.log.exception( "Error parsing pending map" )
576 main.log.error( repr( pendingMap ) )
577
578 intentAddResult = bool( intentAddResult and not missingIntents and
579 installedCheck )
580 if not intentAddResult:
581 main.log.error( "Error in pushing host intents to ONOS" )
582
583 main.step( "Intent Anti-Entropy dispersion" )
584 for i in range(100):
585 correct = True
586 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700587 for cli in main.CLIs:
Jon Hall85794ff2015-07-08 14:12:30 -0700588 onosIds = []
589 ids = cli.getAllIntentsId()
590 onosIds.append( ids )
591 main.log.debug( "Intents in " + cli.name + ": " +
592 str( sorted( onosIds ) ) )
593 if sorted( ids ) != sorted( intentIds ):
594 main.log.warn( "Set of intent IDs doesn't match" )
595 correct = False
596 break
597 else:
598 intents = json.loads( cli.intents() )
599 for intent in intents:
600 if intent[ 'state' ] != "INSTALLED":
601 main.log.warn( "Intent " + intent[ 'id' ] +
602 " is " + intent[ 'state' ] )
603 correct = False
604 break
605 if correct:
606 break
607 else:
608 time.sleep(1)
609 if not intentStop:
610 intentStop = time.time()
611 global gossipTime
612 gossipTime = intentStop - intentStart
613 main.log.info( "It took about " + str( gossipTime ) +
614 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700615 gossipPeriod = int( main.params['timers']['gossip'] )
616 maxGossipTime = gossipPeriod * len( main.nodes )
Jon Hall85794ff2015-07-08 14:12:30 -0700617 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700618 expect=maxGossipTime, actual=gossipTime,
Jon Hall85794ff2015-07-08 14:12:30 -0700619 onpass="ECM anti-entropy for intents worked within " +
620 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700621 onfail="Intent ECM anti-entropy took too long. " +
622 "Expected time:{}, Actual time:{}".format( maxGossipTime,
623 gossipTime ) )
624 if gossipTime <= maxGossipTime:
Jon Hall85794ff2015-07-08 14:12:30 -0700625 intentAddResult = True
626
627 if not intentAddResult or "key" in pendingMap:
628 import time
629 installedCheck = True
630 main.log.info( "Sleeping 60 seconds to see if intents are found" )
631 time.sleep( 60 )
632 onosIds = main.ONOScli1.getAllIntentsId()
633 main.log.info( "Submitted intents: " + str( intentIds ) )
634 main.log.info( "Intents in ONOS: " + str( onosIds ) )
635 # Print the intent states
636 intents = main.ONOScli1.intents()
637 intentStates = []
638 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
639 count = 0
640 try:
641 for intent in json.loads( intents ):
642 # Iter through intents of a node
643 state = intent.get( 'state', None )
644 if "INSTALLED" not in state:
645 installedCheck = False
646 intentId = intent.get( 'id', None )
647 intentStates.append( ( intentId, state ) )
648 except ( ValueError, TypeError ):
649 main.log.exception( "Error parsing intents" )
650 # add submitted intents not in the store
651 tmplist = [ i for i, s in intentStates ]
652 for i in intentIds:
653 if i not in tmplist:
654 intentStates.append( ( i, " - " ) )
655 intentStates.sort()
656 for i, s in intentStates:
657 count += 1
658 main.log.info( "%-6s%-15s%-15s" %
659 ( str( count ), str( i ), str( s ) ) )
660 leaders = main.ONOScli1.leaders()
661 try:
662 missing = False
663 if leaders:
664 parsedLeaders = json.loads( leaders )
665 main.log.warn( json.dumps( parsedLeaders,
666 sort_keys=True,
667 indent=4,
668 separators=( ',', ': ' ) ) )
669 # check for all intent partitions
670 # check for election
671 topics = []
672 for i in range( 14 ):
673 topics.append( "intent-partition-" + str( i ) )
674 # FIXME: this should only be after we start the app
675 topics.append( "org.onosproject.election" )
676 main.log.debug( topics )
677 ONOStopics = [ j['topic'] for j in parsedLeaders ]
678 for topic in topics:
679 if topic not in ONOStopics:
680 main.log.error( "Error: " + topic +
681 " not in leaders" )
682 missing = True
683 else:
684 main.log.error( "leaders() returned None" )
685 except ( ValueError, TypeError ):
686 main.log.exception( "Error parsing leaders" )
687 main.log.error( repr( leaders ) )
688 # Check all nodes
689 if missing:
690 response = main.ONOScli1.leaders( jsonFormat=False)
691 main.log.warn( "ONOS1 leaders output: \n" +
692 str( response ) )
693 partitions = main.ONOScli1.partitions()
694 try:
695 if partitions :
696 parsedPartitions = json.loads( partitions )
697 main.log.warn( json.dumps( parsedPartitions,
698 sort_keys=True,
699 indent=4,
700 separators=( ',', ': ' ) ) )
701 # TODO check for a leader in all paritions
702 # TODO check for consistency among nodes
703 else:
704 main.log.error( "partitions() returned None" )
705 except ( ValueError, TypeError ):
706 main.log.exception( "Error parsing partitions" )
707 main.log.error( repr( partitions ) )
708 pendingMap = main.ONOScli1.pendingMap()
709 try:
710 if pendingMap :
711 parsedPending = json.loads( pendingMap )
712 main.log.warn( json.dumps( parsedPending,
713 sort_keys=True,
714 indent=4,
715 separators=( ',', ': ' ) ) )
716 # TODO check something here?
717 else:
718 main.log.error( "pendingMap() returned None" )
719 except ( ValueError, TypeError ):
720 main.log.exception( "Error parsing pending map" )
721 main.log.error( repr( pendingMap ) )
722
723 def CASE4( self, main ):
724 """
725 Ping across added host intents
726 """
727 import json
728 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700729 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700730 assert main, "main not defined"
731 assert utilities.assert_equals, "utilities.assert_equals not defined"
732 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700733 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700734 "functionality and check the state of " +\
735 "the intent"
736 main.step( "Ping across added host intents" )
737 PingResult = main.TRUE
738 for i in range( 8, 18 ):
739 ping = main.Mininet1.pingHost( src="h" + str( i ),
740 target="h" + str( i + 10 ) )
741 PingResult = PingResult and ping
742 if ping == main.FALSE:
743 main.log.warn( "Ping failed between h" + str( i ) +
744 " and h" + str( i + 10 ) )
745 elif ping == main.TRUE:
746 main.log.info( "Ping test passed!" )
747 # Don't set PingResult or you'd override failures
748 if PingResult == main.FALSE:
749 main.log.error(
750 "Intents have not been installed correctly, pings failed." )
751 # TODO: pretty print
752 main.log.warn( "ONOS1 intents: " )
753 try:
754 tmpIntents = main.ONOScli1.intents()
755 main.log.warn( json.dumps( json.loads( tmpIntents ),
756 sort_keys=True,
757 indent=4,
758 separators=( ',', ': ' ) ) )
759 except ( ValueError, TypeError ):
760 main.log.warn( repr( tmpIntents ) )
761 utilities.assert_equals(
762 expect=main.TRUE,
763 actual=PingResult,
764 onpass="Intents have been installed correctly and pings work",
765 onfail="Intents have not been installed correctly, pings failed." )
766
767 main.step( "Check Intent state" )
768 installedCheck = True
769 # Print the intent states
770 intents = main.ONOScli1.intents()
771 intentStates = []
772 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
773 count = 0
774 # Iter through intents of a node
775 try:
776 for intent in json.loads( intents ):
777 state = intent.get( 'state', None )
778 if "INSTALLED" not in state:
779 installedCheck = False
780 intentId = intent.get( 'id', None )
781 intentStates.append( ( intentId, state ) )
782 except ( ValueError, TypeError ):
783 main.log.exception( "Error parsing intents." )
784 # Print states
785 intentStates.sort()
786 for i, s in intentStates:
787 count += 1
788 main.log.info( "%-6s%-15s%-15s" %
789 ( str( count ), str( i ), str( s ) ) )
790 utilities.assert_equals( expect=True, actual=installedCheck,
791 onpass="Intents are all INSTALLED",
792 onfail="Intents are not all in " +
793 "INSTALLED state" )
794
795 main.step( "Check leadership of topics" )
796 leaders = main.ONOScli1.leaders()
797 topicCheck = main.TRUE
798 try:
799 if leaders:
800 parsedLeaders = json.loads( leaders )
801 main.log.warn( json.dumps( parsedLeaders,
802 sort_keys=True,
803 indent=4,
804 separators=( ',', ': ' ) ) )
805 # check for all intent partitions
806 # check for election
807 # TODO: Look at Devices as topics now that it uses this system
808 topics = []
809 for i in range( 14 ):
810 topics.append( "intent-partition-" + str( i ) )
811 # FIXME: this should only be after we start the app
812 # FIXME: topics.append( "org.onosproject.election" )
813 # Print leaders output
814 main.log.debug( topics )
815 ONOStopics = [ j['topic'] for j in parsedLeaders ]
816 for topic in topics:
817 if topic not in ONOStopics:
818 main.log.error( "Error: " + topic +
819 " not in leaders" )
820 topicCheck = main.FALSE
821 else:
822 main.log.error( "leaders() returned None" )
823 topicCheck = main.FALSE
824 except ( ValueError, TypeError ):
825 topicCheck = main.FALSE
826 main.log.exception( "Error parsing leaders" )
827 main.log.error( repr( leaders ) )
828 # TODO: Check for a leader of these topics
829 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
830 onpass="intent Partitions is in leaders",
831 onfail="Some topics were lost " )
832 # Print partitions
833 partitions = main.ONOScli1.partitions()
834 try:
835 if partitions :
836 parsedPartitions = json.loads( partitions )
837 main.log.warn( json.dumps( parsedPartitions,
838 sort_keys=True,
839 indent=4,
840 separators=( ',', ': ' ) ) )
841 # TODO check for a leader in all paritions
842 # TODO check for consistency among nodes
843 else:
844 main.log.error( "partitions() returned None" )
845 except ( ValueError, TypeError ):
846 main.log.exception( "Error parsing partitions" )
847 main.log.error( repr( partitions ) )
848 # Print Pending Map
849 pendingMap = main.ONOScli1.pendingMap()
850 try:
851 if pendingMap :
852 parsedPending = json.loads( pendingMap )
853 main.log.warn( json.dumps( parsedPending,
854 sort_keys=True,
855 indent=4,
856 separators=( ',', ': ' ) ) )
857 # TODO check something here?
858 else:
859 main.log.error( "pendingMap() returned None" )
860 except ( ValueError, TypeError ):
861 main.log.exception( "Error parsing pending map" )
862 main.log.error( repr( pendingMap ) )
863
864 if not installedCheck:
865 main.log.info( "Waiting 60 seconds to see if the state of " +
866 "intents change" )
867 time.sleep( 60 )
868 # Print the intent states
869 intents = main.ONOScli1.intents()
870 intentStates = []
871 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
872 count = 0
873 # Iter through intents of a node
874 try:
875 for intent in json.loads( intents ):
876 state = intent.get( 'state', None )
877 if "INSTALLED" not in state:
878 installedCheck = False
879 intentId = intent.get( 'id', None )
880 intentStates.append( ( intentId, state ) )
881 except ( ValueError, TypeError ):
882 main.log.exception( "Error parsing intents." )
883 intentStates.sort()
884 for i, s in intentStates:
885 count += 1
886 main.log.info( "%-6s%-15s%-15s" %
887 ( str( count ), str( i ), str( s ) ) )
888 leaders = main.ONOScli1.leaders()
889 try:
890 missing = False
891 if leaders:
892 parsedLeaders = json.loads( leaders )
893 main.log.warn( json.dumps( parsedLeaders,
894 sort_keys=True,
895 indent=4,
896 separators=( ',', ': ' ) ) )
897 # check for all intent partitions
898 # check for election
899 topics = []
900 for i in range( 14 ):
901 topics.append( "intent-partition-" + str( i ) )
902 # FIXME: this should only be after we start the app
903 topics.append( "org.onosproject.election" )
904 main.log.debug( topics )
905 ONOStopics = [ j['topic'] for j in parsedLeaders ]
906 for topic in topics:
907 if topic not in ONOStopics:
908 main.log.error( "Error: " + topic +
909 " not in leaders" )
910 missing = True
911 else:
912 main.log.error( "leaders() returned None" )
913 except ( ValueError, TypeError ):
914 main.log.exception( "Error parsing leaders" )
915 main.log.error( repr( leaders ) )
916 if missing:
917 response = main.ONOScli1.leaders( jsonFormat=False)
918 main.log.warn( "ONOS1 leaders output: \n" +
919 str( response ) )
920 partitions = main.ONOScli1.partitions()
921 try:
922 if partitions :
923 parsedPartitions = json.loads( partitions )
924 main.log.warn( json.dumps( parsedPartitions,
925 sort_keys=True,
926 indent=4,
927 separators=( ',', ': ' ) ) )
928 # TODO check for a leader in all paritions
929 # TODO check for consistency among nodes
930 else:
931 main.log.error( "partitions() returned None" )
932 except ( ValueError, TypeError ):
933 main.log.exception( "Error parsing partitions" )
934 main.log.error( repr( partitions ) )
935 pendingMap = main.ONOScli1.pendingMap()
936 try:
937 if pendingMap :
938 parsedPending = json.loads( pendingMap )
939 main.log.warn( json.dumps( parsedPending,
940 sort_keys=True,
941 indent=4,
942 separators=( ',', ': ' ) ) )
943 # TODO check something here?
944 else:
945 main.log.error( "pendingMap() returned None" )
946 except ( ValueError, TypeError ):
947 main.log.exception( "Error parsing pending map" )
948 main.log.error( repr( pendingMap ) )
949 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -0700950 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700951 main.step( "Wait a minute then ping again" )
952 # the wait is above
953 PingResult = main.TRUE
954 for i in range( 8, 18 ):
955 ping = main.Mininet1.pingHost( src="h" + str( i ),
956 target="h" + str( i + 10 ) )
957 PingResult = PingResult and ping
958 if ping == main.FALSE:
959 main.log.warn( "Ping failed between h" + str( i ) +
960 " and h" + str( i + 10 ) )
961 elif ping == main.TRUE:
962 main.log.info( "Ping test passed!" )
963 # Don't set PingResult or you'd override failures
964 if PingResult == main.FALSE:
965 main.log.error(
966 "Intents have not been installed correctly, pings failed." )
967 # TODO: pretty print
968 main.log.warn( "ONOS1 intents: " )
969 try:
970 tmpIntents = main.ONOScli1.intents()
971 main.log.warn( json.dumps( json.loads( tmpIntents ),
972 sort_keys=True,
973 indent=4,
974 separators=( ',', ': ' ) ) )
975 except ( ValueError, TypeError ):
976 main.log.warn( repr( tmpIntents ) )
977 utilities.assert_equals(
978 expect=main.TRUE,
979 actual=PingResult,
980 onpass="Intents have been installed correctly and pings work",
981 onfail="Intents have not been installed correctly, pings failed." )
982
983 def CASE5( self, main ):
984 """
985 Reading state of ONOS
986 """
987 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700988 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700989 assert main, "main not defined"
990 assert utilities.assert_equals, "utilities.assert_equals not defined"
991
992 main.case( "Setting up and gathering data for current state" )
993 # The general idea for this test case is to pull the state of
994 # ( intents,flows, topology,... ) from each ONOS node
995 # We can then compare them with each other and also with past states
996
997 main.step( "Check that each switch has a master" )
998 global mastershipState
999 mastershipState = '[]'
1000
1001 # Assert that each device has a master
1002 rolesNotNull = main.ONOScli1.rolesNotNull()
1003 utilities.assert_equals(
1004 expect=main.TRUE,
1005 actual=rolesNotNull,
1006 onpass="Each device has a master",
1007 onfail="Some devices don't have a master assigned" )
1008
1009 main.step( "Get the Mastership of each switch" )
1010 ONOS1Mastership = main.ONOScli1.roles()
1011 # TODO: Make this a meaningful check
1012 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1013 main.log.error( "Error in getting ONOS roles" )
1014 main.log.warn(
1015 "ONOS1 mastership response: " +
1016 repr( ONOS1Mastership ) )
1017 consistentMastership = main.FALSE
1018 else:
1019 mastershipState = ONOS1Mastership
1020 consistentMastership = main.TRUE
1021
1022 main.step( "Get the intents from each controller" )
1023 global intentState
1024 intentState = []
1025 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1026 intentCheck = main.FALSE
1027 if "Error" in ONOS1Intents or not ONOS1Intents:
1028 main.log.error( "Error in getting ONOS intents" )
1029 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1030 else:
1031 intentCheck = main.TRUE
1032
1033 main.step( "Get the flows from each controller" )
1034 global flowState
1035 flowState = []
1036 flowCheck = main.FALSE
1037 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
1038 if "Error" in ONOS1Flows or not ONOS1Flows:
1039 main.log.error( "Error in getting ONOS flows" )
1040 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
1041 else:
1042 # TODO: Do a better check, maybe compare flows on switches?
1043 flowState = ONOS1Flows
1044 flowCheck = main.TRUE
1045
1046 main.step( "Get the OF Table entries" )
1047 global flows
1048 flows = []
1049 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001050 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001051 if flowCheck == main.FALSE:
1052 for table in flows:
1053 main.log.warn( table )
1054 # TODO: Compare switch flow tables with ONOS flow tables
1055
1056 main.step( "Collecting topology information from ONOS" )
1057 devices = []
1058 devices.append( main.ONOScli1.devices() )
1059 hosts = []
1060 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1061 ports = []
1062 ports.append( main.ONOScli1.ports() )
1063 links = []
1064 links.append( main.ONOScli1.links() )
1065 clusters = []
1066 clusters.append( main.ONOScli1.clusters() )
1067
1068 main.step( "Each host has an IP address" )
1069 ipResult = main.TRUE
1070 for controller in range( 0, len( hosts ) ):
1071 controllerStr = str( controller + 1 )
1072 for host in hosts[ controller ]:
1073 if host is None or host.get( 'ipAddresses', [] ) == []:
1074 main.log.error(
1075 "DEBUG:Error with host ips on controller" +
1076 controllerStr + ": " + str( host ) )
1077 ipResult = main.FALSE
1078 utilities.assert_equals(
1079 expect=main.TRUE,
1080 actual=ipResult,
1081 onpass="The ips of the hosts aren't empty",
1082 onfail="The ip of at least one host is missing" )
1083
1084 # there should always only be one cluster
1085 main.step( "There is only one dataplane cluster" )
1086 try:
1087 numClusters = len( json.loads( clusters[ 0 ] ) )
1088 except ( ValueError, TypeError ):
1089 main.log.exception( "Error parsing clusters[0]: " +
1090 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001091 numClusters = "ERROR"
Jon Hall85794ff2015-07-08 14:12:30 -07001092 clusterResults = main.FALSE
1093 if numClusters == 1:
1094 clusterResults = main.TRUE
1095 utilities.assert_equals(
1096 expect=1,
1097 actual=numClusters,
1098 onpass="ONOS shows 1 SCC",
1099 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1100
1101 main.step( "Comparing ONOS topology to MN" )
1102 devicesResults = main.TRUE
1103 linksResults = main.TRUE
1104 hostsResults = main.TRUE
1105 mnSwitches = main.Mininet1.getSwitches()
1106 mnLinks = main.Mininet1.getLinks()
1107 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001108 for controller in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07001109 controllerStr = str( controller + 1 )
1110 if devices[ controller ] and ports[ controller ] and\
1111 "Error" not in devices[ controller ] and\
1112 "Error" not in ports[ controller ]:
Jon Hall6e709752016-02-01 13:38:46 -08001113 currentDevicesResult = main.Mininet1.compareSwitches(
1114 mnSwitches,
1115 json.loads( devices[ controller ] ),
1116 json.loads( ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001117 else:
1118 currentDevicesResult = main.FALSE
1119 utilities.assert_equals( expect=main.TRUE,
1120 actual=currentDevicesResult,
1121 onpass="ONOS" + controllerStr +
1122 " Switches view is correct",
1123 onfail="ONOS" + controllerStr +
1124 " Switches view is incorrect" )
1125 if links[ controller ] and "Error" not in links[ controller ]:
1126 currentLinksResult = main.Mininet1.compareLinks(
1127 mnSwitches, mnLinks,
1128 json.loads( links[ controller ] ) )
1129 else:
1130 currentLinksResult = main.FALSE
1131 utilities.assert_equals( expect=main.TRUE,
1132 actual=currentLinksResult,
1133 onpass="ONOS" + controllerStr +
1134 " links view is correct",
1135 onfail="ONOS" + controllerStr +
1136 " links view is incorrect" )
1137
1138 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1139 currentHostsResult = main.Mininet1.compareHosts(
1140 mnHosts,
1141 hosts[ controller ] )
1142 else:
1143 currentHostsResult = main.FALSE
1144 utilities.assert_equals( expect=main.TRUE,
1145 actual=currentHostsResult,
1146 onpass="ONOS" + controllerStr +
1147 " hosts exist in Mininet",
1148 onfail="ONOS" + controllerStr +
1149 " hosts don't match Mininet" )
1150
1151 devicesResults = devicesResults and currentDevicesResult
1152 linksResults = linksResults and currentLinksResult
1153 hostsResults = hostsResults and currentHostsResult
1154
1155 main.step( "Device information is correct" )
1156 utilities.assert_equals(
1157 expect=main.TRUE,
1158 actual=devicesResults,
1159 onpass="Device information is correct",
1160 onfail="Device information is incorrect" )
1161
1162 main.step( "Links are correct" )
1163 utilities.assert_equals(
1164 expect=main.TRUE,
1165 actual=linksResults,
1166 onpass="Link are correct",
1167 onfail="Links are incorrect" )
1168
1169 main.step( "Hosts are correct" )
1170 utilities.assert_equals(
1171 expect=main.TRUE,
1172 actual=hostsResults,
1173 onpass="Hosts are correct",
1174 onfail="Hosts are incorrect" )
1175
1176 def CASE6( self, main ):
1177 """
1178 The Failure case.
1179 """
1180 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001181 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001182 assert main, "main not defined"
1183 assert utilities.assert_equals, "utilities.assert_equals not defined"
1184
1185 # Reset non-persistent variables
1186 try:
1187 iCounterValue = 0
1188 except NameError:
1189 main.log.error( "iCounterValue not defined, setting to 0" )
1190 iCounterValue = 0
1191
1192 main.case( "Restart ONOS node" )
Jon Hall783bbf92015-07-23 14:33:19 -07001193 main.caseExplanation = "Killing ONOS process and restart cli " +\
Jon Hall85794ff2015-07-08 14:12:30 -07001194 "sessions once onos is up."
Jon Hall96091e62015-09-21 17:34:17 -07001195
1196 main.step( "Checking ONOS Logs for errors" )
1197 for node in main.nodes:
1198 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1199 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1200
Jon Hall85794ff2015-07-08 14:12:30 -07001201 main.step( "Killing ONOS processes" )
Jon Halle1a3b752015-07-22 13:02:46 -07001202 killResult = main.ONOSbench.onosKill( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001203 start = time.time()
1204 utilities.assert_equals( expect=main.TRUE, actual=killResult,
1205 onpass="ONOS Killed",
1206 onfail="Error killing ONOS" )
1207
1208 main.step( "Checking if ONOS is up yet" )
1209 count = 0
1210 while count < 10:
Jon Halle1a3b752015-07-22 13:02:46 -07001211 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001212 if onos1Isup == main.TRUE:
1213 elapsed = time.time() - start
1214 break
1215 else:
1216 count = count + 1
1217 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
1218 onpass="ONOS is back up",
1219 onfail="ONOS failed to start" )
1220
1221 main.log.step( "Starting ONOS CLI sessions" )
Jon Halle1a3b752015-07-22 13:02:46 -07001222 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001223 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1224 onpass="ONOS cli startup successful",
1225 onfail="ONOS cli startup failed" )
1226
1227 if elapsed:
1228 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
1229 str( elapsed ) )
1230 main.restartTime = elapsed
1231 else:
1232 main.restartTime = -1
1233 time.sleep( 5 )
1234 # rerun on election apps
1235 main.ONOScli1.electionTestRun()
1236
1237 def CASE7( self, main ):
1238 """
1239 Check state after ONOS failure
1240 """
1241 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001242 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001243 assert main, "main not defined"
1244 assert utilities.assert_equals, "utilities.assert_equals not defined"
1245 main.case( "Running ONOS Constant State Tests" )
Jon Hall6e709752016-02-01 13:38:46 -08001246
Jon Hall85794ff2015-07-08 14:12:30 -07001247 main.step( "Check that each switch has a master" )
1248 # Assert that each device has a master
1249 rolesNotNull = main.ONOScli1.rolesNotNull()
1250 utilities.assert_equals(
1251 expect=main.TRUE,
1252 actual=rolesNotNull,
1253 onpass="Each device has a master",
1254 onfail="Some devices don't have a master assigned" )
1255
1256 main.step( "Check if switch roles are consistent across all nodes" )
1257 ONOS1Mastership = main.ONOScli1.roles()
1258 # FIXME: Refactor this whole case for single instance
1259 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1260 main.log.error( "Error in getting ONOS mastership" )
1261 main.log.warn( "ONOS1 mastership response: " +
1262 repr( ONOS1Mastership ) )
1263 consistentMastership = main.FALSE
1264 else:
1265 consistentMastership = main.TRUE
1266 utilities.assert_equals(
1267 expect=main.TRUE,
1268 actual=consistentMastership,
1269 onpass="Switch roles are consistent across all ONOS nodes",
1270 onfail="ONOS nodes have different views of switch roles" )
1271
1272 description2 = "Compare switch roles from before failure"
1273 main.step( description2 )
1274
1275 currentJson = json.loads( ONOS1Mastership )
1276 oldJson = json.loads( mastershipState )
1277 mastershipCheck = main.TRUE
1278 for i in range( 1, 29 ):
1279 switchDPID = str(
1280 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1281
1282 current = [ switch[ 'master' ] for switch in currentJson
1283 if switchDPID in switch[ 'id' ] ]
1284 old = [ switch[ 'master' ] for switch in oldJson
1285 if switchDPID in switch[ 'id' ] ]
1286 if current == old:
1287 mastershipCheck = mastershipCheck and main.TRUE
1288 else:
1289 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1290 mastershipCheck = main.FALSE
1291 utilities.assert_equals(
1292 expect=main.TRUE,
1293 actual=mastershipCheck,
1294 onpass="Mastership of Switches was not changed",
1295 onfail="Mastership of some switches changed" )
1296 mastershipCheck = mastershipCheck and consistentMastership
1297
1298 main.step( "Get the intents and compare across all nodes" )
1299 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1300 intentCheck = main.FALSE
1301 if "Error" in ONOS1Intents or not ONOS1Intents:
1302 main.log.error( "Error in getting ONOS intents" )
1303 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1304 else:
1305 intentCheck = main.TRUE
1306 utilities.assert_equals(
1307 expect=main.TRUE,
1308 actual=intentCheck,
1309 onpass="Intents are consistent across all ONOS nodes",
1310 onfail="ONOS nodes have different views of intents" )
1311 # Print the intent states
1312 intents = []
1313 intents.append( ONOS1Intents )
1314 intentStates = []
1315 for node in intents: # Iter through ONOS nodes
1316 nodeStates = []
1317 # Iter through intents of a node
1318 for intent in json.loads( node ):
1319 nodeStates.append( intent[ 'state' ] )
1320 intentStates.append( nodeStates )
1321 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1322 main.log.info( dict( out ) )
1323
1324 # NOTE: Store has no durability, so intents are lost across system
1325 # restarts
1326 """
1327 main.step( "Compare current intents with intents before the failure" )
1328 # NOTE: this requires case 5 to pass for intentState to be set.
1329 # maybe we should stop the test if that fails?
1330 sameIntents = main.FALSE
1331 if intentState and intentState == ONOSIntents[ 0 ]:
1332 sameIntents = main.TRUE
1333 main.log.info( "Intents are consistent with before failure" )
1334 # TODO: possibly the states have changed? we may need to figure out
1335 # what the acceptable states are
1336 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1337 sameIntents = main.TRUE
1338 try:
1339 before = json.loads( intentState )
1340 after = json.loads( ONOSIntents[ 0 ] )
1341 for intent in before:
1342 if intent not in after:
1343 sameIntents = main.FALSE
1344 main.log.debug( "Intent is not currently in ONOS " +
1345 "(at least in the same form):" )
1346 main.log.debug( json.dumps( intent ) )
1347 except ( ValueError, TypeError ):
1348 main.log.exception( "Exception printing intents" )
1349 main.log.debug( repr( ONOSIntents[0] ) )
1350 main.log.debug( repr( intentState ) )
1351 if sameIntents == main.FALSE:
1352 try:
1353 main.log.debug( "ONOS intents before: " )
1354 main.log.debug( json.dumps( json.loads( intentState ),
1355 sort_keys=True, indent=4,
1356 separators=( ',', ': ' ) ) )
1357 main.log.debug( "Current ONOS intents: " )
1358 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1359 sort_keys=True, indent=4,
1360 separators=( ',', ': ' ) ) )
1361 except ( ValueError, TypeError ):
1362 main.log.exception( "Exception printing intents" )
1363 main.log.debug( repr( ONOSIntents[0] ) )
1364 main.log.debug( repr( intentState ) )
1365 utilities.assert_equals(
1366 expect=main.TRUE,
1367 actual=sameIntents,
1368 onpass="Intents are consistent with before failure",
1369 onfail="The Intents changed during failure" )
1370 intentCheck = intentCheck and sameIntents
1371 """
1372 main.step( "Get the OF Table entries and compare to before " +
1373 "component failure" )
1374 FlowTables = main.TRUE
Jon Hall85794ff2015-07-08 14:12:30 -07001375 for i in range( 28 ):
1376 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08001377 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
1378 FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
Jon Hall85794ff2015-07-08 14:12:30 -07001379 if FlowTables == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08001380 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
1381
Jon Hall85794ff2015-07-08 14:12:30 -07001382 utilities.assert_equals(
1383 expect=main.TRUE,
1384 actual=FlowTables,
1385 onpass="No changes were found in the flow tables",
1386 onfail="Changes were found in the flow tables" )
1387
1388 main.step( "Leadership Election is still functional" )
1389 # Test of LeadershipElection
1390
Jon Halle1a3b752015-07-22 13:02:46 -07001391 leader = main.nodes[0].ip_address
Jon Hall85794ff2015-07-08 14:12:30 -07001392 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001393 for controller in range( 1, main.numCtrls + 1 ):
Jon Hall85794ff2015-07-08 14:12:30 -07001394 # loop through ONOScli handlers
1395 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
1396 leaderN = node.electionTestLeader()
1397 # verify leader is ONOS1
1398 # NOTE even though we restarted ONOS, it is the only one so onos 1
1399 # must be leader
1400 if leaderN == leader:
1401 # all is well
1402 pass
1403 elif leaderN == main.FALSE:
1404 # error in response
1405 main.log.error( "Something is wrong with " +
1406 "electionTestLeader function, check the" +
1407 " error logs" )
1408 leaderResult = main.FALSE
1409 elif leader != leaderN:
1410 leaderResult = main.FALSE
1411 main.log.error( "ONOS" + str( controller ) + " sees " +
1412 str( leaderN ) +
1413 " as the leader of the election app. " +
1414 "Leader should be " + str( leader ) )
1415 utilities.assert_equals(
1416 expect=main.TRUE,
1417 actual=leaderResult,
1418 onpass="Leadership election passed",
1419 onfail="Something went wrong with Leadership election" )
1420
1421 def CASE8( self, main ):
1422 """
1423 Compare topo
1424 """
1425 import json
1426 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001427 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001428 assert main, "main not defined"
1429 assert utilities.assert_equals, "utilities.assert_equals not defined"
1430
1431 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07001432 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall85794ff2015-07-08 14:12:30 -07001433 " and ONOS"
Jon Hall85794ff2015-07-08 14:12:30 -07001434 topoResult = main.FALSE
1435 elapsed = 0
1436 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08001437 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall85794ff2015-07-08 14:12:30 -07001438 startTime = time.time()
1439 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08001440 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07001441 devicesResults = main.TRUE
1442 linksResults = main.TRUE
1443 hostsResults = main.TRUE
1444 hostAttachmentResults = True
Jon Hall85794ff2015-07-08 14:12:30 -07001445 count += 1
1446 cliStart = time.time()
1447 devices = []
1448 devices.append( main.ONOScli1.devices() )
1449 hosts = []
1450 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1451 ipResult = main.TRUE
1452 for controller in range( 0, len( hosts ) ):
1453 controllerStr = str( controller + 1 )
1454 for host in hosts[ controller ]:
1455 if host is None or host.get( 'ipAddresses', [] ) == []:
1456 main.log.error(
1457 "DEBUG:Error with host ips on controller" +
1458 controllerStr + ": " + str( host ) )
1459 ipResult = main.FALSE
1460 ports = []
1461 ports.append( main.ONOScli1.ports() )
1462 links = []
1463 links.append( main.ONOScli1.links() )
1464 clusters = []
1465 clusters.append( main.ONOScli1.clusters() )
1466
1467 elapsed = time.time() - startTime
1468 cliTime = time.time() - cliStart
1469 print "CLI time: " + str( cliTime )
1470
1471 mnSwitches = main.Mininet1.getSwitches()
1472 mnLinks = main.Mininet1.getLinks()
1473 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001474 for controller in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07001475 controllerStr = str( controller + 1 )
1476 if devices[ controller ] and ports[ controller ] and\
1477 "Error" not in devices[ controller ] and\
1478 "Error" not in ports[ controller ]:
1479
Jon Hallc6793552016-01-19 14:18:37 -08001480 try:
1481 currentDevicesResult = main.Mininet1.compareSwitches(
1482 mnSwitches,
1483 json.loads( devices[ controller ] ),
1484 json.loads( ports[ controller ] ) )
1485 except ( TypeError, ValueError ) as e:
1486 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
1487 devices[ controller ], ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001488 else:
1489 currentDevicesResult = main.FALSE
1490 utilities.assert_equals( expect=main.TRUE,
1491 actual=currentDevicesResult,
1492 onpass="ONOS" + controllerStr +
1493 " Switches view is correct",
1494 onfail="ONOS" + controllerStr +
1495 " Switches view is incorrect" )
1496
1497 if links[ controller ] and "Error" not in links[ controller ]:
1498 currentLinksResult = main.Mininet1.compareLinks(
1499 mnSwitches, mnLinks,
1500 json.loads( links[ controller ] ) )
1501 else:
1502 currentLinksResult = main.FALSE
1503 utilities.assert_equals( expect=main.TRUE,
1504 actual=currentLinksResult,
1505 onpass="ONOS" + controllerStr +
1506 " links view is correct",
1507 onfail="ONOS" + controllerStr +
1508 " links view is incorrect" )
1509
1510 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1511 currentHostsResult = main.Mininet1.compareHosts(
1512 mnHosts,
1513 hosts[ controller ] )
1514 else:
1515 currentHostsResult = main.FALSE
1516 utilities.assert_equals( expect=main.TRUE,
1517 actual=currentHostsResult,
1518 onpass="ONOS" + controllerStr +
1519 " hosts exist in Mininet",
1520 onfail="ONOS" + controllerStr +
1521 " hosts don't match Mininet" )
1522 # CHECKING HOST ATTACHMENT POINTS
1523 hostAttachment = True
1524 zeroHosts = False
1525 # FIXME: topo-HA/obelisk specific mappings:
1526 # key is mac and value is dpid
1527 mappings = {}
1528 for i in range( 1, 29 ): # hosts 1 through 28
1529 # set up correct variables:
1530 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
1531 if i == 1:
1532 deviceId = "1000".zfill(16)
1533 elif i == 2:
1534 deviceId = "2000".zfill(16)
1535 elif i == 3:
1536 deviceId = "3000".zfill(16)
1537 elif i == 4:
1538 deviceId = "3004".zfill(16)
1539 elif i == 5:
1540 deviceId = "5000".zfill(16)
1541 elif i == 6:
1542 deviceId = "6000".zfill(16)
1543 elif i == 7:
1544 deviceId = "6007".zfill(16)
1545 elif i >= 8 and i <= 17:
1546 dpid = '3' + str( i ).zfill( 3 )
1547 deviceId = dpid.zfill(16)
1548 elif i >= 18 and i <= 27:
1549 dpid = '6' + str( i ).zfill( 3 )
1550 deviceId = dpid.zfill(16)
1551 elif i == 28:
1552 deviceId = "2800".zfill(16)
1553 mappings[ macId ] = deviceId
1554 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1555 if hosts[ controller ] == []:
1556 main.log.warn( "There are no hosts discovered" )
1557 zeroHosts = True
1558 else:
1559 for host in hosts[ controller ]:
1560 mac = None
1561 location = None
1562 device = None
1563 port = None
1564 try:
1565 mac = host.get( 'mac' )
1566 assert mac, "mac field could not be found for this host object"
1567
1568 location = host.get( 'location' )
1569 assert location, "location field could not be found for this host object"
1570
1571 # Trim the protocol identifier off deviceId
1572 device = str( location.get( 'elementId' ) ).split(':')[1]
1573 assert device, "elementId field could not be found for this host location object"
1574
1575 port = location.get( 'port' )
1576 assert port, "port field could not be found for this host location object"
1577
1578 # Now check if this matches where they should be
1579 if mac and device and port:
1580 if str( port ) != "1":
1581 main.log.error( "The attachment port is incorrect for " +
1582 "host " + str( mac ) +
1583 ". Expected: 1 Actual: " + str( port) )
1584 hostAttachment = False
1585 if device != mappings[ str( mac ) ]:
1586 main.log.error( "The attachment device is incorrect for " +
1587 "host " + str( mac ) +
1588 ". Expected: " + mappings[ str( mac ) ] +
1589 " Actual: " + device )
1590 hostAttachment = False
1591 else:
1592 hostAttachment = False
1593 except AssertionError:
1594 main.log.exception( "Json object not as expected" )
1595 main.log.error( repr( host ) )
1596 hostAttachment = False
1597 else:
1598 main.log.error( "No hosts json output or \"Error\"" +
1599 " in output. hosts = " +
1600 repr( hosts[ controller ] ) )
1601 if zeroHosts is False:
1602 hostAttachment = True
1603
Jon Hall85794ff2015-07-08 14:12:30 -07001604 devicesResults = devicesResults and currentDevicesResult
1605 linksResults = linksResults and currentLinksResult
1606 hostsResults = hostsResults and currentHostsResult
1607 hostAttachmentResults = hostAttachmentResults and\
1608 hostAttachment
1609
1610 # "consistent" results don't make sense for single instance
1611 # there should always only be one cluster
1612 numClusters = len( json.loads( clusters[ 0 ] ) )
1613 clusterResults = main.FALSE
1614 if numClusters == 1:
1615 clusterResults = main.TRUE
1616 utilities.assert_equals(
1617 expect=1,
1618 actual=numClusters,
1619 onpass="ONOS shows 1 SCC",
1620 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1621
1622 topoResult = ( devicesResults and linksResults
1623 and hostsResults and ipResult and clusterResults and
1624 hostAttachmentResults )
1625
1626 topoResult = topoResult and int( count <= 2 )
1627 note = "note it takes about " + str( int( cliTime ) ) + \
1628 " seconds for the test to make all the cli calls to fetch " +\
1629 "the topology from each ONOS instance"
1630 main.log.info(
1631 "Very crass estimate for topology discovery/convergence( " +
1632 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1633 str( count ) + " tries" )
1634 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1635 onpass="Topology Check Test successful",
1636 onfail="Topology Check Test NOT successful" )
1637
1638 def CASE9( self, main ):
1639 """
1640 Link s3-s28 down
1641 """
1642 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001643 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001644 assert main, "main not defined"
1645 assert utilities.assert_equals, "utilities.assert_equals not defined"
1646 # NOTE: You should probably run a topology check after this
1647
1648 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1649
1650 description = "Turn off a link to ensure that Link Discovery " +\
1651 "is working properly"
1652 main.case( description )
1653
1654 main.step( "Kill Link between s3 and s28" )
1655 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
1656 main.log.info( "Waiting " + str( linkSleep ) +
1657 " seconds for link down to be discovered" )
1658 time.sleep( linkSleep )
1659 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
1660 onpass="Link down successful",
1661 onfail="Failed to bring link down" )
1662 # TODO do some sort of check here
1663
1664 def CASE10( self, main ):
1665 """
1666 Link s3-s28 up
1667 """
1668 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001669 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001670 assert main, "main not defined"
1671 assert utilities.assert_equals, "utilities.assert_equals not defined"
1672 # NOTE: You should probably run a topology check after this
1673
1674 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1675
1676 description = "Restore a link to ensure that Link Discovery is " + \
1677 "working properly"
1678 main.case( description )
1679
1680 main.step( "Bring link between s3 and s28 back up" )
1681 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
1682 main.log.info( "Waiting " + str( linkSleep ) +
1683 " seconds for link up to be discovered" )
1684 time.sleep( linkSleep )
1685 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
1686 onpass="Link up successful",
1687 onfail="Failed to bring link up" )
1688 # TODO do some sort of check here
1689
1690 def CASE11( self, main ):
1691 """
1692 Switch Down
1693 """
1694 # NOTE: You should probably run a topology check after this
1695 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001696 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001697 assert main, "main not defined"
1698 assert utilities.assert_equals, "utilities.assert_equals not defined"
1699
1700 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1701
1702 description = "Killing a switch to ensure it is discovered correctly"
1703 main.case( description )
1704 switch = main.params[ 'kill' ][ 'switch' ]
1705 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1706
1707 # TODO: Make this switch parameterizable
1708 main.step( "Kill " + switch )
1709 main.log.info( "Deleting " + switch )
1710 main.Mininet1.delSwitch( switch )
1711 main.log.info( "Waiting " + str( switchSleep ) +
1712 " seconds for switch down to be discovered" )
1713 time.sleep( switchSleep )
1714 device = main.ONOScli1.getDevice( dpid=switchDPID )
1715 # Peek at the deleted switch
1716 main.log.warn( str( device ) )
1717 result = main.FALSE
1718 if device and device[ 'available' ] is False:
1719 result = main.TRUE
1720 utilities.assert_equals( expect=main.TRUE, actual=result,
1721 onpass="Kill switch successful",
1722 onfail="Failed to kill switch?" )
1723
1724 def CASE12( self, main ):
1725 """
1726 Switch Up
1727 """
1728 # NOTE: You should probably run a topology check after this
1729 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001730 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001731 assert main, "main not defined"
1732 assert utilities.assert_equals, "utilities.assert_equals not defined"
1733
1734 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1735 switch = main.params[ 'kill' ][ 'switch' ]
1736 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1737 links = main.params[ 'kill' ][ 'links' ].split()
1738 description = "Adding a switch to ensure it is discovered correctly"
1739 main.case( description )
1740
1741 main.step( "Add back " + switch )
1742 main.Mininet1.addSwitch( switch, dpid=switchDPID )
1743 for peer in links:
1744 main.Mininet1.addLink( switch, peer )
1745 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07001746 for i in range( main.numCtrls ):
1747 ipList.append( main.nodes[ i ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001748 main.Mininet1.assignSwController( sw=switch, ip=ipList )
1749 main.log.info( "Waiting " + str( switchSleep ) +
1750 " seconds for switch up to be discovered" )
1751 time.sleep( switchSleep )
1752 device = main.ONOScli1.getDevice( dpid=switchDPID )
1753 # Peek at the deleted switch
1754 main.log.warn( str( device ) )
1755 result = main.FALSE
1756 if device and device[ 'available' ]:
1757 result = main.TRUE
1758 utilities.assert_equals( expect=main.TRUE, actual=result,
1759 onpass="add switch successful",
1760 onfail="Failed to add switch?" )
1761
1762 def CASE13( self, main ):
1763 """
1764 Clean up
1765 """
1766 import os
1767 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001768 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001769 assert main, "main not defined"
1770 assert utilities.assert_equals, "utilities.assert_equals not defined"
1771 # printing colors to terminal
1772 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
1773 'blue': '\033[94m', 'green': '\033[92m',
1774 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
1775 main.case( "Test Cleanup" )
1776 main.step( "Killing tcpdumps" )
1777 main.Mininet2.stopTcpdump()
1778
1779 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07001780 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall85794ff2015-07-08 14:12:30 -07001781 main.step( "Copying MN pcap and ONOS log files to test station" )
1782 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
1783 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07001784 # NOTE: MN Pcap file is being saved to logdir.
1785 # We scp this file as MN and TestON aren't necessarily the same vm
1786
1787 # FIXME: To be replaced with a Jenkin's post script
Jon Hall85794ff2015-07-08 14:12:30 -07001788 # TODO: Load these from params
1789 # NOTE: must end in /
1790 logFolder = "/opt/onos/log/"
1791 logFiles = [ "karaf.log", "karaf.log.1" ]
1792 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07001793 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07001794 for node in main.nodes:
1795 dstName = main.logdir + "/" + node.name + "-" + f
1796 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
1797 logFolder + f, dstName )
Jon Hall85794ff2015-07-08 14:12:30 -07001798 # std*.log's
1799 # NOTE: must end in /
1800 logFolder = "/opt/onos/var/"
1801 logFiles = [ "stderr.log", "stdout.log" ]
1802 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07001803 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07001804 for node in main.nodes:
1805 dstName = main.logdir + "/" + node.name + "-" + f
1806 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
1807 logFolder + f, dstName )
1808 else:
1809 main.log.debug( "skipping saving log files" )
Jon Hall85794ff2015-07-08 14:12:30 -07001810
Jon Hall85794ff2015-07-08 14:12:30 -07001811 main.step( "Stopping Mininet" )
1812 mnResult = main.Mininet1.stopNet()
1813 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
1814 onpass="Mininet stopped",
1815 onfail="MN cleanup NOT successful" )
1816
1817 main.step( "Checking ONOS Logs for errors" )
Jon Hall96091e62015-09-21 17:34:17 -07001818 for node in main.nodes:
1819 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1820 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001821
1822 try:
1823 timerLog = open( main.logdir + "/Timers.csv", 'w')
1824 # Overwrite with empty line and close
1825 labels = "Gossip Intents, Restart"
1826 data = str( gossipTime ) + ", " + str( main.restartTime )
1827 timerLog.write( labels + "\n" + data )
1828 timerLog.close()
1829 except NameError, e:
1830 main.log.exception(e)
1831
1832 def CASE14( self, main ):
1833 """
1834 start election app on all onos nodes
1835 """
Jon Halle1a3b752015-07-22 13:02:46 -07001836 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001837 assert main, "main not defined"
1838 assert utilities.assert_equals, "utilities.assert_equals not defined"
1839
1840 main.case("Start Leadership Election app")
1841 main.step( "Install leadership election app" )
1842 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
1843 utilities.assert_equals(
1844 expect=main.TRUE,
1845 actual=appResult,
1846 onpass="Election app installed",
1847 onfail="Something went wrong with installing Leadership election" )
1848
1849 main.step( "Run for election on each node" )
1850 leaderResult = main.ONOScli1.electionTestRun()
1851 # check for leader
1852 leader = main.ONOScli1.electionTestLeader()
1853 # verify leader is ONOS1
Jon Halle1a3b752015-07-22 13:02:46 -07001854 if leader == main.nodes[0].ip_address:
Jon Hall85794ff2015-07-08 14:12:30 -07001855 # all is well
1856 pass
1857 elif leader is None:
1858 # No leader elected
1859 main.log.error( "No leader was elected" )
1860 leaderResult = main.FALSE
1861 elif leader == main.FALSE:
1862 # error in response
1863 # TODO: add check for "Command not found:" in the driver, this
1864 # means the app isn't loaded
1865 main.log.error( "Something is wrong with electionTestLeader" +
1866 " function, check the error logs" )
1867 leaderResult = main.FALSE
1868 else:
1869 # error in response
1870 main.log.error(
1871 "Unexpected response from electionTestLeader function:'" +
1872 str( leader ) +
1873 "'" )
1874 leaderResult = main.FALSE
1875 utilities.assert_equals(
1876 expect=main.TRUE,
1877 actual=leaderResult,
1878 onpass="Successfully ran for leadership",
1879 onfail="Failed to run for leadership" )
1880
1881 def CASE15( self, main ):
1882 """
1883 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07001884 15.1 Run election on each node
1885 15.2 Check that each node has the same leaders and candidates
1886 15.3 Find current leader and withdraw
1887 15.4 Check that a new node was elected leader
1888 15.5 Check that that new leader was the candidate of old leader
1889 15.6 Run for election on old leader
1890 15.7 Check that oldLeader is a candidate, and leader if only 1 node
1891 15.8 Make sure that the old leader was added to the candidate list
1892
1893 old and new variable prefixes refer to data from before vs after
1894 withdrawl and later before withdrawl vs after re-election
Jon Hall85794ff2015-07-08 14:12:30 -07001895 """
acsmars71adceb2015-08-31 15:09:26 -07001896 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001897 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001898 assert main, "main not defined"
1899 assert utilities.assert_equals, "utilities.assert_equals not defined"
acsmars71adceb2015-08-31 15:09:26 -07001900 assert main.CLIs, "main.CLIs not defined"
1901 assert main.nodes, "main.nodes not defined"
1902
Jon Hall85794ff2015-07-08 14:12:30 -07001903 description = "Check that Leadership Election is still functional"
1904 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07001905 # NOTE: Need to re-run since being a canidate is not persistant
1906 # TODO: add check for "Command not found:" in the driver, this
1907 # means the election test app isn't loaded
acsmars2c2fcdd2015-08-25 17:14:13 -07001908
acsmars71adceb2015-08-31 15:09:26 -07001909 oldLeaders = [] # leaders by node before withdrawl from candidates
1910 newLeaders = [] # leaders by node after withdrawl from candidates
1911 oldAllCandidates = [] # list of lists of each nodes' candidates before
1912 newAllCandidates = [] # list of lists of each nodes' candidates after
1913 oldCandidates = [] # list of candidates from node 0 before withdrawl
1914 newCandidates = [] # list of candidates from node 0 after withdrawl
1915 oldLeader = '' # the old leader from oldLeaders, None if not same
1916 newLeader = '' # the new leaders fron newLoeaders, None if not same
1917 oldLeaderCLI = None # the CLI of the old leader used for re-electing
1918 expectNoLeader = False # True when there is only one leader
1919 if main.numCtrls == 1:
1920 expectNoLeader = True
acsmars2c2fcdd2015-08-25 17:14:13 -07001921
acsmars71adceb2015-08-31 15:09:26 -07001922 main.step( "Run for election on each node" )
1923 electionResult = main.TRUE
1924
1925 for cli in main.CLIs: # run test election on each node
1926 if cli.electionTestRun() == main.FALSE:
1927 electionResult = main.FALSE
1928
1929 utilities.assert_equals(
1930 expect=main.TRUE,
1931 actual=electionResult,
1932 onpass="All nodes successfully ran for leadership",
1933 onfail="At least one node failed to run for leadership" )
1934
acsmars3a72bde2015-09-02 14:16:22 -07001935 if electionResult == main.FALSE:
1936 main.log.error(
1937 "Skipping Test Case because Election Test App isn't loaded" )
1938 main.skipCase()
1939
acsmars71adceb2015-08-31 15:09:26 -07001940 main.step( "Check that each node shows the same leader and candidates" )
1941 sameResult = main.TRUE
1942 failMessage = "Nodes have different leaders"
1943 for cli in main.CLIs:
1944 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
1945 oldAllCandidates.append( node )
1946 oldLeaders.append( node[ 0 ] )
1947 oldCandidates = oldAllCandidates[ 0 ]
1948
1949 # Check that each node has the same leader. Defines oldLeader
1950 if len( set( oldLeaders ) ) != 1:
1951 sameResult = main.FALSE
1952 main.log.error( "More than one leader present:" + str( oldLeaders ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001953 oldLeader = None
1954 else:
acsmars71adceb2015-08-31 15:09:26 -07001955 oldLeader = oldLeaders[ 0 ]
1956
1957 # Check that each node's candidate list is the same
acsmars29233db2015-11-04 11:15:00 -08001958 candidateDiscrepancy = False # Boolean of candidate mismatches
acsmars71adceb2015-08-31 15:09:26 -07001959 for candidates in oldAllCandidates:
1960 if set( candidates ) != set( oldCandidates ):
1961 sameResult = main.FALSE
acsmars29233db2015-11-04 11:15:00 -08001962 candidateDiscrepancy = True
1963
1964 if candidateDiscrepancy:
1965 failMessage += " and candidates"
acsmars71adceb2015-08-31 15:09:26 -07001966
1967 utilities.assert_equals(
1968 expect=main.TRUE,
1969 actual=sameResult,
1970 onpass="Leadership is consistent for the election topic",
1971 onfail=failMessage )
1972
1973 main.step( "Find current leader and withdraw" )
1974 withdrawResult = main.TRUE
1975 # do some sanity checking on leader before using it
1976 if oldLeader is None:
1977 main.log.error( "Leadership isn't consistent." )
1978 withdrawResult = main.FALSE
1979 # Get the CLI of the oldLeader
1980 for i in range( len( main.CLIs ) ):
1981 if oldLeader == main.nodes[ i ].ip_address:
1982 oldLeaderCLI = main.CLIs[ i ]
1983 break
1984 else: # FOR/ELSE statement
1985 main.log.error( "Leader election, could not find current leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07001986 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07001987 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall85794ff2015-07-08 14:12:30 -07001988 utilities.assert_equals(
1989 expect=main.TRUE,
1990 actual=withdrawResult,
1991 onpass="Node was withdrawn from election",
1992 onfail="Node was not withdrawn from election" )
1993
acsmars71adceb2015-08-31 15:09:26 -07001994 main.step( "Check that a new node was elected leader" )
1995
1996 # FIXME: use threads
1997 newLeaderResult = main.TRUE
1998 failMessage = "Nodes have different leaders"
1999
2000 # Get new leaders and candidates
2001 for cli in main.CLIs:
2002 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2003 # elections might no have finished yet
2004 if node[ 0 ] == 'none' and not expectNoLeader:
2005 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2006 "sure elections are complete." )
2007 time.sleep(5)
2008 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2009 # election still isn't done or there is a problem
2010 if node[ 0 ] == 'none':
2011 main.log.error( "No leader was elected on at least 1 node" )
2012 newLeaderResult = main.FALSE
2013 newAllCandidates.append( node )
2014 newLeaders.append( node[ 0 ] )
2015 newCandidates = newAllCandidates[ 0 ]
2016
2017 # Check that each node has the same leader. Defines newLeader
2018 if len( set( newLeaders ) ) != 1:
2019 newLeaderResult = main.FALSE
2020 main.log.error( "Nodes have different leaders: " +
2021 str( newLeaders ) )
2022 newLeader = None
2023 else:
2024 newLeader = newLeaders[ 0 ]
2025
2026 # Check that each node's candidate list is the same
2027 for candidates in newAllCandidates:
2028 if set( candidates ) != set( newCandidates ):
2029 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002030 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002031
2032 # Check that the new leader is not the older leader, which was withdrawn
2033 if newLeader == oldLeader:
2034 newLeaderResult = main.FALSE
2035 main.log.error( "All nodes still see old leader: " + oldLeader +
2036 " as the current leader" )
2037
Jon Hall85794ff2015-07-08 14:12:30 -07002038 utilities.assert_equals(
2039 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002040 actual=newLeaderResult,
2041 onpass="Leadership election passed",
2042 onfail="Something went wrong with Leadership election" )
Jon Hall85794ff2015-07-08 14:12:30 -07002043
acsmars71adceb2015-08-31 15:09:26 -07002044 main.step( "Check that that new leader was the candidate of old leader")
2045 # candidates[ 2 ] should be come the top candidate after withdrawl
2046 correctCandidateResult = main.TRUE
2047 if expectNoLeader:
2048 if newLeader == 'none':
2049 main.log.info( "No leader expected. None found. Pass" )
2050 correctCandidateResult = main.TRUE
2051 else:
2052 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2053 correctCandidateResult = main.FALSE
2054 elif newLeader != oldCandidates[ 2 ]:
2055 correctCandidateResult = main.FALSE
2056 main.log.error( "Candidate " + newLeader + " was elected. " +
2057 oldCandidates[ 2 ] + " should have had priority." )
2058
2059 utilities.assert_equals(
2060 expect=main.TRUE,
2061 actual=correctCandidateResult,
2062 onpass="Correct Candidate Elected",
2063 onfail="Incorrect Candidate Elected" )
2064
Jon Hall85794ff2015-07-08 14:12:30 -07002065 main.step( "Run for election on old leader( just so everyone " +
2066 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002067 if oldLeaderCLI is not None:
2068 runResult = oldLeaderCLI.electionTestRun()
Jon Hall85794ff2015-07-08 14:12:30 -07002069 else:
acsmars71adceb2015-08-31 15:09:26 -07002070 main.log.error( "No old leader to re-elect" )
Jon Hall85794ff2015-07-08 14:12:30 -07002071 runResult = main.FALSE
2072 utilities.assert_equals(
2073 expect=main.TRUE,
2074 actual=runResult,
2075 onpass="App re-ran for election",
2076 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002077 main.step(
2078 "Check that oldLeader is a candidate, and leader if only 1 node" )
2079 # verify leader didn't just change
2080 positionResult = main.TRUE
2081 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
Jon Hall85794ff2015-07-08 14:12:30 -07002082
acsmars71adceb2015-08-31 15:09:26 -07002083 # Reset and reuse the new candidate and leaders lists
2084 newAllCandidates = []
2085 newCandidates = []
2086 newLeaders = []
2087 for cli in main.CLIs:
2088 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2089 if oldLeader not in node: # election might no have finished yet
2090 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2091 "be sure elections are complete" )
2092 time.sleep(5)
2093 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2094 if oldLeader not in node: # election still isn't done, errors
2095 main.log.error(
2096 "Old leader was not elected on at least one node" )
2097 positionResult = main.FALSE
2098 newAllCandidates.append( node )
2099 newLeaders.append( node[ 0 ] )
2100 newCandidates = newAllCandidates[ 0 ]
2101
2102 # Check that each node has the same leader. Defines newLeader
2103 if len( set( newLeaders ) ) != 1:
2104 positionResult = main.FALSE
2105 main.log.error( "Nodes have different leaders: " +
2106 str( newLeaders ) )
2107 newLeader = None
Jon Hall85794ff2015-07-08 14:12:30 -07002108 else:
acsmars71adceb2015-08-31 15:09:26 -07002109 newLeader = newLeaders[ 0 ]
2110
2111 # Check that each node's candidate list is the same
2112 for candidates in newAllCandidates:
2113 if set( candidates ) != set( newCandidates ):
2114 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002115 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002116
2117 # Check that the re-elected node is last on the candidate List
2118 if oldLeader != newCandidates[ -1 ]:
2119 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
2120 str( newCandidates ) )
2121 positionResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07002122
2123 utilities.assert_equals(
2124 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002125 actual=positionResult,
Jon Hall85794ff2015-07-08 14:12:30 -07002126 onpass="Old leader successfully re-ran for election",
2127 onfail="Something went wrong with Leadership election after " +
2128 "the old leader re-ran for election" )
2129
2130 def CASE16( self, main ):
2131 """
2132 Install Distributed Primitives app
2133 """
Jon Halle1a3b752015-07-22 13:02:46 -07002134 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002135 assert main, "main not defined"
2136 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002137 assert main.CLIs, "main.CLIs not defined"
2138 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002139
2140 # Variables for the distributed primitives tests
2141 global pCounterName
2142 global iCounterName
2143 global pCounterValue
2144 global iCounterValue
2145 global onosSet
2146 global onosSetName
2147 pCounterName = "TestON-Partitions"
2148 iCounterName = "TestON-inMemory"
2149 pCounterValue = 0
2150 iCounterValue = 0
2151 onosSet = set([])
2152 onosSetName = "TestON-set"
2153
2154 description = "Install Primitives app"
2155 main.case( description )
2156 main.step( "Install Primitives app" )
2157 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002158 appResults = main.CLIs[0].activateApp( appName )
Jon Hall85794ff2015-07-08 14:12:30 -07002159 utilities.assert_equals( expect=main.TRUE,
2160 actual=appResults,
2161 onpass="Primitives app activated",
2162 onfail="Primitives app not activated" )
2163
2164 def CASE17( self, main ):
2165 """
2166 Check for basic functionality with distributed primitives
2167 """
Jon Hall85794ff2015-07-08 14:12:30 -07002168 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002169 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002170 assert main, "main not defined"
2171 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002172 assert main.CLIs, "main.CLIs not defined"
2173 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002174 assert pCounterName, "pCounterName not defined"
2175 assert iCounterName, "iCounterName not defined"
2176 assert onosSetName, "onosSetName not defined"
2177 # NOTE: assert fails if value is 0/None/Empty/False
2178 try:
2179 pCounterValue
2180 except NameError:
2181 main.log.error( "pCounterValue not defined, setting to 0" )
2182 pCounterValue = 0
2183 try:
2184 iCounterValue
2185 except NameError:
2186 main.log.error( "iCounterValue not defined, setting to 0" )
2187 iCounterValue = 0
2188 try:
2189 onosSet
2190 except NameError:
2191 main.log.error( "onosSet not defined, setting to empty Set" )
2192 onosSet = set([])
2193 # Variables for the distributed primitives tests. These are local only
2194 addValue = "a"
2195 addAllValue = "a b c d e f"
2196 retainValue = "c d e f"
2197
2198 description = "Check for basic functionality with distributed " +\
2199 "primitives"
2200 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07002201 main.caseExplanation = "Test the methods of the distributed " +\
2202 "primitives (counters and sets) throught the cli"
Jon Hall85794ff2015-07-08 14:12:30 -07002203 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07002204 # Partitioned counters
2205 main.step( "Increment then get a default counter on each node" )
Jon Hall85794ff2015-07-08 14:12:30 -07002206 pCounters = []
2207 threads = []
2208 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07002209 for i in range( main.numCtrls ):
2210 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2211 name="counterAddAndGet-" + str( i ),
Jon Hall85794ff2015-07-08 14:12:30 -07002212 args=[ pCounterName ] )
2213 pCounterValue += 1
2214 addedPValues.append( pCounterValue )
2215 threads.append( t )
2216 t.start()
2217
2218 for t in threads:
2219 t.join()
2220 pCounters.append( t.result )
2221 # Check that counter incremented numController times
2222 pCounterResults = True
2223 for i in addedPValues:
2224 tmpResult = i in pCounters
2225 pCounterResults = pCounterResults and tmpResult
2226 if not tmpResult:
2227 main.log.error( str( i ) + " is not in partitioned "
2228 "counter incremented results" )
2229 utilities.assert_equals( expect=True,
2230 actual=pCounterResults,
2231 onpass="Default counter incremented",
2232 onfail="Error incrementing default" +
2233 " counter" )
2234
Jon Halle1a3b752015-07-22 13:02:46 -07002235 main.step( "Get then Increment a default counter on each node" )
2236 pCounters = []
2237 threads = []
2238 addedPValues = []
2239 for i in range( main.numCtrls ):
2240 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2241 name="counterGetAndAdd-" + str( i ),
2242 args=[ pCounterName ] )
2243 addedPValues.append( pCounterValue )
2244 pCounterValue += 1
2245 threads.append( t )
2246 t.start()
2247
2248 for t in threads:
2249 t.join()
2250 pCounters.append( t.result )
2251 # Check that counter incremented numController times
2252 pCounterResults = True
2253 for i in addedPValues:
2254 tmpResult = i in pCounters
2255 pCounterResults = pCounterResults and tmpResult
2256 if not tmpResult:
2257 main.log.error( str( i ) + " is not in partitioned "
2258 "counter incremented results" )
2259 utilities.assert_equals( expect=True,
2260 actual=pCounterResults,
2261 onpass="Default counter incremented",
2262 onfail="Error incrementing default" +
2263 " counter" )
2264
2265 main.step( "Counters we added have the correct values" )
2266 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2267 utilities.assert_equals( expect=main.TRUE,
2268 actual=incrementCheck,
2269 onpass="Added counters are correct",
2270 onfail="Added counters are incorrect" )
2271
2272 main.step( "Add -8 to then get a default counter on each node" )
2273 pCounters = []
2274 threads = []
2275 addedPValues = []
2276 for i in range( main.numCtrls ):
2277 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2278 name="counterIncrement-" + str( i ),
2279 args=[ pCounterName ],
2280 kwargs={ "delta": -8 } )
2281 pCounterValue += -8
2282 addedPValues.append( pCounterValue )
2283 threads.append( t )
2284 t.start()
2285
2286 for t in threads:
2287 t.join()
2288 pCounters.append( t.result )
2289 # Check that counter incremented numController times
2290 pCounterResults = True
2291 for i in addedPValues:
2292 tmpResult = i in pCounters
2293 pCounterResults = pCounterResults and tmpResult
2294 if not tmpResult:
2295 main.log.error( str( i ) + " is not in partitioned "
2296 "counter incremented results" )
2297 utilities.assert_equals( expect=True,
2298 actual=pCounterResults,
2299 onpass="Default counter incremented",
2300 onfail="Error incrementing default" +
2301 " counter" )
2302
2303 main.step( "Add 5 to then get a default counter on each node" )
2304 pCounters = []
2305 threads = []
2306 addedPValues = []
2307 for i in range( main.numCtrls ):
2308 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2309 name="counterIncrement-" + str( i ),
2310 args=[ pCounterName ],
2311 kwargs={ "delta": 5 } )
2312 pCounterValue += 5
2313 addedPValues.append( pCounterValue )
2314 threads.append( t )
2315 t.start()
2316
2317 for t in threads:
2318 t.join()
2319 pCounters.append( t.result )
2320 # Check that counter incremented numController times
2321 pCounterResults = True
2322 for i in addedPValues:
2323 tmpResult = i in pCounters
2324 pCounterResults = pCounterResults and tmpResult
2325 if not tmpResult:
2326 main.log.error( str( i ) + " is not in partitioned "
2327 "counter incremented results" )
2328 utilities.assert_equals( expect=True,
2329 actual=pCounterResults,
2330 onpass="Default counter incremented",
2331 onfail="Error incrementing default" +
2332 " counter" )
2333
2334 main.step( "Get then add 5 to a default counter on each node" )
2335 pCounters = []
2336 threads = []
2337 addedPValues = []
2338 for i in range( main.numCtrls ):
2339 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2340 name="counterIncrement-" + str( i ),
2341 args=[ pCounterName ],
2342 kwargs={ "delta": 5 } )
2343 addedPValues.append( pCounterValue )
2344 pCounterValue += 5
2345 threads.append( t )
2346 t.start()
2347
2348 for t in threads:
2349 t.join()
2350 pCounters.append( t.result )
2351 # Check that counter incremented numController times
2352 pCounterResults = True
2353 for i in addedPValues:
2354 tmpResult = i in pCounters
2355 pCounterResults = pCounterResults and tmpResult
2356 if not tmpResult:
2357 main.log.error( str( i ) + " is not in partitioned "
2358 "counter incremented results" )
2359 utilities.assert_equals( expect=True,
2360 actual=pCounterResults,
2361 onpass="Default counter incremented",
2362 onfail="Error incrementing default" +
2363 " counter" )
2364
2365 main.step( "Counters we added have the correct values" )
2366 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2367 utilities.assert_equals( expect=main.TRUE,
2368 actual=incrementCheck,
2369 onpass="Added counters are correct",
2370 onfail="Added counters are incorrect" )
2371
2372 # In-Memory counters
2373 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall85794ff2015-07-08 14:12:30 -07002374 iCounters = []
2375 addedIValues = []
2376 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002377 for i in range( main.numCtrls ):
2378 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002379 name="icounterIncrement-" + str( i ),
2380 args=[ iCounterName ],
2381 kwargs={ "inMemory": True } )
2382 iCounterValue += 1
2383 addedIValues.append( iCounterValue )
2384 threads.append( t )
2385 t.start()
2386
2387 for t in threads:
2388 t.join()
2389 iCounters.append( t.result )
2390 # Check that counter incremented numController times
2391 iCounterResults = True
2392 for i in addedIValues:
2393 tmpResult = i in iCounters
2394 iCounterResults = iCounterResults and tmpResult
2395 if not tmpResult:
2396 main.log.error( str( i ) + " is not in the in-memory "
2397 "counter incremented results" )
2398 utilities.assert_equals( expect=True,
2399 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07002400 onpass="In-memory counter incremented",
2401 onfail="Error incrementing in-memory" +
Jon Hall85794ff2015-07-08 14:12:30 -07002402 " counter" )
2403
Jon Halle1a3b752015-07-22 13:02:46 -07002404 main.step( "Get then Increment a in-memory counter on each node" )
2405 iCounters = []
2406 threads = []
2407 addedIValues = []
2408 for i in range( main.numCtrls ):
2409 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2410 name="counterGetAndAdd-" + str( i ),
2411 args=[ iCounterName ],
2412 kwargs={ "inMemory": True } )
2413 addedIValues.append( iCounterValue )
2414 iCounterValue += 1
2415 threads.append( t )
2416 t.start()
2417
2418 for t in threads:
2419 t.join()
2420 iCounters.append( t.result )
2421 # Check that counter incremented numController times
2422 iCounterResults = True
2423 for i in addedIValues:
2424 tmpResult = i in iCounters
2425 iCounterResults = iCounterResults and tmpResult
2426 if not tmpResult:
2427 main.log.error( str( i ) + " is not in in-memory "
2428 "counter incremented results" )
2429 utilities.assert_equals( expect=True,
2430 actual=iCounterResults,
2431 onpass="In-memory counter incremented",
2432 onfail="Error incrementing in-memory" +
2433 " counter" )
2434
2435 main.step( "Counters we added have the correct values" )
2436 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2437 utilities.assert_equals( expect=main.TRUE,
2438 actual=incrementCheck,
2439 onpass="Added counters are correct",
2440 onfail="Added counters are incorrect" )
2441
2442 main.step( "Add -8 to then get a in-memory counter on each node" )
2443 iCounters = []
2444 threads = []
2445 addedIValues = []
2446 for i in range( main.numCtrls ):
2447 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2448 name="counterIncrement-" + str( i ),
2449 args=[ iCounterName ],
2450 kwargs={ "delta": -8, "inMemory": True } )
2451 iCounterValue += -8
2452 addedIValues.append( iCounterValue )
2453 threads.append( t )
2454 t.start()
2455
2456 for t in threads:
2457 t.join()
2458 iCounters.append( t.result )
2459 # Check that counter incremented numController times
2460 iCounterResults = True
2461 for i in addedIValues:
2462 tmpResult = i in iCounters
2463 iCounterResults = iCounterResults and tmpResult
2464 if not tmpResult:
2465 main.log.error( str( i ) + " is not in in-memory "
2466 "counter incremented results" )
2467 utilities.assert_equals( expect=True,
2468 actual=pCounterResults,
2469 onpass="In-memory counter incremented",
2470 onfail="Error incrementing in-memory" +
2471 " counter" )
2472
2473 main.step( "Add 5 to then get a in-memory counter on each node" )
2474 iCounters = []
2475 threads = []
2476 addedIValues = []
2477 for i in range( main.numCtrls ):
2478 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2479 name="counterIncrement-" + str( i ),
2480 args=[ iCounterName ],
2481 kwargs={ "delta": 5, "inMemory": True } )
2482 iCounterValue += 5
2483 addedIValues.append( iCounterValue )
2484 threads.append( t )
2485 t.start()
2486
2487 for t in threads:
2488 t.join()
2489 iCounters.append( t.result )
2490 # Check that counter incremented numController times
2491 iCounterResults = True
2492 for i in addedIValues:
2493 tmpResult = i in iCounters
2494 iCounterResults = iCounterResults and tmpResult
2495 if not tmpResult:
2496 main.log.error( str( i ) + " is not in in-memory "
2497 "counter incremented results" )
2498 utilities.assert_equals( expect=True,
2499 actual=pCounterResults,
2500 onpass="In-memory counter incremented",
2501 onfail="Error incrementing in-memory" +
2502 " counter" )
2503
2504 main.step( "Get then add 5 to a in-memory counter on each node" )
2505 iCounters = []
2506 threads = []
2507 addedIValues = []
2508 for i in range( main.numCtrls ):
2509 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2510 name="counterIncrement-" + str( i ),
2511 args=[ iCounterName ],
2512 kwargs={ "delta": 5, "inMemory": True } )
2513 addedIValues.append( iCounterValue )
2514 iCounterValue += 5
2515 threads.append( t )
2516 t.start()
2517
2518 for t in threads:
2519 t.join()
2520 iCounters.append( t.result )
2521 # Check that counter incremented numController times
2522 iCounterResults = True
2523 for i in addedIValues:
2524 tmpResult = i in iCounters
2525 iCounterResults = iCounterResults and tmpResult
2526 if not tmpResult:
2527 main.log.error( str( i ) + " is not in in-memory "
2528 "counter incremented results" )
2529 utilities.assert_equals( expect=True,
2530 actual=iCounterResults,
2531 onpass="In-memory counter incremented",
2532 onfail="Error incrementing in-memory" +
2533 " counter" )
2534
2535 main.step( "Counters we added have the correct values" )
2536 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2537 utilities.assert_equals( expect=main.TRUE,
2538 actual=incrementCheck,
2539 onpass="Added counters are correct",
2540 onfail="Added counters are incorrect" )
2541
Jon Hall85794ff2015-07-08 14:12:30 -07002542 main.step( "Check counters are consistant across nodes" )
Jon Hall57b50432015-10-22 10:20:10 -07002543 onosCounters, consistentCounterResults = main.Counters.consistentCheck()
Jon Hall85794ff2015-07-08 14:12:30 -07002544 utilities.assert_equals( expect=main.TRUE,
2545 actual=consistentCounterResults,
2546 onpass="ONOS counters are consistent " +
2547 "across nodes",
2548 onfail="ONOS Counters are inconsistent " +
2549 "across nodes" )
2550
2551 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07002552 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
2553 incrementCheck = incrementCheck and \
2554 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall85794ff2015-07-08 14:12:30 -07002555 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07002556 actual=incrementCheck,
Jon Hall85794ff2015-07-08 14:12:30 -07002557 onpass="Added counters are correct",
2558 onfail="Added counters are incorrect" )
2559 # DISTRIBUTED SETS
2560 main.step( "Distributed Set get" )
2561 size = len( onosSet )
2562 getResponses = []
2563 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002564 for i in range( main.numCtrls ):
2565 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002566 name="setTestGet-" + str( i ),
2567 args=[ onosSetName ] )
2568 threads.append( t )
2569 t.start()
2570 for t in threads:
2571 t.join()
2572 getResponses.append( t.result )
2573
2574 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002575 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002576 if isinstance( getResponses[ i ], list):
2577 current = set( getResponses[ i ] )
2578 if len( current ) == len( getResponses[ i ] ):
2579 # no repeats
2580 if onosSet != current:
2581 main.log.error( "ONOS" + str( i + 1 ) +
2582 " has incorrect view" +
2583 " of set " + onosSetName + ":\n" +
2584 str( getResponses[ i ] ) )
2585 main.log.debug( "Expected: " + str( onosSet ) )
2586 main.log.debug( "Actual: " + str( current ) )
2587 getResults = main.FALSE
2588 else:
2589 # error, set is not a set
2590 main.log.error( "ONOS" + str( i + 1 ) +
2591 " has repeat elements in" +
2592 " set " + onosSetName + ":\n" +
2593 str( getResponses[ i ] ) )
2594 getResults = main.FALSE
2595 elif getResponses[ i ] == main.ERROR:
2596 getResults = main.FALSE
2597 utilities.assert_equals( expect=main.TRUE,
2598 actual=getResults,
2599 onpass="Set elements are correct",
2600 onfail="Set elements are incorrect" )
2601
2602 main.step( "Distributed Set size" )
2603 sizeResponses = []
2604 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002605 for i in range( main.numCtrls ):
2606 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002607 name="setTestSize-" + str( i ),
2608 args=[ onosSetName ] )
2609 threads.append( t )
2610 t.start()
2611 for t in threads:
2612 t.join()
2613 sizeResponses.append( t.result )
2614
2615 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002616 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002617 if size != sizeResponses[ i ]:
2618 sizeResults = main.FALSE
2619 main.log.error( "ONOS" + str( i + 1 ) +
2620 " expected a size of " + str( size ) +
2621 " for set " + onosSetName +
2622 " but got " + str( sizeResponses[ i ] ) )
2623 utilities.assert_equals( expect=main.TRUE,
2624 actual=sizeResults,
2625 onpass="Set sizes are correct",
2626 onfail="Set sizes are incorrect" )
2627
2628 main.step( "Distributed Set add()" )
2629 onosSet.add( addValue )
2630 addResponses = []
2631 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002632 for i in range( main.numCtrls ):
2633 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002634 name="setTestAdd-" + str( i ),
2635 args=[ onosSetName, addValue ] )
2636 threads.append( t )
2637 t.start()
2638 for t in threads:
2639 t.join()
2640 addResponses.append( t.result )
2641
2642 # main.TRUE = successfully changed the set
2643 # main.FALSE = action resulted in no change in set
2644 # main.ERROR - Some error in executing the function
2645 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002646 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002647 if addResponses[ i ] == main.TRUE:
2648 # All is well
2649 pass
2650 elif addResponses[ i ] == main.FALSE:
2651 # Already in set, probably fine
2652 pass
2653 elif addResponses[ i ] == main.ERROR:
2654 # Error in execution
2655 addResults = main.FALSE
2656 else:
2657 # unexpected result
2658 addResults = main.FALSE
2659 if addResults != main.TRUE:
2660 main.log.error( "Error executing set add" )
2661
2662 # Check if set is still correct
2663 size = len( onosSet )
2664 getResponses = []
2665 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002666 for i in range( main.numCtrls ):
2667 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002668 name="setTestGet-" + str( i ),
2669 args=[ onosSetName ] )
2670 threads.append( t )
2671 t.start()
2672 for t in threads:
2673 t.join()
2674 getResponses.append( t.result )
2675 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002676 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002677 if isinstance( getResponses[ i ], list):
2678 current = set( getResponses[ i ] )
2679 if len( current ) == len( getResponses[ i ] ):
2680 # no repeats
2681 if onosSet != current:
2682 main.log.error( "ONOS" + str( i + 1 ) +
2683 " has incorrect view" +
2684 " of set " + onosSetName + ":\n" +
2685 str( getResponses[ i ] ) )
2686 main.log.debug( "Expected: " + str( onosSet ) )
2687 main.log.debug( "Actual: " + str( current ) )
2688 getResults = main.FALSE
2689 else:
2690 # error, set is not a set
2691 main.log.error( "ONOS" + str( i + 1 ) +
2692 " has repeat elements in" +
2693 " set " + onosSetName + ":\n" +
2694 str( getResponses[ i ] ) )
2695 getResults = main.FALSE
2696 elif getResponses[ i ] == main.ERROR:
2697 getResults = main.FALSE
2698 sizeResponses = []
2699 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002700 for i in range( main.numCtrls ):
2701 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002702 name="setTestSize-" + str( i ),
2703 args=[ onosSetName ] )
2704 threads.append( t )
2705 t.start()
2706 for t in threads:
2707 t.join()
2708 sizeResponses.append( t.result )
2709 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002710 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002711 if size != sizeResponses[ i ]:
2712 sizeResults = main.FALSE
2713 main.log.error( "ONOS" + str( i + 1 ) +
2714 " expected a size of " + str( size ) +
2715 " for set " + onosSetName +
2716 " but got " + str( sizeResponses[ i ] ) )
2717 addResults = addResults and getResults and sizeResults
2718 utilities.assert_equals( expect=main.TRUE,
2719 actual=addResults,
2720 onpass="Set add correct",
2721 onfail="Set add was incorrect" )
2722
2723 main.step( "Distributed Set addAll()" )
2724 onosSet.update( addAllValue.split() )
2725 addResponses = []
2726 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002727 for i in range( main.numCtrls ):
2728 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002729 name="setTestAddAll-" + str( i ),
2730 args=[ onosSetName, addAllValue ] )
2731 threads.append( t )
2732 t.start()
2733 for t in threads:
2734 t.join()
2735 addResponses.append( t.result )
2736
2737 # main.TRUE = successfully changed the set
2738 # main.FALSE = action resulted in no change in set
2739 # main.ERROR - Some error in executing the function
2740 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002741 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002742 if addResponses[ i ] == main.TRUE:
2743 # All is well
2744 pass
2745 elif addResponses[ i ] == main.FALSE:
2746 # Already in set, probably fine
2747 pass
2748 elif addResponses[ i ] == main.ERROR:
2749 # Error in execution
2750 addAllResults = main.FALSE
2751 else:
2752 # unexpected result
2753 addAllResults = main.FALSE
2754 if addAllResults != main.TRUE:
2755 main.log.error( "Error executing set addAll" )
2756
2757 # Check if set is still correct
2758 size = len( onosSet )
2759 getResponses = []
2760 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002761 for i in range( main.numCtrls ):
2762 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002763 name="setTestGet-" + str( i ),
2764 args=[ onosSetName ] )
2765 threads.append( t )
2766 t.start()
2767 for t in threads:
2768 t.join()
2769 getResponses.append( t.result )
2770 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002771 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002772 if isinstance( getResponses[ i ], list):
2773 current = set( getResponses[ i ] )
2774 if len( current ) == len( getResponses[ i ] ):
2775 # no repeats
2776 if onosSet != current:
2777 main.log.error( "ONOS" + str( i + 1 ) +
2778 " has incorrect view" +
2779 " of set " + onosSetName + ":\n" +
2780 str( getResponses[ i ] ) )
2781 main.log.debug( "Expected: " + str( onosSet ) )
2782 main.log.debug( "Actual: " + str( current ) )
2783 getResults = main.FALSE
2784 else:
2785 # error, set is not a set
2786 main.log.error( "ONOS" + str( i + 1 ) +
2787 " has repeat elements in" +
2788 " set " + onosSetName + ":\n" +
2789 str( getResponses[ i ] ) )
2790 getResults = main.FALSE
2791 elif getResponses[ i ] == main.ERROR:
2792 getResults = main.FALSE
2793 sizeResponses = []
2794 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002795 for i in range( main.numCtrls ):
2796 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002797 name="setTestSize-" + str( i ),
2798 args=[ onosSetName ] )
2799 threads.append( t )
2800 t.start()
2801 for t in threads:
2802 t.join()
2803 sizeResponses.append( t.result )
2804 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002805 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002806 if size != sizeResponses[ i ]:
2807 sizeResults = main.FALSE
2808 main.log.error( "ONOS" + str( i + 1 ) +
2809 " expected a size of " + str( size ) +
2810 " for set " + onosSetName +
2811 " but got " + str( sizeResponses[ i ] ) )
2812 addAllResults = addAllResults and getResults and sizeResults
2813 utilities.assert_equals( expect=main.TRUE,
2814 actual=addAllResults,
2815 onpass="Set addAll correct",
2816 onfail="Set addAll was incorrect" )
2817
2818 main.step( "Distributed Set contains()" )
2819 containsResponses = []
2820 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002821 for i in range( main.numCtrls ):
2822 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002823 name="setContains-" + str( i ),
2824 args=[ onosSetName ],
2825 kwargs={ "values": addValue } )
2826 threads.append( t )
2827 t.start()
2828 for t in threads:
2829 t.join()
2830 # NOTE: This is the tuple
2831 containsResponses.append( t.result )
2832
2833 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002834 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002835 if containsResponses[ i ] == main.ERROR:
2836 containsResults = main.FALSE
2837 else:
2838 containsResults = containsResults and\
2839 containsResponses[ i ][ 1 ]
2840 utilities.assert_equals( expect=main.TRUE,
2841 actual=containsResults,
2842 onpass="Set contains is functional",
2843 onfail="Set contains failed" )
2844
2845 main.step( "Distributed Set containsAll()" )
2846 containsAllResponses = []
2847 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002848 for i in range( main.numCtrls ):
2849 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002850 name="setContainsAll-" + str( i ),
2851 args=[ onosSetName ],
2852 kwargs={ "values": addAllValue } )
2853 threads.append( t )
2854 t.start()
2855 for t in threads:
2856 t.join()
2857 # NOTE: This is the tuple
2858 containsAllResponses.append( t.result )
2859
2860 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002861 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002862 if containsResponses[ i ] == main.ERROR:
2863 containsResults = main.FALSE
2864 else:
2865 containsResults = containsResults and\
2866 containsResponses[ i ][ 1 ]
2867 utilities.assert_equals( expect=main.TRUE,
2868 actual=containsAllResults,
2869 onpass="Set containsAll is functional",
2870 onfail="Set containsAll failed" )
2871
2872 main.step( "Distributed Set remove()" )
2873 onosSet.remove( addValue )
2874 removeResponses = []
2875 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002876 for i in range( main.numCtrls ):
2877 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002878 name="setTestRemove-" + str( i ),
2879 args=[ onosSetName, addValue ] )
2880 threads.append( t )
2881 t.start()
2882 for t in threads:
2883 t.join()
2884 removeResponses.append( t.result )
2885
2886 # main.TRUE = successfully changed the set
2887 # main.FALSE = action resulted in no change in set
2888 # main.ERROR - Some error in executing the function
2889 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002890 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002891 if removeResponses[ i ] == main.TRUE:
2892 # All is well
2893 pass
2894 elif removeResponses[ i ] == main.FALSE:
2895 # not in set, probably fine
2896 pass
2897 elif removeResponses[ i ] == main.ERROR:
2898 # Error in execution
2899 removeResults = main.FALSE
2900 else:
2901 # unexpected result
2902 removeResults = main.FALSE
2903 if removeResults != main.TRUE:
2904 main.log.error( "Error executing set remove" )
2905
2906 # Check if set is still correct
2907 size = len( onosSet )
2908 getResponses = []
2909 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002910 for i in range( main.numCtrls ):
2911 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002912 name="setTestGet-" + str( i ),
2913 args=[ onosSetName ] )
2914 threads.append( t )
2915 t.start()
2916 for t in threads:
2917 t.join()
2918 getResponses.append( t.result )
2919 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002920 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002921 if isinstance( getResponses[ i ], list):
2922 current = set( getResponses[ i ] )
2923 if len( current ) == len( getResponses[ i ] ):
2924 # no repeats
2925 if onosSet != current:
2926 main.log.error( "ONOS" + str( i + 1 ) +
2927 " has incorrect view" +
2928 " of set " + onosSetName + ":\n" +
2929 str( getResponses[ i ] ) )
2930 main.log.debug( "Expected: " + str( onosSet ) )
2931 main.log.debug( "Actual: " + str( current ) )
2932 getResults = main.FALSE
2933 else:
2934 # error, set is not a set
2935 main.log.error( "ONOS" + str( i + 1 ) +
2936 " has repeat elements in" +
2937 " set " + onosSetName + ":\n" +
2938 str( getResponses[ i ] ) )
2939 getResults = main.FALSE
2940 elif getResponses[ i ] == main.ERROR:
2941 getResults = main.FALSE
2942 sizeResponses = []
2943 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002944 for i in range( main.numCtrls ):
2945 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002946 name="setTestSize-" + str( i ),
2947 args=[ onosSetName ] )
2948 threads.append( t )
2949 t.start()
2950 for t in threads:
2951 t.join()
2952 sizeResponses.append( t.result )
2953 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002954 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002955 if size != sizeResponses[ i ]:
2956 sizeResults = main.FALSE
2957 main.log.error( "ONOS" + str( i + 1 ) +
2958 " expected a size of " + str( size ) +
2959 " for set " + onosSetName +
2960 " but got " + str( sizeResponses[ i ] ) )
2961 removeResults = removeResults and getResults and sizeResults
2962 utilities.assert_equals( expect=main.TRUE,
2963 actual=removeResults,
2964 onpass="Set remove correct",
2965 onfail="Set remove was incorrect" )
2966
2967 main.step( "Distributed Set removeAll()" )
2968 onosSet.difference_update( addAllValue.split() )
2969 removeAllResponses = []
2970 threads = []
2971 try:
Jon Halle1a3b752015-07-22 13:02:46 -07002972 for i in range( main.numCtrls ):
2973 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002974 name="setTestRemoveAll-" + str( i ),
2975 args=[ onosSetName, addAllValue ] )
2976 threads.append( t )
2977 t.start()
2978 for t in threads:
2979 t.join()
2980 removeAllResponses.append( t.result )
2981 except Exception, e:
2982 main.log.exception(e)
2983
2984 # main.TRUE = successfully changed the set
2985 # main.FALSE = action resulted in no change in set
2986 # main.ERROR - Some error in executing the function
2987 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002988 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07002989 if removeAllResponses[ i ] == main.TRUE:
2990 # All is well
2991 pass
2992 elif removeAllResponses[ i ] == main.FALSE:
2993 # not in set, probably fine
2994 pass
2995 elif removeAllResponses[ i ] == main.ERROR:
2996 # Error in execution
2997 removeAllResults = main.FALSE
2998 else:
2999 # unexpected result
3000 removeAllResults = main.FALSE
3001 if removeAllResults != main.TRUE:
3002 main.log.error( "Error executing set removeAll" )
3003
3004 # Check if set is still correct
3005 size = len( onosSet )
3006 getResponses = []
3007 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003008 for i in range( main.numCtrls ):
3009 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003010 name="setTestGet-" + str( i ),
3011 args=[ onosSetName ] )
3012 threads.append( t )
3013 t.start()
3014 for t in threads:
3015 t.join()
3016 getResponses.append( t.result )
3017 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003018 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003019 if isinstance( getResponses[ i ], list):
3020 current = set( getResponses[ i ] )
3021 if len( current ) == len( getResponses[ i ] ):
3022 # no repeats
3023 if onosSet != current:
3024 main.log.error( "ONOS" + str( i + 1 ) +
3025 " has incorrect view" +
3026 " of set " + onosSetName + ":\n" +
3027 str( getResponses[ i ] ) )
3028 main.log.debug( "Expected: " + str( onosSet ) )
3029 main.log.debug( "Actual: " + str( current ) )
3030 getResults = main.FALSE
3031 else:
3032 # error, set is not a set
3033 main.log.error( "ONOS" + str( i + 1 ) +
3034 " has repeat elements in" +
3035 " set " + onosSetName + ":\n" +
3036 str( getResponses[ i ] ) )
3037 getResults = main.FALSE
3038 elif getResponses[ i ] == main.ERROR:
3039 getResults = main.FALSE
3040 sizeResponses = []
3041 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003042 for i in range( main.numCtrls ):
3043 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003044 name="setTestSize-" + str( i ),
3045 args=[ onosSetName ] )
3046 threads.append( t )
3047 t.start()
3048 for t in threads:
3049 t.join()
3050 sizeResponses.append( t.result )
3051 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003052 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003053 if size != sizeResponses[ i ]:
3054 sizeResults = main.FALSE
3055 main.log.error( "ONOS" + str( i + 1 ) +
3056 " expected a size of " + str( size ) +
3057 " for set " + onosSetName +
3058 " but got " + str( sizeResponses[ i ] ) )
3059 removeAllResults = removeAllResults and getResults and sizeResults
3060 utilities.assert_equals( expect=main.TRUE,
3061 actual=removeAllResults,
3062 onpass="Set removeAll correct",
3063 onfail="Set removeAll was incorrect" )
3064
3065 main.step( "Distributed Set addAll()" )
3066 onosSet.update( addAllValue.split() )
3067 addResponses = []
3068 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003069 for i in range( main.numCtrls ):
3070 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003071 name="setTestAddAll-" + str( i ),
3072 args=[ onosSetName, addAllValue ] )
3073 threads.append( t )
3074 t.start()
3075 for t in threads:
3076 t.join()
3077 addResponses.append( t.result )
3078
3079 # main.TRUE = successfully changed the set
3080 # main.FALSE = action resulted in no change in set
3081 # main.ERROR - Some error in executing the function
3082 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003083 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003084 if addResponses[ i ] == main.TRUE:
3085 # All is well
3086 pass
3087 elif addResponses[ i ] == main.FALSE:
3088 # Already in set, probably fine
3089 pass
3090 elif addResponses[ i ] == main.ERROR:
3091 # Error in execution
3092 addAllResults = main.FALSE
3093 else:
3094 # unexpected result
3095 addAllResults = main.FALSE
3096 if addAllResults != main.TRUE:
3097 main.log.error( "Error executing set addAll" )
3098
3099 # Check if set is still correct
3100 size = len( onosSet )
3101 getResponses = []
3102 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003103 for i in range( main.numCtrls ):
3104 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003105 name="setTestGet-" + str( i ),
3106 args=[ onosSetName ] )
3107 threads.append( t )
3108 t.start()
3109 for t in threads:
3110 t.join()
3111 getResponses.append( t.result )
3112 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003113 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003114 if isinstance( getResponses[ i ], list):
3115 current = set( getResponses[ i ] )
3116 if len( current ) == len( getResponses[ i ] ):
3117 # no repeats
3118 if onosSet != current:
3119 main.log.error( "ONOS" + str( i + 1 ) +
3120 " has incorrect view" +
3121 " of set " + onosSetName + ":\n" +
3122 str( getResponses[ i ] ) )
3123 main.log.debug( "Expected: " + str( onosSet ) )
3124 main.log.debug( "Actual: " + str( current ) )
3125 getResults = main.FALSE
3126 else:
3127 # error, set is not a set
3128 main.log.error( "ONOS" + str( i + 1 ) +
3129 " has repeat elements in" +
3130 " set " + onosSetName + ":\n" +
3131 str( getResponses[ i ] ) )
3132 getResults = main.FALSE
3133 elif getResponses[ i ] == main.ERROR:
3134 getResults = main.FALSE
3135 sizeResponses = []
3136 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003137 for i in range( main.numCtrls ):
3138 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003139 name="setTestSize-" + str( i ),
3140 args=[ onosSetName ] )
3141 threads.append( t )
3142 t.start()
3143 for t in threads:
3144 t.join()
3145 sizeResponses.append( t.result )
3146 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003147 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003148 if size != sizeResponses[ i ]:
3149 sizeResults = main.FALSE
3150 main.log.error( "ONOS" + str( i + 1 ) +
3151 " expected a size of " + str( size ) +
3152 " for set " + onosSetName +
3153 " but got " + str( sizeResponses[ i ] ) )
3154 addAllResults = addAllResults and getResults and sizeResults
3155 utilities.assert_equals( expect=main.TRUE,
3156 actual=addAllResults,
3157 onpass="Set addAll correct",
3158 onfail="Set addAll was incorrect" )
3159
3160 main.step( "Distributed Set clear()" )
3161 onosSet.clear()
3162 clearResponses = []
3163 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003164 for i in range( main.numCtrls ):
3165 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003166 name="setTestClear-" + str( i ),
3167 args=[ onosSetName, " "], # Values doesn't matter
3168 kwargs={ "clear": True } )
3169 threads.append( t )
3170 t.start()
3171 for t in threads:
3172 t.join()
3173 clearResponses.append( t.result )
3174
3175 # main.TRUE = successfully changed the set
3176 # main.FALSE = action resulted in no change in set
3177 # main.ERROR - Some error in executing the function
3178 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003179 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003180 if clearResponses[ i ] == main.TRUE:
3181 # All is well
3182 pass
3183 elif clearResponses[ i ] == main.FALSE:
3184 # Nothing set, probably fine
3185 pass
3186 elif clearResponses[ i ] == main.ERROR:
3187 # Error in execution
3188 clearResults = main.FALSE
3189 else:
3190 # unexpected result
3191 clearResults = main.FALSE
3192 if clearResults != main.TRUE:
3193 main.log.error( "Error executing set clear" )
3194
3195 # Check if set is still correct
3196 size = len( onosSet )
3197 getResponses = []
3198 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003199 for i in range( main.numCtrls ):
3200 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003201 name="setTestGet-" + str( i ),
3202 args=[ onosSetName ] )
3203 threads.append( t )
3204 t.start()
3205 for t in threads:
3206 t.join()
3207 getResponses.append( t.result )
3208 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003209 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003210 if isinstance( getResponses[ i ], list):
3211 current = set( getResponses[ i ] )
3212 if len( current ) == len( getResponses[ i ] ):
3213 # no repeats
3214 if onosSet != current:
3215 main.log.error( "ONOS" + str( i + 1 ) +
3216 " has incorrect view" +
3217 " of set " + onosSetName + ":\n" +
3218 str( getResponses[ i ] ) )
3219 main.log.debug( "Expected: " + str( onosSet ) )
3220 main.log.debug( "Actual: " + str( current ) )
3221 getResults = main.FALSE
3222 else:
3223 # error, set is not a set
3224 main.log.error( "ONOS" + str( i + 1 ) +
3225 " has repeat elements in" +
3226 " set " + onosSetName + ":\n" +
3227 str( getResponses[ i ] ) )
3228 getResults = main.FALSE
3229 elif getResponses[ i ] == main.ERROR:
3230 getResults = main.FALSE
3231 sizeResponses = []
3232 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003233 for i in range( main.numCtrls ):
3234 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003235 name="setTestSize-" + str( i ),
3236 args=[ onosSetName ] )
3237 threads.append( t )
3238 t.start()
3239 for t in threads:
3240 t.join()
3241 sizeResponses.append( t.result )
3242 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003243 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003244 if size != sizeResponses[ i ]:
3245 sizeResults = main.FALSE
3246 main.log.error( "ONOS" + str( i + 1 ) +
3247 " expected a size of " + str( size ) +
3248 " for set " + onosSetName +
3249 " but got " + str( sizeResponses[ i ] ) )
3250 clearResults = clearResults and getResults and sizeResults
3251 utilities.assert_equals( expect=main.TRUE,
3252 actual=clearResults,
3253 onpass="Set clear correct",
3254 onfail="Set clear was incorrect" )
3255
3256 main.step( "Distributed Set addAll()" )
3257 onosSet.update( addAllValue.split() )
3258 addResponses = []
3259 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003260 for i in range( main.numCtrls ):
3261 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003262 name="setTestAddAll-" + str( i ),
3263 args=[ onosSetName, addAllValue ] )
3264 threads.append( t )
3265 t.start()
3266 for t in threads:
3267 t.join()
3268 addResponses.append( t.result )
3269
3270 # main.TRUE = successfully changed the set
3271 # main.FALSE = action resulted in no change in set
3272 # main.ERROR - Some error in executing the function
3273 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003274 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003275 if addResponses[ i ] == main.TRUE:
3276 # All is well
3277 pass
3278 elif addResponses[ i ] == main.FALSE:
3279 # Already in set, probably fine
3280 pass
3281 elif addResponses[ i ] == main.ERROR:
3282 # Error in execution
3283 addAllResults = main.FALSE
3284 else:
3285 # unexpected result
3286 addAllResults = main.FALSE
3287 if addAllResults != main.TRUE:
3288 main.log.error( "Error executing set addAll" )
3289
3290 # Check if set is still correct
3291 size = len( onosSet )
3292 getResponses = []
3293 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003294 for i in range( main.numCtrls ):
3295 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003296 name="setTestGet-" + str( i ),
3297 args=[ onosSetName ] )
3298 threads.append( t )
3299 t.start()
3300 for t in threads:
3301 t.join()
3302 getResponses.append( t.result )
3303 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003304 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003305 if isinstance( getResponses[ i ], list):
3306 current = set( getResponses[ i ] )
3307 if len( current ) == len( getResponses[ i ] ):
3308 # no repeats
3309 if onosSet != current:
3310 main.log.error( "ONOS" + str( i + 1 ) +
3311 " has incorrect view" +
3312 " of set " + onosSetName + ":\n" +
3313 str( getResponses[ i ] ) )
3314 main.log.debug( "Expected: " + str( onosSet ) )
3315 main.log.debug( "Actual: " + str( current ) )
3316 getResults = main.FALSE
3317 else:
3318 # error, set is not a set
3319 main.log.error( "ONOS" + str( i + 1 ) +
3320 " has repeat elements in" +
3321 " set " + onosSetName + ":\n" +
3322 str( getResponses[ i ] ) )
3323 getResults = main.FALSE
3324 elif getResponses[ i ] == main.ERROR:
3325 getResults = main.FALSE
3326 sizeResponses = []
3327 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003328 for i in range( main.numCtrls ):
3329 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003330 name="setTestSize-" + str( i ),
3331 args=[ onosSetName ] )
3332 threads.append( t )
3333 t.start()
3334 for t in threads:
3335 t.join()
3336 sizeResponses.append( t.result )
3337 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003338 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003339 if size != sizeResponses[ i ]:
3340 sizeResults = main.FALSE
3341 main.log.error( "ONOS" + str( i + 1 ) +
3342 " expected a size of " + str( size ) +
3343 " for set " + onosSetName +
3344 " but got " + str( sizeResponses[ i ] ) )
3345 addAllResults = addAllResults and getResults and sizeResults
3346 utilities.assert_equals( expect=main.TRUE,
3347 actual=addAllResults,
3348 onpass="Set addAll correct",
3349 onfail="Set addAll was incorrect" )
3350
3351 main.step( "Distributed Set retain()" )
3352 onosSet.intersection_update( retainValue.split() )
3353 retainResponses = []
3354 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003355 for i in range( main.numCtrls ):
3356 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003357 name="setTestRetain-" + str( i ),
3358 args=[ onosSetName, retainValue ],
3359 kwargs={ "retain": True } )
3360 threads.append( t )
3361 t.start()
3362 for t in threads:
3363 t.join()
3364 retainResponses.append( t.result )
3365
3366 # main.TRUE = successfully changed the set
3367 # main.FALSE = action resulted in no change in set
3368 # main.ERROR - Some error in executing the function
3369 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003370 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003371 if retainResponses[ i ] == main.TRUE:
3372 # All is well
3373 pass
3374 elif retainResponses[ i ] == main.FALSE:
3375 # Already in set, probably fine
3376 pass
3377 elif retainResponses[ i ] == main.ERROR:
3378 # Error in execution
3379 retainResults = main.FALSE
3380 else:
3381 # unexpected result
3382 retainResults = main.FALSE
3383 if retainResults != main.TRUE:
3384 main.log.error( "Error executing set retain" )
3385
3386 # Check if set is still correct
3387 size = len( onosSet )
3388 getResponses = []
3389 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003390 for i in range( main.numCtrls ):
3391 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003392 name="setTestGet-" + str( i ),
3393 args=[ onosSetName ] )
3394 threads.append( t )
3395 t.start()
3396 for t in threads:
3397 t.join()
3398 getResponses.append( t.result )
3399 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003400 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003401 if isinstance( getResponses[ i ], list):
3402 current = set( getResponses[ i ] )
3403 if len( current ) == len( getResponses[ i ] ):
3404 # no repeats
3405 if onosSet != current:
3406 main.log.error( "ONOS" + str( i + 1 ) +
3407 " has incorrect view" +
3408 " of set " + onosSetName + ":\n" +
3409 str( getResponses[ i ] ) )
3410 main.log.debug( "Expected: " + str( onosSet ) )
3411 main.log.debug( "Actual: " + str( current ) )
3412 getResults = main.FALSE
3413 else:
3414 # error, set is not a set
3415 main.log.error( "ONOS" + str( i + 1 ) +
3416 " has repeat elements in" +
3417 " set " + onosSetName + ":\n" +
3418 str( getResponses[ i ] ) )
3419 getResults = main.FALSE
3420 elif getResponses[ i ] == main.ERROR:
3421 getResults = main.FALSE
3422 sizeResponses = []
3423 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003424 for i in range( main.numCtrls ):
3425 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003426 name="setTestSize-" + str( i ),
3427 args=[ onosSetName ] )
3428 threads.append( t )
3429 t.start()
3430 for t in threads:
3431 t.join()
3432 sizeResponses.append( t.result )
3433 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003434 for i in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07003435 if size != sizeResponses[ i ]:
3436 sizeResults = main.FALSE
3437 main.log.error( "ONOS" + str( i + 1 ) +
3438 " expected a size of " +
3439 str( size ) + " for set " + onosSetName +
3440 " but got " + str( sizeResponses[ i ] ) )
3441 retainResults = retainResults and getResults and sizeResults
3442 utilities.assert_equals( expect=main.TRUE,
3443 actual=retainResults,
3444 onpass="Set retain correct",
3445 onfail="Set retain was incorrect" )
3446
Jon Hall2a5002c2015-08-21 16:49:11 -07003447 # Transactional maps
3448 main.step( "Partitioned Transactional maps put" )
3449 tMapValue = "Testing"
3450 numKeys = 100
3451 putResult = True
3452 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
3453 if len( putResponses ) == 100:
3454 for i in putResponses:
3455 if putResponses[ i ][ 'value' ] != tMapValue:
3456 putResult = False
3457 else:
3458 putResult = False
3459 if not putResult:
3460 main.log.debug( "Put response values: " + str( putResponses ) )
3461 utilities.assert_equals( expect=True,
3462 actual=putResult,
3463 onpass="Partitioned Transactional Map put successful",
3464 onfail="Partitioned Transactional Map put values are incorrect" )
3465
3466 main.step( "Partitioned Transactional maps get" )
3467 getCheck = True
3468 for n in range( 1, numKeys + 1 ):
3469 getResponses = []
3470 threads = []
3471 valueCheck = True
3472 for i in range( main.numCtrls ):
3473 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
3474 name="TMap-get-" + str( i ),
3475 args=[ "Key" + str ( n ) ] )
3476 threads.append( t )
3477 t.start()
3478 for t in threads:
3479 t.join()
3480 getResponses.append( t.result )
3481 for node in getResponses:
3482 if node != tMapValue:
3483 valueCheck = False
3484 if not valueCheck:
3485 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
3486 main.log.warn( getResponses )
3487 getCheck = getCheck and valueCheck
3488 utilities.assert_equals( expect=True,
3489 actual=getCheck,
3490 onpass="Partitioned Transactional Map get values were correct",
3491 onfail="Partitioned Transactional Map values incorrect" )
3492
3493 main.step( "In-memory Transactional maps put" )
3494 tMapValue = "Testing"
3495 numKeys = 100
3496 putResult = True
3497 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
3498 if len( putResponses ) == 100:
3499 for i in putResponses:
3500 if putResponses[ i ][ 'value' ] != tMapValue:
3501 putResult = False
3502 else:
3503 putResult = False
3504 if not putResult:
3505 main.log.debug( "Put response values: " + str( putResponses ) )
3506 utilities.assert_equals( expect=True,
3507 actual=putResult,
3508 onpass="In-Memory Transactional Map put successful",
3509 onfail="In-Memory Transactional Map put values are incorrect" )
3510
3511 main.step( "In-Memory Transactional maps get" )
3512 getCheck = True
3513 for n in range( 1, numKeys + 1 ):
3514 getResponses = []
3515 threads = []
3516 valueCheck = True
3517 for i in range( main.numCtrls ):
3518 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
3519 name="TMap-get-" + str( i ),
3520 args=[ "Key" + str ( n ) ],
3521 kwargs={ "inMemory": True } )
3522 threads.append( t )
3523 t.start()
3524 for t in threads:
3525 t.join()
3526 getResponses.append( t.result )
3527 for node in getResponses:
3528 if node != tMapValue:
3529 valueCheck = False
3530 if not valueCheck:
3531 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
3532 main.log.warn( getResponses )
3533 getCheck = getCheck and valueCheck
3534 utilities.assert_equals( expect=True,
3535 actual=getCheck,
3536 onpass="In-Memory Transactional Map get values were correct",
3537 onfail="In-Memory Transactional Map values incorrect" )