blob: d77450a95910cb7c31e2a9d6b7e4eccbbd49538a [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAclusterRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hall5cf14d52015-07-16 12:15:19 -070051 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
52 "initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
70 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070071 global ONOS1Port
72 global ONOS2Port
73 global ONOS3Port
74 global ONOS4Port
75 global ONOS5Port
76 global ONOS6Port
77 global ONOS7Port
78 # These are for csv plotting in jenkins
79 global labels
80 global data
81 labels = []
82 data = []
83
84 # FIXME: just get controller port from params?
85 # TODO: do we really need all these?
86 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
87 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
88 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
89 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
90 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
91 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
92 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
93
Jon Halle1a3b752015-07-22 13:02:46 -070094 try:
95 fileName = "Counters"
96 path = main.params[ 'imports' ][ 'path' ]
97 main.Counters = imp.load_source( fileName,
98 path + fileName + ".py" )
99 except Exception as e:
100 main.log.exception( e )
101 main.cleanup()
102 main.exit()
103
104 main.CLIs = []
105 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700106 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700107 for i in range( 1, main.numCtrls + 1 ):
108 try:
109 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
110 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
111 ipList.append( main.nodes[ -1 ].ip_address )
112 except AttributeError:
113 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700114
115 main.step( "Create cell file" )
116 cellAppString = main.params[ 'ENV' ][ 'appString' ]
117 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
118 main.Mininet1.ip_address,
119 cellAppString, ipList )
120 main.step( "Applying cell variable to environment" )
121 cellResult = main.ONOSbench.setCell( cellName )
122 verifyResult = main.ONOSbench.verifyCell()
123
124 # FIXME:this is short term fix
125 main.log.info( "Removing raft logs" )
126 main.ONOSbench.onosRemoveRaftLogs()
127
128 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700129 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700130 main.ONOSbench.onosUninstall( node.ip_address )
131
132 # Make sure ONOS is DEAD
133 main.log.info( "Killing any ONOS processes" )
134 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700135 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700136 killed = main.ONOSbench.onosKill( node.ip_address )
137 killResults = killResults and killed
138
139 cleanInstallResult = main.TRUE
140 gitPullResult = main.TRUE
141
142 main.step( "Starting Mininet" )
143 # scp topo file to mininet
144 # TODO: move to params?
145 topoName = "obelisk.py"
146 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700147 main.ONOSbench.scp( main.Mininet1,
148 filePath + topoName,
149 main.Mininet1.home,
150 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700151 mnResult = main.Mininet1.startNet( )
152 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
153 onpass="Mininet Started",
154 onfail="Error starting Mininet" )
155
156 main.step( "Git checkout and pull " + gitBranch )
157 if PULLCODE:
158 main.ONOSbench.gitCheckout( gitBranch )
159 gitPullResult = main.ONOSbench.gitPull()
160 # values of 1 or 3 are good
161 utilities.assert_lesser( expect=0, actual=gitPullResult,
162 onpass="Git pull successful",
163 onfail="Git pull failed" )
164 main.ONOSbench.getVersion( report=True )
165
166 main.step( "Using mvn clean install" )
167 cleanInstallResult = main.TRUE
168 if PULLCODE and gitPullResult == main.TRUE:
169 cleanInstallResult = main.ONOSbench.cleanInstall()
170 else:
171 main.log.warn( "Did not pull new code so skipping mvn " +
172 "clean install" )
173 utilities.assert_equals( expect=main.TRUE,
174 actual=cleanInstallResult,
175 onpass="MCI successful",
176 onfail="MCI failed" )
177 # GRAPHS
178 # NOTE: important params here:
179 # job = name of Jenkins job
180 # Plot Name = Plot-HA, only can be used if multiple plots
181 # index = The number of the graph under plot name
182 job = "HAclusterRestart"
183 plotName = "Plot-HA"
184 graphs = '<ac:structured-macro ac:name="html">\n'
185 graphs += '<ac:plain-text-body><![CDATA[\n'
186 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
187 '/plot/' + plotName + '/getPlot?index=0' +\
188 '&width=500&height=300"' +\
189 'noborder="0" width="500" height="300" scrolling="yes" ' +\
190 'seamless="seamless"></iframe>\n'
191 graphs += ']]></ac:plain-text-body>\n'
192 graphs += '</ac:structured-macro>\n'
193 main.log.wiki(graphs)
194
195 main.step( "Creating ONOS package" )
196 packageResult = main.ONOSbench.onosPackage()
197 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
198 onpass="ONOS package successful",
199 onfail="ONOS package failed" )
200
201 main.step( "Installing ONOS package" )
202 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700203 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700204 tmpResult = main.ONOSbench.onosInstall( options="-f",
205 node=node.ip_address )
206 onosInstallResult = onosInstallResult and tmpResult
207 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
208 onpass="ONOS install successful",
209 onfail="ONOS install failed" )
210
211 main.step( "Checking if ONOS is up yet" )
212 for i in range( 2 ):
213 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700214 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700215 started = main.ONOSbench.isup( node.ip_address )
216 if not started:
217 main.log.error( node.name + " didn't start!" )
218 main.ONOSbench.onosStop( node.ip_address )
219 main.ONOSbench.onosStart( node.ip_address )
220 onosIsupResult = onosIsupResult and started
221 if onosIsupResult == main.TRUE:
222 break
223 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
224 onpass="ONOS startup successful",
225 onfail="ONOS startup failed" )
226
227 main.log.step( "Starting ONOS CLI sessions" )
228 cliResults = main.TRUE
229 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700230 for i in range( main.numCtrls ):
231 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700232 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700233 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700234 threads.append( t )
235 t.start()
236
237 for t in threads:
238 t.join()
239 cliResults = cliResults and t.result
240 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
241 onpass="ONOS cli startup successful",
242 onfail="ONOS cli startup failed" )
243
244 if main.params[ 'tcpdump' ].lower() == "true":
245 main.step( "Start Packet Capture MN" )
246 main.Mininet2.startTcpdump(
247 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
248 + "-MN.pcap",
249 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
250 port=main.params[ 'MNtcpdump' ][ 'port' ] )
251
252 main.step( "App Ids check" )
253 appCheck = main.TRUE
254 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700255 for i in range( main.numCtrls ):
256 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700257 name="appToIDCheck-" + str( i ),
258 args=[] )
259 threads.append( t )
260 t.start()
261
262 for t in threads:
263 t.join()
264 appCheck = appCheck and t.result
265 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700266 main.log.warn( main.CLIs[0].apps() )
267 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700268 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
269 onpass="App Ids seem to be correct",
270 onfail="Something is wrong with app Ids" )
271
272 if cliResults == main.FALSE:
273 main.log.error( "Failed to start ONOS, stopping test" )
274 main.cleanup()
275 main.exit()
276
277 def CASE2( self, main ):
278 """
279 Assign devices to controllers
280 """
281 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700282 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700283 assert main, "main not defined"
284 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700285 assert main.CLIs, "main.CLIs not defined"
286 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700287 assert ONOS1Port, "ONOS1Port not defined"
288 assert ONOS2Port, "ONOS2Port not defined"
289 assert ONOS3Port, "ONOS3Port not defined"
290 assert ONOS4Port, "ONOS4Port not defined"
291 assert ONOS5Port, "ONOS5Port not defined"
292 assert ONOS6Port, "ONOS6Port not defined"
293 assert ONOS7Port, "ONOS7Port not defined"
294
295 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700296 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700297 "and check that an ONOS node becomes the " +\
298 "master of the device."
299 main.step( "Assign switches to controllers" )
300
301 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700302 for i in range( main.numCtrls ):
303 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700304 swList = []
305 for i in range( 1, 29 ):
306 swList.append( "s" + str( i ) )
307 main.Mininet1.assignSwController( sw=swList, ip=ipList )
308
309 mastershipCheck = main.TRUE
310 for i in range( 1, 29 ):
311 response = main.Mininet1.getSwController( "s" + str( i ) )
312 try:
313 main.log.info( str( response ) )
314 except Exception:
315 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700316 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700317 if re.search( "tcp:" + node.ip_address, response ):
318 mastershipCheck = mastershipCheck and main.TRUE
319 else:
320 main.log.error( "Error, node " + node.ip_address + " is " +
321 "not in the list of controllers s" +
322 str( i ) + " is connecting to." )
323 mastershipCheck = main.FALSE
324 utilities.assert_equals(
325 expect=main.TRUE,
326 actual=mastershipCheck,
327 onpass="Switch mastership assigned correctly",
328 onfail="Switches not assigned correctly to controllers" )
329
330 def CASE21( self, main ):
331 """
332 Assign mastership to controllers
333 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700334 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700335 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700336 assert main, "main not defined"
337 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700338 assert main.CLIs, "main.CLIs not defined"
339 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700340 assert ONOS1Port, "ONOS1Port not defined"
341 assert ONOS2Port, "ONOS2Port not defined"
342 assert ONOS3Port, "ONOS3Port not defined"
343 assert ONOS4Port, "ONOS4Port not defined"
344 assert ONOS5Port, "ONOS5Port not defined"
345 assert ONOS6Port, "ONOS6Port not defined"
346 assert ONOS7Port, "ONOS7Port not defined"
347
348 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700349 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700350 "device. Then manually assign" +\
351 " mastership to specific ONOS nodes using" +\
352 " 'device-role'"
353 main.step( "Assign mastership of switches to specific controllers" )
354 # Manually assign mastership to the controller we want
355 roleCall = main.TRUE
356
357 ipList = [ ]
358 deviceList = []
359 try:
360 # Assign mastership to specific controllers. This assignment was
361 # determined for a 7 node cluser, but will work with any sized
362 # cluster
363 for i in range( 1, 29 ): # switches 1 through 28
364 # set up correct variables:
365 if i == 1:
366 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700367 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
369 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700370 c = 1 % main.numCtrls
371 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
373 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700374 c = 1 % main.numCtrls
375 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700376 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
377 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700378 c = 3 % main.numCtrls
379 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700380 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
381 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700382 c = 2 % main.numCtrls
383 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700384 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
385 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700386 c = 2 % main.numCtrls
387 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700388 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
389 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700390 c = 5 % main.numCtrls
391 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700392 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
393 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700394 c = 4 % main.numCtrls
395 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700396 dpid = '3' + str( i ).zfill( 3 )
397 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
398 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700399 c = 6 % main.numCtrls
400 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700401 dpid = '6' + str( i ).zfill( 3 )
402 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
403 elif i == 28:
404 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700405 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700406 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
407 else:
408 main.log.error( "You didn't write an else statement for " +
409 "switch s" + str( i ) )
410 roleCall = main.FALSE
411 # Assign switch
412 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
413 # TODO: make this controller dynamic
414 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
415 ip )
416 ipList.append( ip )
417 deviceList.append( deviceId )
418 except ( AttributeError, AssertionError ):
419 main.log.exception( "Something is wrong with ONOS device view" )
420 main.log.info( main.ONOScli1.devices() )
421 utilities.assert_equals(
422 expect=main.TRUE,
423 actual=roleCall,
424 onpass="Re-assigned switch mastership to designated controller",
425 onfail="Something wrong with deviceRole calls" )
426
427 main.step( "Check mastership was correctly assigned" )
428 roleCheck = main.TRUE
429 # NOTE: This is due to the fact that device mastership change is not
430 # atomic and is actually a multi step process
431 time.sleep( 5 )
432 for i in range( len( ipList ) ):
433 ip = ipList[i]
434 deviceId = deviceList[i]
435 # Check assignment
436 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
437 if ip in master:
438 roleCheck = roleCheck and main.TRUE
439 else:
440 roleCheck = roleCheck and main.FALSE
441 main.log.error( "Error, controller " + ip + " is not" +
442 " master " + "of device " +
443 str( deviceId ) + ". Master is " +
444 repr( master ) + "." )
445 utilities.assert_equals(
446 expect=main.TRUE,
447 actual=roleCheck,
448 onpass="Switches were successfully reassigned to designated " +
449 "controller",
450 onfail="Switches were not successfully reassigned" )
451
452 def CASE3( self, main ):
453 """
454 Assign intents
455 """
456 import time
457 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700458 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 assert main, "main not defined"
460 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700461 assert main.CLIs, "main.CLIs not defined"
462 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 try:
464 labels
465 except NameError:
466 main.log.error( "labels not defined, setting to []" )
467 labels = []
468 try:
469 data
470 except NameError:
471 main.log.error( "data not defined, setting to []" )
472 data = []
473 # NOTE: we must reinstall intents until we have a persistant intent
474 # datastore!
475 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700476 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700477 "assign predetermined host-to-host intents." +\
478 " After installation, check that the intent" +\
479 " is distributed to all nodes and the state" +\
480 " is INSTALLED"
481
482 # install onos-app-fwd
483 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700484 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700485 utilities.assert_equals( expect=main.TRUE, actual=installResults,
486 onpass="Install fwd successful",
487 onfail="Install fwd failed" )
488
489 main.step( "Check app ids" )
490 appCheck = main.TRUE
491 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700492 for i in range( main.numCtrls ):
493 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700494 name="appToIDCheck-" + str( i ),
495 args=[] )
496 threads.append( t )
497 t.start()
498
499 for t in threads:
500 t.join()
501 appCheck = appCheck and t.result
502 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700503 main.log.warn( main.CLIs[0].apps() )
504 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700505 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
506 onpass="App Ids seem to be correct",
507 onfail="Something is wrong with app Ids" )
508
509 main.step( "Discovering Hosts( Via pingall for now )" )
510 # FIXME: Once we have a host discovery mechanism, use that instead
511 # REACTIVE FWD test
512 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700513 passMsg = "Reactive Pingall test passed"
514 time1 = time.time()
515 pingResult = main.Mininet1.pingall()
516 time2 = time.time()
517 if not pingResult:
518 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700519 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700520 passMsg += " on the second try"
521 utilities.assert_equals(
522 expect=main.TRUE,
523 actual=pingResult,
524 onpass= passMsg,
525 onfail="Reactive Pingall failed, " +
526 "one or more ping pairs failed" )
527 main.log.info( "Time for pingall: %2f seconds" %
528 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700529 # timeout for fwd flows
530 time.sleep( 11 )
531 # uninstall onos-app-fwd
532 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700533 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700534 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
535 onpass="Uninstall fwd successful",
536 onfail="Uninstall fwd failed" )
537
538 main.step( "Check app ids" )
539 threads = []
540 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700541 for i in range( main.numCtrls ):
542 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700543 name="appToIDCheck-" + str( i ),
544 args=[] )
545 threads.append( t )
546 t.start()
547
548 for t in threads:
549 t.join()
550 appCheck2 = appCheck2 and t.result
551 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700552 main.log.warn( main.CLIs[0].apps() )
553 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700554 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
555 onpass="App Ids seem to be correct",
556 onfail="Something is wrong with app Ids" )
557
558 main.step( "Add host intents via cli" )
559 intentIds = []
560 # TODO: move the host numbers to params
561 # Maybe look at all the paths we ping?
562 intentAddResult = True
563 hostResult = main.TRUE
564 for i in range( 8, 18 ):
565 main.log.info( "Adding host intent between h" + str( i ) +
566 " and h" + str( i + 10 ) )
567 host1 = "00:00:00:00:00:" + \
568 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
569 host2 = "00:00:00:00:00:" + \
570 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
571 # NOTE: getHost can return None
572 host1Dict = main.ONOScli1.getHost( host1 )
573 host2Dict = main.ONOScli1.getHost( host2 )
574 host1Id = None
575 host2Id = None
576 if host1Dict and host2Dict:
577 host1Id = host1Dict.get( 'id', None )
578 host2Id = host2Dict.get( 'id', None )
579 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700580 nodeNum = ( i % main.numCtrls )
581 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700582 if tmpId:
583 main.log.info( "Added intent with id: " + tmpId )
584 intentIds.append( tmpId )
585 else:
586 main.log.error( "addHostIntent returned: " +
587 repr( tmpId ) )
588 else:
589 main.log.error( "Error, getHost() failed for h" + str( i ) +
590 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700591 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700592 main.log.warn( "Hosts output: " )
593 try:
594 main.log.warn( json.dumps( json.loads( hosts ),
595 sort_keys=True,
596 indent=4,
597 separators=( ',', ': ' ) ) )
598 except ( ValueError, TypeError ):
599 main.log.warn( repr( hosts ) )
600 hostResult = main.FALSE
601 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
602 onpass="Found a host id for each host",
603 onfail="Error looking up host ids" )
604
605 intentStart = time.time()
606 onosIds = main.ONOScli1.getAllIntentsId()
607 main.log.info( "Submitted intents: " + str( intentIds ) )
608 main.log.info( "Intents in ONOS: " + str( onosIds ) )
609 for intent in intentIds:
610 if intent in onosIds:
611 pass # intent submitted is in onos
612 else:
613 intentAddResult = False
614 if intentAddResult:
615 intentStop = time.time()
616 else:
617 intentStop = None
618 # Print the intent states
619 intents = main.ONOScli1.intents()
620 intentStates = []
621 installedCheck = True
622 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
623 count = 0
624 try:
625 for intent in json.loads( intents ):
626 state = intent.get( 'state', None )
627 if "INSTALLED" not in state:
628 installedCheck = False
629 intentId = intent.get( 'id', None )
630 intentStates.append( ( intentId, state ) )
631 except ( ValueError, TypeError ):
632 main.log.exception( "Error parsing intents" )
633 # add submitted intents not in the store
634 tmplist = [ i for i, s in intentStates ]
635 missingIntents = False
636 for i in intentIds:
637 if i not in tmplist:
638 intentStates.append( ( i, " - " ) )
639 missingIntents = True
640 intentStates.sort()
641 for i, s in intentStates:
642 count += 1
643 main.log.info( "%-6s%-15s%-15s" %
644 ( str( count ), str( i ), str( s ) ) )
645 leaders = main.ONOScli1.leaders()
646 try:
647 missing = False
648 if leaders:
649 parsedLeaders = json.loads( leaders )
650 main.log.warn( json.dumps( parsedLeaders,
651 sort_keys=True,
652 indent=4,
653 separators=( ',', ': ' ) ) )
654 # check for all intent partitions
655 topics = []
656 for i in range( 14 ):
657 topics.append( "intent-partition-" + str( i ) )
658 main.log.debug( topics )
659 ONOStopics = [ j['topic'] for j in parsedLeaders ]
660 for topic in topics:
661 if topic not in ONOStopics:
662 main.log.error( "Error: " + topic +
663 " not in leaders" )
664 missing = True
665 else:
666 main.log.error( "leaders() returned None" )
667 except ( ValueError, TypeError ):
668 main.log.exception( "Error parsing leaders" )
669 main.log.error( repr( leaders ) )
670 # Check all nodes
671 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700672 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700673 response = node.leaders( jsonFormat=False)
674 main.log.warn( str( node.name ) + " leaders output: \n" +
675 str( response ) )
676
677 partitions = main.ONOScli1.partitions()
678 try:
679 if partitions :
680 parsedPartitions = json.loads( partitions )
681 main.log.warn( json.dumps( parsedPartitions,
682 sort_keys=True,
683 indent=4,
684 separators=( ',', ': ' ) ) )
685 # TODO check for a leader in all paritions
686 # TODO check for consistency among nodes
687 else:
688 main.log.error( "partitions() returned None" )
689 except ( ValueError, TypeError ):
690 main.log.exception( "Error parsing partitions" )
691 main.log.error( repr( partitions ) )
692 pendingMap = main.ONOScli1.pendingMap()
693 try:
694 if pendingMap :
695 parsedPending = json.loads( pendingMap )
696 main.log.warn( json.dumps( parsedPending,
697 sort_keys=True,
698 indent=4,
699 separators=( ',', ': ' ) ) )
700 # TODO check something here?
701 else:
702 main.log.error( "pendingMap() returned None" )
703 except ( ValueError, TypeError ):
704 main.log.exception( "Error parsing pending map" )
705 main.log.error( repr( pendingMap ) )
706
707 intentAddResult = bool( intentAddResult and not missingIntents and
708 installedCheck )
709 if not intentAddResult:
710 main.log.error( "Error in pushing host intents to ONOS" )
711
712 main.step( "Intent Anti-Entropy dispersion" )
713 for i in range(100):
714 correct = True
715 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700716 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700717 onosIds = []
718 ids = cli.getAllIntentsId()
719 onosIds.append( ids )
720 main.log.debug( "Intents in " + cli.name + ": " +
721 str( sorted( onosIds ) ) )
722 if sorted( ids ) != sorted( intentIds ):
723 main.log.warn( "Set of intent IDs doesn't match" )
724 correct = False
725 break
726 else:
727 intents = json.loads( cli.intents() )
728 for intent in intents:
729 if intent[ 'state' ] != "INSTALLED":
730 main.log.warn( "Intent " + intent[ 'id' ] +
731 " is " + intent[ 'state' ] )
732 correct = False
733 break
734 if correct:
735 break
736 else:
737 time.sleep(1)
738 if not intentStop:
739 intentStop = time.time()
740 global gossipTime
741 gossipTime = intentStop - intentStart
742 main.log.info( "It took about " + str( gossipTime ) +
743 " seconds for all intents to appear in each node" )
744 append = False
745 title = "Gossip Intents"
746 count = 1
747 while append is False:
748 curTitle = title + str( count )
749 if curTitle not in labels:
750 labels.append( curTitle )
751 data.append( str( gossipTime ) )
752 append = True
753 else:
754 count += 1
755 # FIXME: make this time configurable/calculate based off of number of
756 # nodes and gossip rounds
757 utilities.assert_greater_equals(
758 expect=40, actual=gossipTime,
759 onpass="ECM anti-entropy for intents worked within " +
760 "expected time",
761 onfail="Intent ECM anti-entropy took too long" )
762 if gossipTime <= 40:
763 intentAddResult = True
764
765 if not intentAddResult or "key" in pendingMap:
766 import time
767 installedCheck = True
768 main.log.info( "Sleeping 60 seconds to see if intents are found" )
769 time.sleep( 60 )
770 onosIds = main.ONOScli1.getAllIntentsId()
771 main.log.info( "Submitted intents: " + str( intentIds ) )
772 main.log.info( "Intents in ONOS: " + str( onosIds ) )
773 # Print the intent states
774 intents = main.ONOScli1.intents()
775 intentStates = []
776 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
777 count = 0
778 try:
779 for intent in json.loads( intents ):
780 # Iter through intents of a node
781 state = intent.get( 'state', None )
782 if "INSTALLED" not in state:
783 installedCheck = False
784 intentId = intent.get( 'id', None )
785 intentStates.append( ( intentId, state ) )
786 except ( ValueError, TypeError ):
787 main.log.exception( "Error parsing intents" )
788 # add submitted intents not in the store
789 tmplist = [ i for i, s in intentStates ]
790 for i in intentIds:
791 if i not in tmplist:
792 intentStates.append( ( i, " - " ) )
793 intentStates.sort()
794 for i, s in intentStates:
795 count += 1
796 main.log.info( "%-6s%-15s%-15s" %
797 ( str( count ), str( i ), str( s ) ) )
798 leaders = main.ONOScli1.leaders()
799 try:
800 missing = False
801 if leaders:
802 parsedLeaders = json.loads( leaders )
803 main.log.warn( json.dumps( parsedLeaders,
804 sort_keys=True,
805 indent=4,
806 separators=( ',', ': ' ) ) )
807 # check for all intent partitions
808 # check for election
809 topics = []
810 for i in range( 14 ):
811 topics.append( "intent-partition-" + str( i ) )
812 # FIXME: this should only be after we start the app
813 topics.append( "org.onosproject.election" )
814 main.log.debug( topics )
815 ONOStopics = [ j['topic'] for j in parsedLeaders ]
816 for topic in topics:
817 if topic not in ONOStopics:
818 main.log.error( "Error: " + topic +
819 " not in leaders" )
820 missing = True
821 else:
822 main.log.error( "leaders() returned None" )
823 except ( ValueError, TypeError ):
824 main.log.exception( "Error parsing leaders" )
825 main.log.error( repr( leaders ) )
826 # Check all nodes
827 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700828 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700829 response = node.leaders( jsonFormat=False)
830 main.log.warn( str( node.name ) + " leaders output: \n" +
831 str( response ) )
832
833 partitions = main.ONOScli1.partitions()
834 try:
835 if partitions :
836 parsedPartitions = json.loads( partitions )
837 main.log.warn( json.dumps( parsedPartitions,
838 sort_keys=True,
839 indent=4,
840 separators=( ',', ': ' ) ) )
841 # TODO check for a leader in all paritions
842 # TODO check for consistency among nodes
843 else:
844 main.log.error( "partitions() returned None" )
845 except ( ValueError, TypeError ):
846 main.log.exception( "Error parsing partitions" )
847 main.log.error( repr( partitions ) )
848 pendingMap = main.ONOScli1.pendingMap()
849 try:
850 if pendingMap :
851 parsedPending = json.loads( pendingMap )
852 main.log.warn( json.dumps( parsedPending,
853 sort_keys=True,
854 indent=4,
855 separators=( ',', ': ' ) ) )
856 # TODO check something here?
857 else:
858 main.log.error( "pendingMap() returned None" )
859 except ( ValueError, TypeError ):
860 main.log.exception( "Error parsing pending map" )
861 main.log.error( repr( pendingMap ) )
862
863 def CASE4( self, main ):
864 """
865 Ping across added host intents
866 """
867 import json
868 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700869 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700870 assert main, "main not defined"
871 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700872 assert main.CLIs, "main.CLIs not defined"
873 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700874 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700875 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700876 "functionality and check the state of " +\
877 "the intent"
878 main.step( "Ping across added host intents" )
879 PingResult = main.TRUE
880 for i in range( 8, 18 ):
881 ping = main.Mininet1.pingHost( src="h" + str( i ),
882 target="h" + str( i + 10 ) )
883 PingResult = PingResult and ping
884 if ping == main.FALSE:
885 main.log.warn( "Ping failed between h" + str( i ) +
886 " and h" + str( i + 10 ) )
887 elif ping == main.TRUE:
888 main.log.info( "Ping test passed!" )
889 # Don't set PingResult or you'd override failures
890 if PingResult == main.FALSE:
891 main.log.error(
892 "Intents have not been installed correctly, pings failed." )
893 # TODO: pretty print
894 main.log.warn( "ONOS1 intents: " )
895 try:
896 tmpIntents = main.ONOScli1.intents()
897 main.log.warn( json.dumps( json.loads( tmpIntents ),
898 sort_keys=True,
899 indent=4,
900 separators=( ',', ': ' ) ) )
901 except ( ValueError, TypeError ):
902 main.log.warn( repr( tmpIntents ) )
903 utilities.assert_equals(
904 expect=main.TRUE,
905 actual=PingResult,
906 onpass="Intents have been installed correctly and pings work",
907 onfail="Intents have not been installed correctly, pings failed." )
908
909 main.step( "Check Intent state" )
910 installedCheck = False
911 loopCount = 0
912 while not installedCheck and loopCount < 40:
913 installedCheck = True
914 # Print the intent states
915 intents = main.ONOScli1.intents()
916 intentStates = []
917 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700918 count = 0
Jon Hall5cf14d52015-07-16 12:15:19 -0700919 # Iter through intents of a node
920 try:
921 for intent in json.loads( intents ):
922 state = intent.get( 'state', None )
923 if "INSTALLED" not in state:
924 installedCheck = False
925 intentId = intent.get( 'id', None )
926 intentStates.append( ( intentId, state ) )
927 except ( ValueError, TypeError ):
928 main.log.exception( "Error parsing intents." )
929 # Print states
930 intentStates.sort()
931 for i, s in intentStates:
932 count += 1
933 main.log.info( "%-6s%-15s%-15s" %
934 ( str( count ), str( i ), str( s ) ) )
935 if not installedCheck:
936 time.sleep( 1 )
937 loopCount += 1
938 utilities.assert_equals( expect=True, actual=installedCheck,
939 onpass="Intents are all INSTALLED",
940 onfail="Intents are not all in " +
941 "INSTALLED state" )
942
943 main.step( "Check leadership of topics" )
944 leaders = main.ONOScli1.leaders()
945 topicCheck = main.TRUE
946 try:
947 if leaders:
948 parsedLeaders = json.loads( leaders )
949 main.log.warn( json.dumps( parsedLeaders,
950 sort_keys=True,
951 indent=4,
952 separators=( ',', ': ' ) ) )
953 # check for all intent partitions
954 # check for election
955 # TODO: Look at Devices as topics now that it uses this system
956 topics = []
957 for i in range( 14 ):
958 topics.append( "intent-partition-" + str( i ) )
959 # FIXME: this should only be after we start the app
960 # FIXME: topics.append( "org.onosproject.election" )
961 # Print leaders output
962 main.log.debug( topics )
963 ONOStopics = [ j['topic'] for j in parsedLeaders ]
964 for topic in topics:
965 if topic not in ONOStopics:
966 main.log.error( "Error: " + topic +
967 " not in leaders" )
968 topicCheck = main.FALSE
969 else:
970 main.log.error( "leaders() returned None" )
971 topicCheck = main.FALSE
972 except ( ValueError, TypeError ):
973 topicCheck = main.FALSE
974 main.log.exception( "Error parsing leaders" )
975 main.log.error( repr( leaders ) )
976 # TODO: Check for a leader of these topics
977 # Check all nodes
978 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700979 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700980 response = node.leaders( jsonFormat=False)
981 main.log.warn( str( node.name ) + " leaders output: \n" +
982 str( response ) )
983
984 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
985 onpass="intent Partitions is in leaders",
986 onfail="Some topics were lost " )
987 # Print partitions
988 partitions = main.ONOScli1.partitions()
989 try:
990 if partitions :
991 parsedPartitions = json.loads( partitions )
992 main.log.warn( json.dumps( parsedPartitions,
993 sort_keys=True,
994 indent=4,
995 separators=( ',', ': ' ) ) )
996 # TODO check for a leader in all paritions
997 # TODO check for consistency among nodes
998 else:
999 main.log.error( "partitions() returned None" )
1000 except ( ValueError, TypeError ):
1001 main.log.exception( "Error parsing partitions" )
1002 main.log.error( repr( partitions ) )
1003 # Print Pending Map
1004 pendingMap = main.ONOScli1.pendingMap()
1005 try:
1006 if pendingMap :
1007 parsedPending = json.loads( pendingMap )
1008 main.log.warn( json.dumps( parsedPending,
1009 sort_keys=True,
1010 indent=4,
1011 separators=( ',', ': ' ) ) )
1012 # TODO check something here?
1013 else:
1014 main.log.error( "pendingMap() returned None" )
1015 except ( ValueError, TypeError ):
1016 main.log.exception( "Error parsing pending map" )
1017 main.log.error( repr( pendingMap ) )
1018
1019 if not installedCheck:
1020 main.log.info( "Waiting 60 seconds to see if the state of " +
1021 "intents change" )
1022 time.sleep( 60 )
1023 # Print the intent states
1024 intents = main.ONOScli1.intents()
1025 intentStates = []
1026 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1027 count = 0
1028 # Iter through intents of a node
1029 try:
1030 for intent in json.loads( intents ):
1031 state = intent.get( 'state', None )
1032 if "INSTALLED" not in state:
1033 installedCheck = False
1034 intentId = intent.get( 'id', None )
1035 intentStates.append( ( intentId, state ) )
1036 except ( ValueError, TypeError ):
1037 main.log.exception( "Error parsing intents." )
1038 intentStates.sort()
1039 for i, s in intentStates:
1040 count += 1
1041 main.log.info( "%-6s%-15s%-15s" %
1042 ( str( count ), str( i ), str( s ) ) )
1043 leaders = main.ONOScli1.leaders()
1044 try:
1045 missing = False
1046 if leaders:
1047 parsedLeaders = json.loads( leaders )
1048 main.log.warn( json.dumps( parsedLeaders,
1049 sort_keys=True,
1050 indent=4,
1051 separators=( ',', ': ' ) ) )
1052 # check for all intent partitions
1053 # check for election
1054 topics = []
1055 for i in range( 14 ):
1056 topics.append( "intent-partition-" + str( i ) )
1057 # FIXME: this should only be after we start the app
1058 topics.append( "org.onosproject.election" )
1059 main.log.debug( topics )
1060 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1061 for topic in topics:
1062 if topic not in ONOStopics:
1063 main.log.error( "Error: " + topic +
1064 " not in leaders" )
1065 missing = True
1066 else:
1067 main.log.error( "leaders() returned None" )
1068 except ( ValueError, TypeError ):
1069 main.log.exception( "Error parsing leaders" )
1070 main.log.error( repr( leaders ) )
1071 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001072 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001073 response = node.leaders( jsonFormat=False)
1074 main.log.warn( str( node.name ) + " leaders output: \n" +
1075 str( response ) )
1076
1077 partitions = main.ONOScli1.partitions()
1078 try:
1079 if partitions :
1080 parsedPartitions = json.loads( partitions )
1081 main.log.warn( json.dumps( parsedPartitions,
1082 sort_keys=True,
1083 indent=4,
1084 separators=( ',', ': ' ) ) )
1085 # TODO check for a leader in all paritions
1086 # TODO check for consistency among nodes
1087 else:
1088 main.log.error( "partitions() returned None" )
1089 except ( ValueError, TypeError ):
1090 main.log.exception( "Error parsing partitions" )
1091 main.log.error( repr( partitions ) )
1092 pendingMap = main.ONOScli1.pendingMap()
1093 try:
1094 if pendingMap :
1095 parsedPending = json.loads( pendingMap )
1096 main.log.warn( json.dumps( parsedPending,
1097 sort_keys=True,
1098 indent=4,
1099 separators=( ',', ': ' ) ) )
1100 # TODO check something here?
1101 else:
1102 main.log.error( "pendingMap() returned None" )
1103 except ( ValueError, TypeError ):
1104 main.log.exception( "Error parsing pending map" )
1105 main.log.error( repr( pendingMap ) )
1106 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001107 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001108 main.step( "Wait a minute then ping again" )
1109 # the wait is above
1110 PingResult = main.TRUE
1111 for i in range( 8, 18 ):
1112 ping = main.Mininet1.pingHost( src="h" + str( i ),
1113 target="h" + str( i + 10 ) )
1114 PingResult = PingResult and ping
1115 if ping == main.FALSE:
1116 main.log.warn( "Ping failed between h" + str( i ) +
1117 " and h" + str( i + 10 ) )
1118 elif ping == main.TRUE:
1119 main.log.info( "Ping test passed!" )
1120 # Don't set PingResult or you'd override failures
1121 if PingResult == main.FALSE:
1122 main.log.error(
1123 "Intents have not been installed correctly, pings failed." )
1124 # TODO: pretty print
1125 main.log.warn( "ONOS1 intents: " )
1126 try:
1127 tmpIntents = main.ONOScli1.intents()
1128 main.log.warn( json.dumps( json.loads( tmpIntents ),
1129 sort_keys=True,
1130 indent=4,
1131 separators=( ',', ': ' ) ) )
1132 except ( ValueError, TypeError ):
1133 main.log.warn( repr( tmpIntents ) )
1134 utilities.assert_equals(
1135 expect=main.TRUE,
1136 actual=PingResult,
1137 onpass="Intents have been installed correctly and pings work",
1138 onfail="Intents have not been installed correctly, pings failed." )
1139
1140 def CASE5( self, main ):
1141 """
1142 Reading state of ONOS
1143 """
1144 import json
1145 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001146 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001147 assert main, "main not defined"
1148 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001149 assert main.CLIs, "main.CLIs not defined"
1150 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001151
1152 main.case( "Setting up and gathering data for current state" )
1153 # The general idea for this test case is to pull the state of
1154 # ( intents,flows, topology,... ) from each ONOS node
1155 # We can then compare them with each other and also with past states
1156
1157 main.step( "Check that each switch has a master" )
1158 global mastershipState
1159 mastershipState = '[]'
1160
1161 # Assert that each device has a master
1162 rolesNotNull = main.TRUE
1163 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001164 for i in range( main.numCtrls ):
1165 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001166 name="rolesNotNull-" + str( i ),
1167 args=[] )
1168 threads.append( t )
1169 t.start()
1170
1171 for t in threads:
1172 t.join()
1173 rolesNotNull = rolesNotNull and t.result
1174 utilities.assert_equals(
1175 expect=main.TRUE,
1176 actual=rolesNotNull,
1177 onpass="Each device has a master",
1178 onfail="Some devices don't have a master assigned" )
1179
1180 main.step( "Get the Mastership of each switch from each controller" )
1181 ONOSMastership = []
1182 mastershipCheck = main.FALSE
1183 consistentMastership = True
1184 rolesResults = True
1185 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001186 for i in range( main.numCtrls ):
1187 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001188 name="roles-" + str( i ),
1189 args=[] )
1190 threads.append( t )
1191 t.start()
1192
1193 for t in threads:
1194 t.join()
1195 ONOSMastership.append( t.result )
1196
Jon Halle1a3b752015-07-22 13:02:46 -07001197 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001198 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1199 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1200 " roles" )
1201 main.log.warn(
1202 "ONOS" + str( i + 1 ) + " mastership response: " +
1203 repr( ONOSMastership[i] ) )
1204 rolesResults = False
1205 utilities.assert_equals(
1206 expect=True,
1207 actual=rolesResults,
1208 onpass="No error in reading roles output",
1209 onfail="Error in reading roles from ONOS" )
1210
1211 main.step( "Check for consistency in roles from each controller" )
1212 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1213 main.log.info(
1214 "Switch roles are consistent across all ONOS nodes" )
1215 else:
1216 consistentMastership = False
1217 utilities.assert_equals(
1218 expect=True,
1219 actual=consistentMastership,
1220 onpass="Switch roles are consistent across all ONOS nodes",
1221 onfail="ONOS nodes have different views of switch roles" )
1222
1223 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001224 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001225 try:
1226 main.log.warn(
1227 "ONOS" + str( i + 1 ) + " roles: ",
1228 json.dumps(
1229 json.loads( ONOSMastership[ i ] ),
1230 sort_keys=True,
1231 indent=4,
1232 separators=( ',', ': ' ) ) )
1233 except ( ValueError, TypeError ):
1234 main.log.warn( repr( ONOSMastership[ i ] ) )
1235 elif rolesResults and consistentMastership:
1236 mastershipCheck = main.TRUE
1237 mastershipState = ONOSMastership[ 0 ]
1238
1239 main.step( "Get the intents from each controller" )
1240 global intentState
1241 intentState = []
1242 ONOSIntents = []
1243 intentCheck = main.FALSE
1244 consistentIntents = True
1245 intentsResults = True
1246 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001247 for i in range( main.numCtrls ):
1248 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001249 name="intents-" + str( i ),
1250 args=[],
1251 kwargs={ 'jsonFormat': True } )
1252 threads.append( t )
1253 t.start()
1254
1255 for t in threads:
1256 t.join()
1257 ONOSIntents.append( t.result )
1258
Jon Halle1a3b752015-07-22 13:02:46 -07001259 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001260 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1261 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1262 " intents" )
1263 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1264 repr( ONOSIntents[ i ] ) )
1265 intentsResults = False
1266 utilities.assert_equals(
1267 expect=True,
1268 actual=intentsResults,
1269 onpass="No error in reading intents output",
1270 onfail="Error in reading intents from ONOS" )
1271
1272 main.step( "Check for consistency in Intents from each controller" )
1273 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1274 main.log.info( "Intents are consistent across all ONOS " +
1275 "nodes" )
1276 else:
1277 consistentIntents = False
1278 main.log.error( "Intents not consistent" )
1279 utilities.assert_equals(
1280 expect=True,
1281 actual=consistentIntents,
1282 onpass="Intents are consistent across all ONOS nodes",
1283 onfail="ONOS nodes have different views of intents" )
1284
1285 if intentsResults:
1286 # Try to make it easy to figure out what is happening
1287 #
1288 # Intent ONOS1 ONOS2 ...
1289 # 0x01 INSTALLED INSTALLING
1290 # ... ... ...
1291 # ... ... ...
1292 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001293 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001294 title += " " * 10 + "ONOS" + str( n + 1 )
1295 main.log.warn( title )
1296 # get all intent keys in the cluster
1297 keys = []
1298 for nodeStr in ONOSIntents:
1299 node = json.loads( nodeStr )
1300 for intent in node:
1301 keys.append( intent.get( 'id' ) )
1302 keys = set( keys )
1303 for key in keys:
1304 row = "%-13s" % key
1305 for nodeStr in ONOSIntents:
1306 node = json.loads( nodeStr )
1307 for intent in node:
1308 if intent.get( 'id', "Error" ) == key:
1309 row += "%-15s" % intent.get( 'state' )
1310 main.log.warn( row )
1311 # End table view
1312
1313 if intentsResults and not consistentIntents:
1314 # print the json objects
1315 n = len(ONOSIntents)
1316 main.log.debug( "ONOS" + str( n ) + " intents: " )
1317 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1318 sort_keys=True,
1319 indent=4,
1320 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001321 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001322 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1323 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1324 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1325 sort_keys=True,
1326 indent=4,
1327 separators=( ',', ': ' ) ) )
1328 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001329 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001330 str( n ) + " intents" )
1331 elif intentsResults and consistentIntents:
1332 intentCheck = main.TRUE
1333 intentState = ONOSIntents[ 0 ]
1334
1335 main.step( "Get the flows from each controller" )
1336 global flowState
1337 flowState = []
1338 ONOSFlows = []
1339 ONOSFlowsJson = []
1340 flowCheck = main.FALSE
1341 consistentFlows = True
1342 flowsResults = True
1343 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001344 for i in range( main.numCtrls ):
1345 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001346 name="flows-" + str( i ),
1347 args=[],
1348 kwargs={ 'jsonFormat': True } )
1349 threads.append( t )
1350 t.start()
1351
1352 # NOTE: Flows command can take some time to run
1353 time.sleep(30)
1354 for t in threads:
1355 t.join()
1356 result = t.result
1357 ONOSFlows.append( result )
1358
Jon Halle1a3b752015-07-22 13:02:46 -07001359 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001360 num = str( i + 1 )
1361 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1362 main.log.error( "Error in getting ONOS" + num + " flows" )
1363 main.log.warn( "ONOS" + num + " flows response: " +
1364 repr( ONOSFlows[ i ] ) )
1365 flowsResults = False
1366 ONOSFlowsJson.append( None )
1367 else:
1368 try:
1369 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1370 except ( ValueError, TypeError ):
1371 # FIXME: change this to log.error?
1372 main.log.exception( "Error in parsing ONOS" + num +
1373 " response as json." )
1374 main.log.error( repr( ONOSFlows[ i ] ) )
1375 ONOSFlowsJson.append( None )
1376 flowsResults = False
1377 utilities.assert_equals(
1378 expect=True,
1379 actual=flowsResults,
1380 onpass="No error in reading flows output",
1381 onfail="Error in reading flows from ONOS" )
1382
1383 main.step( "Check for consistency in Flows from each controller" )
1384 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1385 if all( tmp ):
1386 main.log.info( "Flow count is consistent across all ONOS nodes" )
1387 else:
1388 consistentFlows = False
1389 utilities.assert_equals(
1390 expect=True,
1391 actual=consistentFlows,
1392 onpass="The flow count is consistent across all ONOS nodes",
1393 onfail="ONOS nodes have different flow counts" )
1394
1395 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001396 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001397 try:
1398 main.log.warn(
1399 "ONOS" + str( i + 1 ) + " flows: " +
1400 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1401 indent=4, separators=( ',', ': ' ) ) )
1402 except ( ValueError, TypeError ):
1403 main.log.warn(
1404 "ONOS" + str( i + 1 ) + " flows: " +
1405 repr( ONOSFlows[ i ] ) )
1406 elif flowsResults and consistentFlows:
1407 flowCheck = main.TRUE
1408 flowState = ONOSFlows[ 0 ]
1409
1410 main.step( "Get the OF Table entries" )
1411 global flows
1412 flows = []
1413 for i in range( 1, 29 ):
Jon Hall9043c902015-07-30 14:23:44 -07001414 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001415 if flowCheck == main.FALSE:
1416 for table in flows:
1417 main.log.warn( table )
1418 # TODO: Compare switch flow tables with ONOS flow tables
1419
1420 main.step( "Start continuous pings" )
1421 main.Mininet2.pingLong(
1422 src=main.params[ 'PING' ][ 'source1' ],
1423 target=main.params[ 'PING' ][ 'target1' ],
1424 pingTime=500 )
1425 main.Mininet2.pingLong(
1426 src=main.params[ 'PING' ][ 'source2' ],
1427 target=main.params[ 'PING' ][ 'target2' ],
1428 pingTime=500 )
1429 main.Mininet2.pingLong(
1430 src=main.params[ 'PING' ][ 'source3' ],
1431 target=main.params[ 'PING' ][ 'target3' ],
1432 pingTime=500 )
1433 main.Mininet2.pingLong(
1434 src=main.params[ 'PING' ][ 'source4' ],
1435 target=main.params[ 'PING' ][ 'target4' ],
1436 pingTime=500 )
1437 main.Mininet2.pingLong(
1438 src=main.params[ 'PING' ][ 'source5' ],
1439 target=main.params[ 'PING' ][ 'target5' ],
1440 pingTime=500 )
1441 main.Mininet2.pingLong(
1442 src=main.params[ 'PING' ][ 'source6' ],
1443 target=main.params[ 'PING' ][ 'target6' ],
1444 pingTime=500 )
1445 main.Mininet2.pingLong(
1446 src=main.params[ 'PING' ][ 'source7' ],
1447 target=main.params[ 'PING' ][ 'target7' ],
1448 pingTime=500 )
1449 main.Mininet2.pingLong(
1450 src=main.params[ 'PING' ][ 'source8' ],
1451 target=main.params[ 'PING' ][ 'target8' ],
1452 pingTime=500 )
1453 main.Mininet2.pingLong(
1454 src=main.params[ 'PING' ][ 'source9' ],
1455 target=main.params[ 'PING' ][ 'target9' ],
1456 pingTime=500 )
1457 main.Mininet2.pingLong(
1458 src=main.params[ 'PING' ][ 'source10' ],
1459 target=main.params[ 'PING' ][ 'target10' ],
1460 pingTime=500 )
1461
1462 main.step( "Collecting topology information from ONOS" )
1463 devices = []
1464 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001465 for i in range( main.numCtrls ):
1466 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001467 name="devices-" + str( i ),
1468 args=[ ] )
1469 threads.append( t )
1470 t.start()
1471
1472 for t in threads:
1473 t.join()
1474 devices.append( t.result )
1475 hosts = []
1476 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001477 for i in range( main.numCtrls ):
1478 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001479 name="hosts-" + str( i ),
1480 args=[ ] )
1481 threads.append( t )
1482 t.start()
1483
1484 for t in threads:
1485 t.join()
1486 try:
1487 hosts.append( json.loads( t.result ) )
1488 except ( ValueError, TypeError ):
1489 # FIXME: better handling of this, print which node
1490 # Maybe use thread name?
1491 main.log.exception( "Error parsing json output of hosts" )
1492 # FIXME: should this be an empty json object instead?
1493 hosts.append( None )
1494
1495 ports = []
1496 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001497 for i in range( main.numCtrls ):
1498 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001499 name="ports-" + str( i ),
1500 args=[ ] )
1501 threads.append( t )
1502 t.start()
1503
1504 for t in threads:
1505 t.join()
1506 ports.append( t.result )
1507 links = []
1508 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001509 for i in range( main.numCtrls ):
1510 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001511 name="links-" + str( i ),
1512 args=[ ] )
1513 threads.append( t )
1514 t.start()
1515
1516 for t in threads:
1517 t.join()
1518 links.append( t.result )
1519 clusters = []
1520 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001521 for i in range( main.numCtrls ):
1522 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001523 name="clusters-" + str( i ),
1524 args=[ ] )
1525 threads.append( t )
1526 t.start()
1527
1528 for t in threads:
1529 t.join()
1530 clusters.append( t.result )
1531 # Compare json objects for hosts and dataplane clusters
1532
1533 # hosts
1534 main.step( "Host view is consistent across ONOS nodes" )
1535 consistentHostsResult = main.TRUE
1536 for controller in range( len( hosts ) ):
1537 controllerStr = str( controller + 1 )
1538 if "Error" not in hosts[ controller ]:
1539 if hosts[ controller ] == hosts[ 0 ]:
1540 continue
1541 else: # hosts not consistent
1542 main.log.error( "hosts from ONOS" +
1543 controllerStr +
1544 " is inconsistent with ONOS1" )
1545 main.log.warn( repr( hosts[ controller ] ) )
1546 consistentHostsResult = main.FALSE
1547
1548 else:
1549 main.log.error( "Error in getting ONOS hosts from ONOS" +
1550 controllerStr )
1551 consistentHostsResult = main.FALSE
1552 main.log.warn( "ONOS" + controllerStr +
1553 " hosts response: " +
1554 repr( hosts[ controller ] ) )
1555 utilities.assert_equals(
1556 expect=main.TRUE,
1557 actual=consistentHostsResult,
1558 onpass="Hosts view is consistent across all ONOS nodes",
1559 onfail="ONOS nodes have different views of hosts" )
1560
1561 main.step( "Each host has an IP address" )
1562 ipResult = main.TRUE
1563 for controller in range( 0, len( hosts ) ):
1564 controllerStr = str( controller + 1 )
1565 for host in hosts[ controller ]:
1566 if not host.get( 'ipAddresses', [ ] ):
1567 main.log.error( "DEBUG:Error with host ips on controller" +
1568 controllerStr + ": " + str( host ) )
1569 ipResult = main.FALSE
1570 utilities.assert_equals(
1571 expect=main.TRUE,
1572 actual=ipResult,
1573 onpass="The ips of the hosts aren't empty",
1574 onfail="The ip of at least one host is missing" )
1575
1576 # Strongly connected clusters of devices
1577 main.step( "Cluster view is consistent across ONOS nodes" )
1578 consistentClustersResult = main.TRUE
1579 for controller in range( len( clusters ) ):
1580 controllerStr = str( controller + 1 )
1581 if "Error" not in clusters[ controller ]:
1582 if clusters[ controller ] == clusters[ 0 ]:
1583 continue
1584 else: # clusters not consistent
1585 main.log.error( "clusters from ONOS" + controllerStr +
1586 " is inconsistent with ONOS1" )
1587 consistentClustersResult = main.FALSE
1588
1589 else:
1590 main.log.error( "Error in getting dataplane clusters " +
1591 "from ONOS" + controllerStr )
1592 consistentClustersResult = main.FALSE
1593 main.log.warn( "ONOS" + controllerStr +
1594 " clusters response: " +
1595 repr( clusters[ controller ] ) )
1596 utilities.assert_equals(
1597 expect=main.TRUE,
1598 actual=consistentClustersResult,
1599 onpass="Clusters view is consistent across all ONOS nodes",
1600 onfail="ONOS nodes have different views of clusters" )
1601 # there should always only be one cluster
1602 main.step( "Cluster view correct across ONOS nodes" )
1603 try:
1604 numClusters = len( json.loads( clusters[ 0 ] ) )
1605 except ( ValueError, TypeError ):
1606 main.log.exception( "Error parsing clusters[0]: " +
1607 repr( clusters[ 0 ] ) )
1608 clusterResults = main.FALSE
1609 if numClusters == 1:
1610 clusterResults = main.TRUE
1611 utilities.assert_equals(
1612 expect=1,
1613 actual=numClusters,
1614 onpass="ONOS shows 1 SCC",
1615 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1616
1617 main.step( "Comparing ONOS topology to MN" )
1618 devicesResults = main.TRUE
1619 linksResults = main.TRUE
1620 hostsResults = main.TRUE
1621 mnSwitches = main.Mininet1.getSwitches()
1622 mnLinks = main.Mininet1.getLinks()
1623 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001624 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001625 controllerStr = str( controller + 1 )
1626 if devices[ controller ] and ports[ controller ] and\
1627 "Error" not in devices[ controller ] and\
1628 "Error" not in ports[ controller ]:
Jon Halle1a3b752015-07-22 13:02:46 -07001629 currentDevicesResult = main.Mininet1.compareSwitches(
1630 mnSwitches,
1631 json.loads( devices[ controller ] ),
1632 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001633 else:
1634 currentDevicesResult = main.FALSE
1635 utilities.assert_equals( expect=main.TRUE,
1636 actual=currentDevicesResult,
1637 onpass="ONOS" + controllerStr +
1638 " Switches view is correct",
1639 onfail="ONOS" + controllerStr +
1640 " Switches view is incorrect" )
1641 if links[ controller ] and "Error" not in links[ controller ]:
1642 currentLinksResult = main.Mininet1.compareLinks(
1643 mnSwitches, mnLinks,
1644 json.loads( links[ controller ] ) )
1645 else:
1646 currentLinksResult = main.FALSE
1647 utilities.assert_equals( expect=main.TRUE,
1648 actual=currentLinksResult,
1649 onpass="ONOS" + controllerStr +
1650 " links view is correct",
1651 onfail="ONOS" + controllerStr +
1652 " links view is incorrect" )
1653
1654 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1655 currentHostsResult = main.Mininet1.compareHosts(
1656 mnHosts,
1657 hosts[ controller ] )
1658 else:
1659 currentHostsResult = main.FALSE
1660 utilities.assert_equals( expect=main.TRUE,
1661 actual=currentHostsResult,
1662 onpass="ONOS" + controllerStr +
1663 " hosts exist in Mininet",
1664 onfail="ONOS" + controllerStr +
1665 " hosts don't match Mininet" )
1666
1667 devicesResults = devicesResults and currentDevicesResult
1668 linksResults = linksResults and currentLinksResult
1669 hostsResults = hostsResults and currentHostsResult
1670
1671 main.step( "Device information is correct" )
1672 utilities.assert_equals(
1673 expect=main.TRUE,
1674 actual=devicesResults,
1675 onpass="Device information is correct",
1676 onfail="Device information is incorrect" )
1677
1678 main.step( "Links are correct" )
1679 utilities.assert_equals(
1680 expect=main.TRUE,
1681 actual=linksResults,
1682 onpass="Link are correct",
1683 onfail="Links are incorrect" )
1684
1685 main.step( "Hosts are correct" )
1686 utilities.assert_equals(
1687 expect=main.TRUE,
1688 actual=hostsResults,
1689 onpass="Hosts are correct",
1690 onfail="Hosts are incorrect" )
1691
1692 def CASE6( self, main ):
1693 """
1694 The Failure case.
1695 """
1696 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001697 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001698 assert main, "main not defined"
1699 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001700 assert main.CLIs, "main.CLIs not defined"
1701 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001702 try:
1703 labels
1704 except NameError:
1705 main.log.error( "labels not defined, setting to []" )
1706 global labels
1707 labels = []
1708 try:
1709 data
1710 except NameError:
1711 main.log.error( "data not defined, setting to []" )
1712 global data
1713 data = []
1714 # Reset non-persistent variables
1715 try:
1716 iCounterValue = 0
1717 except NameError:
1718 main.log.error( "iCounterValue not defined, setting to 0" )
1719 iCounterValue = 0
1720
1721 main.case( "Restart entire ONOS cluster" )
1722
Jon Hall5ec6b1b2015-09-17 18:20:14 -07001723 main.step( "Checking ONOS Logs for errors" )
1724 for node in main.nodes:
1725 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1726 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1727
Jon Hall5cf14d52015-07-16 12:15:19 -07001728 main.step( "Killing ONOS nodes" )
1729 killResults = main.TRUE
1730 killTime = time.time()
Jon Halle1a3b752015-07-22 13:02:46 -07001731 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001732 killed = main.ONOSbench.onosKill( node.ip_address )
1733 killResults = killResults and killed
1734 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1735 onpass="ONOS nodes killed",
1736 onfail="ONOS kill unsuccessful" )
1737
1738 main.step( "Checking if ONOS is up yet" )
1739 for i in range( 2 ):
1740 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001741 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001742 started = main.ONOSbench.isup( node.ip_address )
1743 if not started:
1744 main.log.error( node.name + " didn't start!" )
1745 onosIsupResult = onosIsupResult and started
1746 if onosIsupResult == main.TRUE:
1747 break
1748 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1749 onpass="ONOS restarted",
1750 onfail="ONOS restart NOT successful" )
1751
1752 main.log.step( "Starting ONOS CLI sessions" )
1753 cliResults = main.TRUE
1754 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001755 for i in range( main.numCtrls ):
1756 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -07001757 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -07001758 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001759 threads.append( t )
1760 t.start()
1761
1762 for t in threads:
1763 t.join()
1764 cliResults = cliResults and t.result
1765 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1766 onpass="ONOS cli started",
1767 onfail="ONOS clis did not restart" )
1768
1769 # Grab the time of restart so we chan check how long the gossip
1770 # protocol has had time to work
1771 main.restartTime = time.time() - killTime
1772 main.log.debug( "Restart time: " + str( main.restartTime ) )
1773 labels.append( "Restart" )
1774 data.append( str( main.restartTime ) )
1775
1776 # FIXME: revisit test plan for election with madan
1777 # Rerun for election on restarted nodes
1778 runResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001779 for cli in main.CLIs:
1780 run = main.CLIs[0].electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001781 if run != main.TRUE:
1782 main.log.error( "Error running for election on " + cli.name )
1783 runResults = runResults and run
1784 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1785 onpass="Reran for election",
1786 onfail="Failed to rerun for election" )
1787
1788 # TODO: Make this configurable
1789 time.sleep( 60 )
Jon Halle1a3b752015-07-22 13:02:46 -07001790 main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) )
1791 main.log.debug( main.CLIs[0].leaders( jsonFormat=False ) )
1792 main.log.debug( main.CLIs[0].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001793
1794 def CASE7( self, main ):
1795 """
1796 Check state after ONOS failure
1797 """
1798 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001799 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001800 assert main, "main not defined"
1801 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001802 assert main.CLIs, "main.CLIs not defined"
1803 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001804 main.case( "Running ONOS Constant State Tests" )
1805
1806 main.step( "Check that each switch has a master" )
1807 # Assert that each device has a master
1808 rolesNotNull = main.TRUE
1809 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001810 for i in range( main.numCtrls ):
1811 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001812 name="rolesNotNull-" + str( i ),
1813 args=[ ] )
1814 threads.append( t )
1815 t.start()
1816
1817 for t in threads:
1818 t.join()
1819 rolesNotNull = rolesNotNull and t.result
1820 utilities.assert_equals(
1821 expect=main.TRUE,
1822 actual=rolesNotNull,
1823 onpass="Each device has a master",
1824 onfail="Some devices don't have a master assigned" )
1825
1826 main.step( "Read device roles from ONOS" )
1827 ONOSMastership = []
1828 mastershipCheck = main.FALSE
1829 consistentMastership = True
1830 rolesResults = True
1831 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001832 for i in range( main.numCtrls ):
1833 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001834 name="roles-" + str( i ),
1835 args=[] )
1836 threads.append( t )
1837 t.start()
1838
1839 for t in threads:
1840 t.join()
1841 ONOSMastership.append( t.result )
1842
Jon Halle1a3b752015-07-22 13:02:46 -07001843 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001844 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1845 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1846 " roles" )
1847 main.log.warn(
1848 "ONOS" + str( i + 1 ) + " mastership response: " +
1849 repr( ONOSMastership[i] ) )
1850 rolesResults = False
1851 utilities.assert_equals(
1852 expect=True,
1853 actual=rolesResults,
1854 onpass="No error in reading roles output",
1855 onfail="Error in reading roles from ONOS" )
1856
1857 main.step( "Check for consistency in roles from each controller" )
1858 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1859 main.log.info(
1860 "Switch roles are consistent across all ONOS nodes" )
1861 else:
1862 consistentMastership = False
1863 utilities.assert_equals(
1864 expect=True,
1865 actual=consistentMastership,
1866 onpass="Switch roles are consistent across all ONOS nodes",
1867 onfail="ONOS nodes have different views of switch roles" )
1868
1869 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001870 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001871 main.log.warn(
1872 "ONOS" + str( i + 1 ) + " roles: ",
1873 json.dumps(
1874 json.loads( ONOSMastership[ i ] ),
1875 sort_keys=True,
1876 indent=4,
1877 separators=( ',', ': ' ) ) )
1878 elif rolesResults and not consistentMastership:
1879 mastershipCheck = main.TRUE
1880
1881 '''
1882 description2 = "Compare switch roles from before failure"
1883 main.step( description2 )
1884 try:
1885 currentJson = json.loads( ONOSMastership[0] )
1886 oldJson = json.loads( mastershipState )
1887 except ( ValueError, TypeError ):
1888 main.log.exception( "Something is wrong with parsing " +
1889 "ONOSMastership[0] or mastershipState" )
1890 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1891 main.log.error( "mastershipState" + repr( mastershipState ) )
1892 main.cleanup()
1893 main.exit()
1894 mastershipCheck = main.TRUE
1895 for i in range( 1, 29 ):
1896 switchDPID = str(
1897 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1898 current = [ switch[ 'master' ] for switch in currentJson
1899 if switchDPID in switch[ 'id' ] ]
1900 old = [ switch[ 'master' ] for switch in oldJson
1901 if switchDPID in switch[ 'id' ] ]
1902 if current == old:
1903 mastershipCheck = mastershipCheck and main.TRUE
1904 else:
1905 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1906 mastershipCheck = main.FALSE
1907 utilities.assert_equals(
1908 expect=main.TRUE,
1909 actual=mastershipCheck,
1910 onpass="Mastership of Switches was not changed",
1911 onfail="Mastership of some switches changed" )
1912 '''
1913 # NOTE: we expect mastership to change on controller failure
1914
1915 main.step( "Get the intents and compare across all nodes" )
1916 ONOSIntents = []
1917 intentCheck = main.FALSE
1918 consistentIntents = True
1919 intentsResults = True
1920 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001921 for i in range( main.numCtrls ):
1922 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001923 name="intents-" + str( i ),
1924 args=[],
1925 kwargs={ 'jsonFormat': True } )
1926 threads.append( t )
1927 t.start()
1928
1929 for t in threads:
1930 t.join()
1931 ONOSIntents.append( t.result )
1932
Jon Halle1a3b752015-07-22 13:02:46 -07001933 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001934 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1935 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1936 " intents" )
1937 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1938 repr( ONOSIntents[ i ] ) )
1939 intentsResults = False
1940 utilities.assert_equals(
1941 expect=True,
1942 actual=intentsResults,
1943 onpass="No error in reading intents output",
1944 onfail="Error in reading intents from ONOS" )
1945
1946 main.step( "Check for consistency in Intents from each controller" )
1947 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1948 main.log.info( "Intents are consistent across all ONOS " +
1949 "nodes" )
1950 else:
1951 consistentIntents = False
1952
1953 # Try to make it easy to figure out what is happening
1954 #
1955 # Intent ONOS1 ONOS2 ...
1956 # 0x01 INSTALLED INSTALLING
1957 # ... ... ...
1958 # ... ... ...
1959 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001960 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001961 title += " " * 10 + "ONOS" + str( n + 1 )
1962 main.log.warn( title )
1963 # get all intent keys in the cluster
1964 keys = []
1965 for nodeStr in ONOSIntents:
1966 node = json.loads( nodeStr )
1967 for intent in node:
1968 keys.append( intent.get( 'id' ) )
1969 keys = set( keys )
1970 for key in keys:
1971 row = "%-13s" % key
1972 for nodeStr in ONOSIntents:
1973 node = json.loads( nodeStr )
1974 for intent in node:
1975 if intent.get( 'id' ) == key:
1976 row += "%-15s" % intent.get( 'state' )
1977 main.log.warn( row )
1978 # End table view
1979
1980 utilities.assert_equals(
1981 expect=True,
1982 actual=consistentIntents,
1983 onpass="Intents are consistent across all ONOS nodes",
1984 onfail="ONOS nodes have different views of intents" )
1985 intentStates = []
1986 for node in ONOSIntents: # Iter through ONOS nodes
1987 nodeStates = []
1988 # Iter through intents of a node
1989 try:
1990 for intent in json.loads( node ):
1991 nodeStates.append( intent[ 'state' ] )
1992 except ( ValueError, TypeError ):
1993 main.log.exception( "Error in parsing intents" )
1994 main.log.error( repr( node ) )
1995 intentStates.append( nodeStates )
1996 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1997 main.log.info( dict( out ) )
1998
1999 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07002000 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002001 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
2002 main.log.warn( json.dumps(
2003 json.loads( ONOSIntents[ i ] ),
2004 sort_keys=True,
2005 indent=4,
2006 separators=( ',', ': ' ) ) )
2007 elif intentsResults and consistentIntents:
2008 intentCheck = main.TRUE
2009
2010 # NOTE: Store has no durability, so intents are lost across system
2011 # restarts
2012 """
2013 main.step( "Compare current intents with intents before the failure" )
2014 # NOTE: this requires case 5 to pass for intentState to be set.
2015 # maybe we should stop the test if that fails?
2016 sameIntents = main.FALSE
2017 if intentState and intentState == ONOSIntents[ 0 ]:
2018 sameIntents = main.TRUE
2019 main.log.info( "Intents are consistent with before failure" )
2020 # TODO: possibly the states have changed? we may need to figure out
2021 # what the acceptable states are
2022 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2023 sameIntents = main.TRUE
2024 try:
2025 before = json.loads( intentState )
2026 after = json.loads( ONOSIntents[ 0 ] )
2027 for intent in before:
2028 if intent not in after:
2029 sameIntents = main.FALSE
2030 main.log.debug( "Intent is not currently in ONOS " +
2031 "(at least in the same form):" )
2032 main.log.debug( json.dumps( intent ) )
2033 except ( ValueError, TypeError ):
2034 main.log.exception( "Exception printing intents" )
2035 main.log.debug( repr( ONOSIntents[0] ) )
2036 main.log.debug( repr( intentState ) )
2037 if sameIntents == main.FALSE:
2038 try:
2039 main.log.debug( "ONOS intents before: " )
2040 main.log.debug( json.dumps( json.loads( intentState ),
2041 sort_keys=True, indent=4,
2042 separators=( ',', ': ' ) ) )
2043 main.log.debug( "Current ONOS intents: " )
2044 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2045 sort_keys=True, indent=4,
2046 separators=( ',', ': ' ) ) )
2047 except ( ValueError, TypeError ):
2048 main.log.exception( "Exception printing intents" )
2049 main.log.debug( repr( ONOSIntents[0] ) )
2050 main.log.debug( repr( intentState ) )
2051 utilities.assert_equals(
2052 expect=main.TRUE,
2053 actual=sameIntents,
2054 onpass="Intents are consistent with before failure",
2055 onfail="The Intents changed during failure" )
2056 intentCheck = intentCheck and sameIntents
2057 """
2058 main.step( "Get the OF Table entries and compare to before " +
2059 "component failure" )
2060 FlowTables = main.TRUE
2061 flows2 = []
2062 for i in range( 28 ):
2063 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall9043c902015-07-30 14:23:44 -07002064 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002065 flows2.append( tmpFlows )
Jon Hall9043c902015-07-30 14:23:44 -07002066 tempResult = main.Mininet1.flowComp(
Jon Hall5cf14d52015-07-16 12:15:19 -07002067 flow1=flows[ i ],
2068 flow2=tmpFlows )
2069 FlowTables = FlowTables and tempResult
2070 if FlowTables == main.FALSE:
2071 main.log.info( "Differences in flow table for switch: s" +
2072 str( i + 1 ) )
2073 utilities.assert_equals(
2074 expect=main.TRUE,
2075 actual=FlowTables,
2076 onpass="No changes were found in the flow tables",
2077 onfail="Changes were found in the flow tables" )
2078
2079 main.Mininet2.pingLongKill()
2080 '''
2081 # main.step( "Check the continuous pings to ensure that no packets " +
2082 # "were dropped during component failure" )
2083 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2084 main.params[ 'TESTONIP' ] )
2085 LossInPings = main.FALSE
2086 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2087 for i in range( 8, 18 ):
2088 main.log.info(
2089 "Checking for a loss in pings along flow from s" +
2090 str( i ) )
2091 LossInPings = main.Mininet2.checkForLoss(
2092 "/tmp/ping.h" +
2093 str( i ) ) or LossInPings
2094 if LossInPings == main.TRUE:
2095 main.log.info( "Loss in ping detected" )
2096 elif LossInPings == main.ERROR:
2097 main.log.info( "There are multiple mininet process running" )
2098 elif LossInPings == main.FALSE:
2099 main.log.info( "No Loss in the pings" )
2100 main.log.info( "No loss of dataplane connectivity" )
2101 # utilities.assert_equals(
2102 # expect=main.FALSE,
2103 # actual=LossInPings,
2104 # onpass="No Loss of connectivity",
2105 # onfail="Loss of dataplane connectivity detected" )
2106
2107 # NOTE: Since intents are not persisted with IntnentStore,
2108 # we expect loss in dataplane connectivity
2109 LossInPings = main.FALSE
2110 '''
2111
2112 main.step( "Leadership Election is still functional" )
2113 # Test of LeadershipElection
2114 leaderList = []
2115 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002116 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002117 leaderN = cli.electionTestLeader()
2118 leaderList.append( leaderN )
2119 if leaderN == main.FALSE:
2120 # error in response
2121 main.log.error( "Something is wrong with " +
2122 "electionTestLeader function, check the" +
2123 " error logs" )
2124 leaderResult = main.FALSE
2125 elif leaderN is None:
2126 main.log.error( cli.name +
2127 " shows no leader for the election-app." )
2128 leaderResult = main.FALSE
2129 if len( set( leaderList ) ) != 1:
2130 leaderResult = main.FALSE
2131 main.log.error(
2132 "Inconsistent view of leader for the election test app" )
2133 # TODO: print the list
2134 utilities.assert_equals(
2135 expect=main.TRUE,
2136 actual=leaderResult,
2137 onpass="Leadership election passed",
2138 onfail="Something went wrong with Leadership election" )
2139
2140 def CASE8( self, main ):
2141 """
2142 Compare topo
2143 """
2144 import json
2145 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002146 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002147 assert main, "main not defined"
2148 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002149 assert main.CLIs, "main.CLIs not defined"
2150 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002151
2152 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002153 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002154 " and ONOS"
2155
2156 main.step( "Comparing ONOS topology to MN" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002157 topoResult = main.FALSE
2158 elapsed = 0
2159 count = 0
2160 main.step( "Collecting topology information from ONOS" )
2161 startTime = time.time()
2162 # Give time for Gossip to work
2163 while topoResult == main.FALSE and elapsed < 60:
Jon Hallba609822015-09-18 12:00:21 -07002164 devicesResults = main.TRUE
2165 linksResults = main.TRUE
2166 hostsResults = main.TRUE
2167 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002168 count += 1
2169 cliStart = time.time()
2170 devices = []
2171 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002172 for i in range( main.numCtrls ):
2173 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002174 name="devices-" + str( i ),
2175 args=[ ] )
2176 threads.append( t )
2177 t.start()
2178
2179 for t in threads:
2180 t.join()
2181 devices.append( t.result )
2182 hosts = []
2183 ipResult = main.TRUE
2184 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002185 for i in range( main.numCtrls ):
2186 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07002187 name="hosts-" + str( i ),
2188 args=[ ] )
2189 threads.append( t )
2190 t.start()
2191
2192 for t in threads:
2193 t.join()
2194 try:
2195 hosts.append( json.loads( t.result ) )
2196 except ( ValueError, TypeError ):
2197 main.log.exception( "Error parsing hosts results" )
2198 main.log.error( repr( t.result ) )
2199 for controller in range( 0, len( hosts ) ):
2200 controllerStr = str( controller + 1 )
2201 for host in hosts[ controller ]:
2202 if host is None or host.get( 'ipAddresses', [] ) == []:
2203 main.log.error(
2204 "DEBUG:Error with host ipAddresses on controller" +
2205 controllerStr + ": " + str( host ) )
2206 ipResult = main.FALSE
2207 ports = []
2208 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002209 for i in range( main.numCtrls ):
2210 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002211 name="ports-" + str( i ),
2212 args=[ ] )
2213 threads.append( t )
2214 t.start()
2215
2216 for t in threads:
2217 t.join()
2218 ports.append( t.result )
2219 links = []
2220 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002221 for i in range( main.numCtrls ):
2222 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002223 name="links-" + str( i ),
2224 args=[ ] )
2225 threads.append( t )
2226 t.start()
2227
2228 for t in threads:
2229 t.join()
2230 links.append( t.result )
2231 clusters = []
2232 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002233 for i in range( main.numCtrls ):
2234 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002235 name="clusters-" + str( i ),
2236 args=[ ] )
2237 threads.append( t )
2238 t.start()
2239
2240 for t in threads:
2241 t.join()
2242 clusters.append( t.result )
2243
2244 elapsed = time.time() - startTime
2245 cliTime = time.time() - cliStart
2246 print "Elapsed time: " + str( elapsed )
2247 print "CLI time: " + str( cliTime )
2248
2249 mnSwitches = main.Mininet1.getSwitches()
2250 mnLinks = main.Mininet1.getLinks()
2251 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002252 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002253 controllerStr = str( controller + 1 )
2254 if devices[ controller ] and ports[ controller ] and\
2255 "Error" not in devices[ controller ] and\
2256 "Error" not in ports[ controller ]:
2257
2258 currentDevicesResult = main.Mininet1.compareSwitches(
2259 mnSwitches,
2260 json.loads( devices[ controller ] ),
2261 json.loads( ports[ controller ] ) )
2262 else:
2263 currentDevicesResult = main.FALSE
2264 utilities.assert_equals( expect=main.TRUE,
2265 actual=currentDevicesResult,
2266 onpass="ONOS" + controllerStr +
2267 " Switches view is correct",
2268 onfail="ONOS" + controllerStr +
2269 " Switches view is incorrect" )
2270
2271 if links[ controller ] and "Error" not in links[ controller ]:
2272 currentLinksResult = main.Mininet1.compareLinks(
2273 mnSwitches, mnLinks,
2274 json.loads( links[ controller ] ) )
2275 else:
2276 currentLinksResult = main.FALSE
2277 utilities.assert_equals( expect=main.TRUE,
2278 actual=currentLinksResult,
2279 onpass="ONOS" + controllerStr +
2280 " links view is correct",
2281 onfail="ONOS" + controllerStr +
2282 " links view is incorrect" )
2283
2284 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2285 currentHostsResult = main.Mininet1.compareHosts(
2286 mnHosts,
2287 hosts[ controller ] )
2288 else:
2289 currentHostsResult = main.FALSE
2290 utilities.assert_equals( expect=main.TRUE,
2291 actual=currentHostsResult,
2292 onpass="ONOS" + controllerStr +
2293 " hosts exist in Mininet",
2294 onfail="ONOS" + controllerStr +
2295 " hosts don't match Mininet" )
2296 # CHECKING HOST ATTACHMENT POINTS
2297 hostAttachment = True
2298 noHosts = False
2299 # FIXME: topo-HA/obelisk specific mappings:
2300 # key is mac and value is dpid
2301 mappings = {}
2302 for i in range( 1, 29 ): # hosts 1 through 28
2303 # set up correct variables:
2304 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2305 if i == 1:
2306 deviceId = "1000".zfill(16)
2307 elif i == 2:
2308 deviceId = "2000".zfill(16)
2309 elif i == 3:
2310 deviceId = "3000".zfill(16)
2311 elif i == 4:
2312 deviceId = "3004".zfill(16)
2313 elif i == 5:
2314 deviceId = "5000".zfill(16)
2315 elif i == 6:
2316 deviceId = "6000".zfill(16)
2317 elif i == 7:
2318 deviceId = "6007".zfill(16)
2319 elif i >= 8 and i <= 17:
2320 dpid = '3' + str( i ).zfill( 3 )
2321 deviceId = dpid.zfill(16)
2322 elif i >= 18 and i <= 27:
2323 dpid = '6' + str( i ).zfill( 3 )
2324 deviceId = dpid.zfill(16)
2325 elif i == 28:
2326 deviceId = "2800".zfill(16)
2327 mappings[ macId ] = deviceId
2328 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2329 if hosts[ controller ] == []:
2330 main.log.warn( "There are no hosts discovered" )
2331 noHosts = True
2332 else:
2333 for host in hosts[ controller ]:
2334 mac = None
2335 location = None
2336 device = None
2337 port = None
2338 try:
2339 mac = host.get( 'mac' )
2340 assert mac, "mac field could not be found for this host object"
2341
2342 location = host.get( 'location' )
2343 assert location, "location field could not be found for this host object"
2344
2345 # Trim the protocol identifier off deviceId
2346 device = str( location.get( 'elementId' ) ).split(':')[1]
2347 assert device, "elementId field could not be found for this host location object"
2348
2349 port = location.get( 'port' )
2350 assert port, "port field could not be found for this host location object"
2351
2352 # Now check if this matches where they should be
2353 if mac and device and port:
2354 if str( port ) != "1":
2355 main.log.error( "The attachment port is incorrect for " +
2356 "host " + str( mac ) +
2357 ". Expected: 1 Actual: " + str( port) )
2358 hostAttachment = False
2359 if device != mappings[ str( mac ) ]:
2360 main.log.error( "The attachment device is incorrect for " +
2361 "host " + str( mac ) +
2362 ". Expected: " + mappings[ str( mac ) ] +
2363 " Actual: " + device )
2364 hostAttachment = False
2365 else:
2366 hostAttachment = False
2367 except AssertionError:
2368 main.log.exception( "Json object not as expected" )
2369 main.log.error( repr( host ) )
2370 hostAttachment = False
2371 else:
2372 main.log.error( "No hosts json output or \"Error\"" +
2373 " in output. hosts = " +
2374 repr( hosts[ controller ] ) )
2375 if noHosts is False:
2376 # TODO: Find a way to know if there should be hosts in a
2377 # given point of the test
2378 hostAttachment = True
2379
2380 # END CHECKING HOST ATTACHMENT POINTS
2381 devicesResults = devicesResults and currentDevicesResult
2382 linksResults = linksResults and currentLinksResult
2383 hostsResults = hostsResults and currentHostsResult
2384 hostAttachmentResults = hostAttachmentResults and\
2385 hostAttachment
2386 topoResult = ( devicesResults and linksResults
2387 and hostsResults and ipResult and
2388 hostAttachmentResults )
2389
2390 # Compare json objects for hosts and dataplane clusters
2391
2392 # hosts
2393 main.step( "Hosts view is consistent across all ONOS nodes" )
2394 consistentHostsResult = main.TRUE
2395 for controller in range( len( hosts ) ):
2396 controllerStr = str( controller + 1 )
2397 if "Error" not in hosts[ controller ]:
2398 if hosts[ controller ] == hosts[ 0 ]:
2399 continue
2400 else: # hosts not consistent
2401 main.log.error( "hosts from ONOS" + controllerStr +
2402 " is inconsistent with ONOS1" )
2403 main.log.warn( repr( hosts[ controller ] ) )
2404 consistentHostsResult = main.FALSE
2405
2406 else:
2407 main.log.error( "Error in getting ONOS hosts from ONOS" +
2408 controllerStr )
2409 consistentHostsResult = main.FALSE
2410 main.log.warn( "ONOS" + controllerStr +
2411 " hosts response: " +
2412 repr( hosts[ controller ] ) )
2413 utilities.assert_equals(
2414 expect=main.TRUE,
2415 actual=consistentHostsResult,
2416 onpass="Hosts view is consistent across all ONOS nodes",
2417 onfail="ONOS nodes have different views of hosts" )
2418
2419 main.step( "Hosts information is correct" )
2420 hostsResults = hostsResults and ipResult
2421 utilities.assert_equals(
2422 expect=main.TRUE,
2423 actual=hostsResults,
2424 onpass="Host information is correct",
2425 onfail="Host information is incorrect" )
2426
2427 main.step( "Host attachment points to the network" )
2428 utilities.assert_equals(
2429 expect=True,
2430 actual=hostAttachmentResults,
2431 onpass="Hosts are correctly attached to the network",
2432 onfail="ONOS did not correctly attach hosts to the network" )
2433
2434 # Strongly connected clusters of devices
2435 main.step( "Clusters view is consistent across all ONOS nodes" )
2436 consistentClustersResult = main.TRUE
2437 for controller in range( len( clusters ) ):
2438 controllerStr = str( controller + 1 )
2439 if "Error" not in clusters[ controller ]:
2440 if clusters[ controller ] == clusters[ 0 ]:
2441 continue
2442 else: # clusters not consistent
2443 main.log.error( "clusters from ONOS" +
2444 controllerStr +
2445 " is inconsistent with ONOS1" )
2446 consistentClustersResult = main.FALSE
2447
2448 else:
2449 main.log.error( "Error in getting dataplane clusters " +
2450 "from ONOS" + controllerStr )
2451 consistentClustersResult = main.FALSE
2452 main.log.warn( "ONOS" + controllerStr +
2453 " clusters response: " +
2454 repr( clusters[ controller ] ) )
2455 utilities.assert_equals(
2456 expect=main.TRUE,
2457 actual=consistentClustersResult,
2458 onpass="Clusters view is consistent across all ONOS nodes",
2459 onfail="ONOS nodes have different views of clusters" )
2460
2461 main.step( "There is only one SCC" )
2462 # there should always only be one cluster
2463 try:
2464 numClusters = len( json.loads( clusters[ 0 ] ) )
2465 except ( ValueError, TypeError ):
2466 main.log.exception( "Error parsing clusters[0]: " +
2467 repr( clusters[0] ) )
2468 clusterResults = main.FALSE
2469 if numClusters == 1:
2470 clusterResults = main.TRUE
2471 utilities.assert_equals(
2472 expect=1,
2473 actual=numClusters,
2474 onpass="ONOS shows 1 SCC",
2475 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2476
2477 topoResult = ( devicesResults and linksResults
2478 and hostsResults and consistentHostsResult
2479 and consistentClustersResult and clusterResults
2480 and ipResult and hostAttachmentResults )
2481
2482 topoResult = topoResult and int( count <= 2 )
2483 note = "note it takes about " + str( int( cliTime ) ) + \
2484 " seconds for the test to make all the cli calls to fetch " +\
2485 "the topology from each ONOS instance"
2486 main.log.info(
2487 "Very crass estimate for topology discovery/convergence( " +
2488 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2489 str( count ) + " tries" )
2490
2491 main.step( "Device information is correct" )
2492 utilities.assert_equals(
2493 expect=main.TRUE,
2494 actual=devicesResults,
2495 onpass="Device information is correct",
2496 onfail="Device information is incorrect" )
2497
2498 main.step( "Links are correct" )
2499 utilities.assert_equals(
2500 expect=main.TRUE,
2501 actual=linksResults,
2502 onpass="Link are correct",
2503 onfail="Links are incorrect" )
2504
2505 # FIXME: move this to an ONOS state case
2506 main.step( "Checking ONOS nodes" )
2507 nodesOutput = []
2508 nodeResults = main.TRUE
2509 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002510 for i in range( main.numCtrls ):
2511 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002512 name="nodes-" + str( i ),
2513 args=[ ] )
2514 threads.append( t )
2515 t.start()
2516
2517 for t in threads:
2518 t.join()
2519 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002520 ips = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002521 for i in nodesOutput:
2522 try:
2523 current = json.loads( i )
2524 for node in current:
2525 currentResult = main.FALSE
2526 if node['ip'] in ips: # node in nodes() output is in cell
2527 if node['state'] == 'ACTIVE':
2528 currentResult = main.TRUE
2529 else:
2530 main.log.error( "Error in ONOS node availability" )
2531 main.log.error(
2532 json.dumps( current,
2533 sort_keys=True,
2534 indent=4,
2535 separators=( ',', ': ' ) ) )
2536 break
2537 nodeResults = nodeResults and currentResult
2538 except ( ValueError, TypeError ):
2539 main.log.error( "Error parsing nodes output" )
2540 main.log.warn( repr( i ) )
2541 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2542 onpass="Nodes check successful",
2543 onfail="Nodes check NOT successful" )
2544
2545 def CASE9( self, main ):
2546 """
2547 Link s3-s28 down
2548 """
2549 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002550 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002551 assert main, "main not defined"
2552 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002553 assert main.CLIs, "main.CLIs not defined"
2554 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002555 # NOTE: You should probably run a topology check after this
2556
2557 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2558
2559 description = "Turn off a link to ensure that Link Discovery " +\
2560 "is working properly"
2561 main.case( description )
2562
2563 main.step( "Kill Link between s3 and s28" )
2564 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2565 main.log.info( "Waiting " + str( linkSleep ) +
2566 " seconds for link down to be discovered" )
2567 time.sleep( linkSleep )
2568 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2569 onpass="Link down successful",
2570 onfail="Failed to bring link down" )
2571 # TODO do some sort of check here
2572
2573 def CASE10( self, main ):
2574 """
2575 Link s3-s28 up
2576 """
2577 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002578 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002579 assert main, "main not defined"
2580 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002581 assert main.CLIs, "main.CLIs not defined"
2582 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002583 # NOTE: You should probably run a topology check after this
2584
2585 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2586
2587 description = "Restore a link to ensure that Link Discovery is " + \
2588 "working properly"
2589 main.case( description )
2590
2591 main.step( "Bring link between s3 and s28 back up" )
2592 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2593 main.log.info( "Waiting " + str( linkSleep ) +
2594 " seconds for link up to be discovered" )
2595 time.sleep( linkSleep )
2596 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2597 onpass="Link up successful",
2598 onfail="Failed to bring link up" )
2599 # TODO do some sort of check here
2600
2601 def CASE11( self, main ):
2602 """
2603 Switch Down
2604 """
2605 # NOTE: You should probably run a topology check after this
2606 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002607 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002608 assert main, "main not defined"
2609 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002610 assert main.CLIs, "main.CLIs not defined"
2611 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002612
2613 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2614
2615 description = "Killing a switch to ensure it is discovered correctly"
2616 main.case( description )
2617 switch = main.params[ 'kill' ][ 'switch' ]
2618 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2619
2620 # TODO: Make this switch parameterizable
2621 main.step( "Kill " + switch )
2622 main.log.info( "Deleting " + switch )
2623 main.Mininet1.delSwitch( switch )
2624 main.log.info( "Waiting " + str( switchSleep ) +
2625 " seconds for switch down to be discovered" )
2626 time.sleep( switchSleep )
2627 device = main.ONOScli1.getDevice( dpid=switchDPID )
2628 # Peek at the deleted switch
2629 main.log.warn( str( device ) )
2630 result = main.FALSE
2631 if device and device[ 'available' ] is False:
2632 result = main.TRUE
2633 utilities.assert_equals( expect=main.TRUE, actual=result,
2634 onpass="Kill switch successful",
2635 onfail="Failed to kill switch?" )
2636
2637 def CASE12( self, main ):
2638 """
2639 Switch Up
2640 """
2641 # NOTE: You should probably run a topology check after this
2642 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002643 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002644 assert main, "main not defined"
2645 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002646 assert main.CLIs, "main.CLIs not defined"
2647 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002648 assert ONOS1Port, "ONOS1Port not defined"
2649 assert ONOS2Port, "ONOS2Port not defined"
2650 assert ONOS3Port, "ONOS3Port not defined"
2651 assert ONOS4Port, "ONOS4Port not defined"
2652 assert ONOS5Port, "ONOS5Port not defined"
2653 assert ONOS6Port, "ONOS6Port not defined"
2654 assert ONOS7Port, "ONOS7Port not defined"
2655
2656 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2657 switch = main.params[ 'kill' ][ 'switch' ]
2658 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2659 links = main.params[ 'kill' ][ 'links' ].split()
2660 description = "Adding a switch to ensure it is discovered correctly"
2661 main.case( description )
2662
2663 main.step( "Add back " + switch )
2664 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2665 for peer in links:
2666 main.Mininet1.addLink( switch, peer )
2667 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002668 for i in range( main.numCtrls ):
2669 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002670 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2671 main.log.info( "Waiting " + str( switchSleep ) +
2672 " seconds for switch up to be discovered" )
2673 time.sleep( switchSleep )
2674 device = main.ONOScli1.getDevice( dpid=switchDPID )
2675 # Peek at the deleted switch
2676 main.log.warn( str( device ) )
2677 result = main.FALSE
2678 if device and device[ 'available' ]:
2679 result = main.TRUE
2680 utilities.assert_equals( expect=main.TRUE, actual=result,
2681 onpass="add switch successful",
2682 onfail="Failed to add switch?" )
2683
2684 def CASE13( self, main ):
2685 """
2686 Clean up
2687 """
2688 import os
2689 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002690 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002691 assert main, "main not defined"
2692 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002693 assert main.CLIs, "main.CLIs not defined"
2694 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002695
2696 # printing colors to terminal
2697 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2698 'blue': '\033[94m', 'green': '\033[92m',
2699 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2700 main.case( "Test Cleanup" )
2701 main.step( "Killing tcpdumps" )
2702 main.Mininet2.stopTcpdump()
2703
2704 testname = main.TEST
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002705 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002706 main.step( "Copying MN pcap and ONOS log files to test station" )
2707 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2708 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002709 # NOTE: MN Pcap file is being saved to logdir.
2710 # We scp this file as MN and TestON aren't necessarily the same vm
2711
2712 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002713 # TODO: Load these from params
2714 # NOTE: must end in /
2715 logFolder = "/opt/onos/log/"
2716 logFiles = [ "karaf.log", "karaf.log.1" ]
2717 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002718 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002719 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002720 dstName = main.logdir + "/" + node.name + "-" + f
2721 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2722 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002723 # std*.log's
2724 # NOTE: must end in /
2725 logFolder = "/opt/onos/var/"
2726 logFiles = [ "stderr.log", "stdout.log" ]
2727 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002728 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002729 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002730 dstName = main.logdir + "/" + node.name + "-" + f
2731 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2732 logFolder + f, dstName )
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002733 else:
2734 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002735
2736 main.step( "Stopping Mininet" )
2737 mnResult = main.Mininet1.stopNet()
2738 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2739 onpass="Mininet stopped",
2740 onfail="MN cleanup NOT successful" )
2741
2742 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002743 for node in main.nodes:
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002744 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2745 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002746
2747 try:
2748 timerLog = open( main.logdir + "/Timers.csv", 'w')
2749 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2750 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2751 timerLog.close()
2752 except NameError, e:
2753 main.log.exception(e)
2754
2755 def CASE14( self, main ):
2756 """
2757 start election app on all onos nodes
2758 """
Jon Halle1a3b752015-07-22 13:02:46 -07002759 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002760 assert main, "main not defined"
2761 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002762 assert main.CLIs, "main.CLIs not defined"
2763 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002764
2765 main.case("Start Leadership Election app")
2766 main.step( "Install leadership election app" )
2767 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2768 utilities.assert_equals(
2769 expect=main.TRUE,
2770 actual=appResult,
2771 onpass="Election app installed",
2772 onfail="Something went wrong with installing Leadership election" )
2773
2774 main.step( "Run for election on each node" )
2775 leaderResult = main.TRUE
2776 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002777 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002778 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002779 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002780 leader = cli.electionTestLeader()
2781 if leader is None or leader == main.FALSE:
2782 main.log.error( cli.name + ": Leader for the election app " +
2783 "should be an ONOS node, instead got '" +
2784 str( leader ) + "'" )
2785 leaderResult = main.FALSE
2786 leaders.append( leader )
2787 utilities.assert_equals(
2788 expect=main.TRUE,
2789 actual=leaderResult,
2790 onpass="Successfully ran for leadership",
2791 onfail="Failed to run for leadership" )
2792
2793 main.step( "Check that each node shows the same leader" )
2794 sameLeader = main.TRUE
2795 if len( set( leaders ) ) != 1:
2796 sameLeader = main.FALSE
2797 main.log.error( "Results of electionTestLeader is order of CLIs:" +
2798 str( leaders ) )
2799 utilities.assert_equals(
2800 expect=main.TRUE,
2801 actual=sameLeader,
2802 onpass="Leadership is consistent for the election topic",
2803 onfail="Nodes have different leaders" )
2804
2805 def CASE15( self, main ):
2806 """
2807 Check that Leadership Election is still functional
acsmars9475b1c2015-08-28 18:02:08 -07002808 15.1 Run election on each node
2809 15.2 Check that each node has the same leaders and candidates
2810 15.3 Find current leader and withdraw
2811 15.4 Check that a new node was elected leader
2812 15.5 Check that that new leader was the candidate of old leader
2813 15.6 Run for election on old leader
2814 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2815 15.8 Make sure that the old leader was added to the candidate list
2816
2817 old and new variable prefixes refer to data from before vs after
2818 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002819 """
2820 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002821 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002822 assert main, "main not defined"
2823 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002824 assert main.CLIs, "main.CLIs not defined"
2825 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002826
Jon Hall5cf14d52015-07-16 12:15:19 -07002827 description = "Check that Leadership Election is still functional"
2828 main.case( description )
2829 # NOTE: Need to re-run since being a canidate is not persistant
acsmars9475b1c2015-08-28 18:02:08 -07002830 # TODO: add check for "Command not found:" in the driver, this
2831 # means the election test app isn't loaded
2832
2833 oldLeaders = [] # leaders by node before withdrawl from candidates
2834 newLeaders = [] # leaders by node after withdrawl from candidates
2835 oldAllCandidates = [] # list of lists of each nodes' candidates before
2836 newAllCandidates = [] # list of lists of each nodes' candidates after
2837 oldCandidates = [] # list of candidates from node 0 before withdrawl
2838 newCandidates = [] # list of candidates from node 0 after withdrawl
2839 oldLeader = '' # the old leader from oldLeaders, None if not same
2840 newLeader = '' # the new leaders fron newLoeaders, None if not same
2841 oldLeaderCLI = None # the CLI of the old leader used for re-electing
acsmars71adceb2015-08-31 15:09:26 -07002842 expectNoLeader = False # True when there is only one leader
2843 if main.numCtrls == 1:
2844 expectNoLeader = True
acsmars9475b1c2015-08-28 18:02:08 -07002845
Jon Hall5cf14d52015-07-16 12:15:19 -07002846 main.step( "Run for election on each node" )
acsmars9475b1c2015-08-28 18:02:08 -07002847 electionResult = main.TRUE
2848
2849 for cli in main.CLIs: # run test election on each node
2850 if cli.electionTestRun() == main.FALSE:
2851 electionResult = main.FALSE
2852
Jon Hall5cf14d52015-07-16 12:15:19 -07002853 utilities.assert_equals(
2854 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002855 actual=electionResult,
2856 onpass="All nodes successfully ran for leadership",
2857 onfail="At least one node failed to run for leadership" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002858
acsmars3a72bde2015-09-02 14:16:22 -07002859 if electionResult == main.FALSE:
2860 main.log.error(
2861 "Skipping Test Case because Election Test App isn't loaded" )
2862 main.skipCase()
2863
acsmars9475b1c2015-08-28 18:02:08 -07002864 main.step( "Check that each node shows the same leader and candidates" )
2865 sameResult = main.TRUE
2866 failMessage = "Nodes have different leaders"
2867 for cli in main.CLIs:
2868 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2869 oldAllCandidates.append( node )
2870 oldLeaders.append( node[ 0 ] )
2871 oldCandidates = oldAllCandidates[ 0 ]
2872
2873 # Check that each node has the same leader. Defines oldLeader
2874 if len( set( oldLeaders ) ) != 1:
2875 sameResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002876 main.log.error( "More than one leader present:" + str( oldLeaders ) )
acsmars9475b1c2015-08-28 18:02:08 -07002877 oldLeader = None
2878 else:
2879 oldLeader = oldLeaders[ 0 ]
2880
2881 # Check that each node's candidate list is the same
2882 for candidates in oldAllCandidates:
2883 if set( candidates ) != set( oldCandidates ):
2884 sameResult = main.FALSE
2885 failMessage += "and candidates"
2886
Jon Hall5cf14d52015-07-16 12:15:19 -07002887 utilities.assert_equals(
2888 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002889 actual=sameResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002890 onpass="Leadership is consistent for the election topic",
acsmars9475b1c2015-08-28 18:02:08 -07002891 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002892
2893 main.step( "Find current leader and withdraw" )
acsmars9475b1c2015-08-28 18:02:08 -07002894 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002895 # do some sanity checking on leader before using it
acsmars9475b1c2015-08-28 18:02:08 -07002896 if oldLeader is None:
2897 main.log.error( "Leadership isn't consistent." )
2898 withdrawResult = main.FALSE
2899 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002900 for i in range( len( main.CLIs ) ):
acsmars9475b1c2015-08-28 18:02:08 -07002901 if oldLeader == main.nodes[ i ].ip_address:
2902 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002903 break
2904 else: # FOR/ELSE statement
2905 main.log.error( "Leader election, could not find current leader" )
2906 if oldLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002907 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002908 utilities.assert_equals(
2909 expect=main.TRUE,
2910 actual=withdrawResult,
2911 onpass="Node was withdrawn from election",
2912 onfail="Node was not withdrawn from election" )
2913
acsmars9475b1c2015-08-28 18:02:08 -07002914 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002915
Jon Hall5cf14d52015-07-16 12:15:19 -07002916 # FIXME: use threads
acsmars9475b1c2015-08-28 18:02:08 -07002917 newLeaderResult = main.TRUE
2918 failMessage = "Nodes have different leaders"
2919
2920 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002921 for cli in main.CLIs:
acsmars9475b1c2015-08-28 18:02:08 -07002922 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
acsmars71adceb2015-08-31 15:09:26 -07002923 # elections might no have finished yet
2924 if node[ 0 ] == 'none' and not expectNoLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002925 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2926 "sure elections are complete." )
2927 time.sleep(5)
2928 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
acsmars71adceb2015-08-31 15:09:26 -07002929 # election still isn't done or there is a problem
2930 if node[ 0 ] == 'none':
2931 main.log.error( "No leader was elected on at least 1 node" )
2932 newLeaderResult = main.FALSE
acsmars9475b1c2015-08-28 18:02:08 -07002933 newAllCandidates.append( node )
2934 newLeaders.append( node[ 0 ] )
2935 newCandidates = newAllCandidates[ 0 ]
2936
2937 # Check that each node has the same leader. Defines newLeader
2938 if len( set( newLeaders ) ) != 1:
2939 newLeaderResult = main.FALSE
2940 main.log.error( "Nodes have different leaders: " +
2941 str( newLeaders ) )
2942 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002943 else:
acsmars9475b1c2015-08-28 18:02:08 -07002944 newLeader = newLeaders[ 0 ]
2945
acsmars71adceb2015-08-31 15:09:26 -07002946 # Check that each node's candidate list is the same
2947 for candidates in newAllCandidates:
2948 if set( candidates ) != set( newCandidates ):
2949 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002950 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002951
acsmars9475b1c2015-08-28 18:02:08 -07002952 # Check that the new leader is not the older leader, which was withdrawn
2953 if newLeader == oldLeader:
2954 newLeaderResult = main.FALSE
2955 main.log.error( "All nodes still see old leader: " + oldLeader +
2956 " as the current leader" )
2957
Jon Hall5cf14d52015-07-16 12:15:19 -07002958 utilities.assert_equals(
2959 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002960 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002961 onpass="Leadership election passed",
2962 onfail="Something went wrong with Leadership election" )
2963
acsmars9475b1c2015-08-28 18:02:08 -07002964 main.step( "Check that that new leader was the candidate of old leader")
2965 # candidates[ 2 ] should be come the top candidate after withdrawl
2966 correctCandidateResult = main.TRUE
acsmars71adceb2015-08-31 15:09:26 -07002967 if expectNoLeader:
2968 if newLeader == 'none':
2969 main.log.info( "No leader expected. None found. Pass" )
2970 correctCandidateResult = main.TRUE
2971 else:
2972 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2973 correctCandidateResult = main.FALSE
2974 elif newLeader != oldCandidates[ 2 ]:
acsmars9475b1c2015-08-28 18:02:08 -07002975 correctCandidateResult = main.FALSE
2976 main.log.error( "Candidate " + newLeader + " was elected. " +
2977 oldCandidates[ 2 ] + " should have had priority." )
2978
2979 utilities.assert_equals(
2980 expect=main.TRUE,
2981 actual=correctCandidateResult,
2982 onpass="Correct Candidate Elected",
2983 onfail="Incorrect Candidate Elected" )
2984
Jon Hall5cf14d52015-07-16 12:15:19 -07002985 main.step( "Run for election on old leader( just so everyone " +
2986 "is in the hat )" )
acsmars9475b1c2015-08-28 18:02:08 -07002987 if oldLeaderCLI is not None:
2988 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002989 else:
acsmars9475b1c2015-08-28 18:02:08 -07002990 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002991 runResult = main.FALSE
2992 utilities.assert_equals(
2993 expect=main.TRUE,
2994 actual=runResult,
2995 onpass="App re-ran for election",
2996 onfail="App failed to run for election" )
acsmars9475b1c2015-08-28 18:02:08 -07002997 main.step(
2998 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002999 # verify leader didn't just change
acsmars9475b1c2015-08-28 18:02:08 -07003000 positionResult = main.TRUE
3001 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
3002
3003 # Reset and reuse the new candidate and leaders lists
3004 newAllCandidates = []
3005 newCandidates = []
3006 newLeaders = []
3007 for cli in main.CLIs:
3008 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
3009 if oldLeader not in node: # election might no have finished yet
3010 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
3011 "be sure elections are complete" )
3012 time.sleep(5)
3013 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
3014 if oldLeader not in node: # election still isn't done, errors
3015 main.log.error(
3016 "Old leader was not elected on at least one node" )
3017 positionResult = main.FALSE
3018 newAllCandidates.append( node )
3019 newLeaders.append( node[ 0 ] )
3020 newCandidates = newAllCandidates[ 0 ]
3021
3022 # Check that each node has the same leader. Defines newLeader
3023 if len( set( newLeaders ) ) != 1:
3024 positionResult = main.FALSE
3025 main.log.error( "Nodes have different leaders: " +
3026 str( newLeaders ) )
3027 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07003028 else:
acsmars9475b1c2015-08-28 18:02:08 -07003029 newLeader = newLeaders[ 0 ]
3030
acsmars71adceb2015-08-31 15:09:26 -07003031 # Check that each node's candidate list is the same
3032 for candidates in newAllCandidates:
3033 if set( candidates ) != set( newCandidates ):
3034 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07003035 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07003036
acsmars9475b1c2015-08-28 18:02:08 -07003037 # Check that the re-elected node is last on the candidate List
3038 if oldLeader != newCandidates[ -1 ]:
3039 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
3040 str( newCandidates ) )
3041 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003042
3043 utilities.assert_equals(
3044 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07003045 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003046 onpass="Old leader successfully re-ran for election",
3047 onfail="Something went wrong with Leadership election after " +
3048 "the old leader re-ran for election" )
3049
3050 def CASE16( self, main ):
3051 """
3052 Install Distributed Primitives app
3053 """
3054 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003055 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003056 assert main, "main not defined"
3057 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003058 assert main.CLIs, "main.CLIs not defined"
3059 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003060
3061 # Variables for the distributed primitives tests
3062 global pCounterName
3063 global iCounterName
3064 global pCounterValue
3065 global iCounterValue
3066 global onosSet
3067 global onosSetName
3068 pCounterName = "TestON-Partitions"
3069 iCounterName = "TestON-inMemory"
3070 pCounterValue = 0
3071 iCounterValue = 0
3072 onosSet = set([])
3073 onosSetName = "TestON-set"
3074
3075 description = "Install Primitives app"
3076 main.case( description )
3077 main.step( "Install Primitives app" )
3078 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07003079 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003080 utilities.assert_equals( expect=main.TRUE,
3081 actual=appResults,
3082 onpass="Primitives app activated",
3083 onfail="Primitives app not activated" )
3084 time.sleep( 5 ) # To allow all nodes to activate
3085
3086 def CASE17( self, main ):
3087 """
3088 Check for basic functionality with distributed primitives
3089 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003090 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003091 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003092 assert main, "main not defined"
3093 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003094 assert main.CLIs, "main.CLIs not defined"
3095 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003096 assert pCounterName, "pCounterName not defined"
3097 assert iCounterName, "iCounterName not defined"
3098 assert onosSetName, "onosSetName not defined"
3099 # NOTE: assert fails if value is 0/None/Empty/False
3100 try:
3101 pCounterValue
3102 except NameError:
3103 main.log.error( "pCounterValue not defined, setting to 0" )
3104 pCounterValue = 0
3105 try:
3106 iCounterValue
3107 except NameError:
3108 main.log.error( "iCounterValue not defined, setting to 0" )
3109 iCounterValue = 0
3110 try:
3111 onosSet
3112 except NameError:
3113 main.log.error( "onosSet not defined, setting to empty Set" )
3114 onosSet = set([])
3115 # Variables for the distributed primitives tests. These are local only
3116 addValue = "a"
3117 addAllValue = "a b c d e f"
3118 retainValue = "c d e f"
3119
3120 description = "Check for basic functionality with distributed " +\
3121 "primitives"
3122 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003123 main.caseExplanation = "Test the methods of the distributed " +\
3124 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003125 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003126 # Partitioned counters
3127 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003128 pCounters = []
3129 threads = []
3130 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003131 for i in range( main.numCtrls ):
3132 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3133 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003134 args=[ pCounterName ] )
3135 pCounterValue += 1
3136 addedPValues.append( pCounterValue )
3137 threads.append( t )
3138 t.start()
3139
3140 for t in threads:
3141 t.join()
3142 pCounters.append( t.result )
3143 # Check that counter incremented numController times
3144 pCounterResults = True
3145 for i in addedPValues:
3146 tmpResult = i in pCounters
3147 pCounterResults = pCounterResults and tmpResult
3148 if not tmpResult:
3149 main.log.error( str( i ) + " is not in partitioned "
3150 "counter incremented results" )
3151 utilities.assert_equals( expect=True,
3152 actual=pCounterResults,
3153 onpass="Default counter incremented",
3154 onfail="Error incrementing default" +
3155 " counter" )
3156
Jon Halle1a3b752015-07-22 13:02:46 -07003157 main.step( "Get then Increment a default counter on each node" )
3158 pCounters = []
3159 threads = []
3160 addedPValues = []
3161 for i in range( main.numCtrls ):
3162 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3163 name="counterGetAndAdd-" + str( i ),
3164 args=[ pCounterName ] )
3165 addedPValues.append( pCounterValue )
3166 pCounterValue += 1
3167 threads.append( t )
3168 t.start()
3169
3170 for t in threads:
3171 t.join()
3172 pCounters.append( t.result )
3173 # Check that counter incremented numController times
3174 pCounterResults = True
3175 for i in addedPValues:
3176 tmpResult = i in pCounters
3177 pCounterResults = pCounterResults and tmpResult
3178 if not tmpResult:
3179 main.log.error( str( i ) + " is not in partitioned "
3180 "counter incremented results" )
3181 utilities.assert_equals( expect=True,
3182 actual=pCounterResults,
3183 onpass="Default counter incremented",
3184 onfail="Error incrementing default" +
3185 " counter" )
3186
3187 main.step( "Counters we added have the correct values" )
3188 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3189 utilities.assert_equals( expect=main.TRUE,
3190 actual=incrementCheck,
3191 onpass="Added counters are correct",
3192 onfail="Added counters are incorrect" )
3193
3194 main.step( "Add -8 to then get a default counter on each node" )
3195 pCounters = []
3196 threads = []
3197 addedPValues = []
3198 for i in range( main.numCtrls ):
3199 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3200 name="counterIncrement-" + str( i ),
3201 args=[ pCounterName ],
3202 kwargs={ "delta": -8 } )
3203 pCounterValue += -8
3204 addedPValues.append( pCounterValue )
3205 threads.append( t )
3206 t.start()
3207
3208 for t in threads:
3209 t.join()
3210 pCounters.append( t.result )
3211 # Check that counter incremented numController times
3212 pCounterResults = True
3213 for i in addedPValues:
3214 tmpResult = i in pCounters
3215 pCounterResults = pCounterResults and tmpResult
3216 if not tmpResult:
3217 main.log.error( str( i ) + " is not in partitioned "
3218 "counter incremented results" )
3219 utilities.assert_equals( expect=True,
3220 actual=pCounterResults,
3221 onpass="Default counter incremented",
3222 onfail="Error incrementing default" +
3223 " counter" )
3224
3225 main.step( "Add 5 to then get a default counter on each node" )
3226 pCounters = []
3227 threads = []
3228 addedPValues = []
3229 for i in range( main.numCtrls ):
3230 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3231 name="counterIncrement-" + str( i ),
3232 args=[ pCounterName ],
3233 kwargs={ "delta": 5 } )
3234 pCounterValue += 5
3235 addedPValues.append( pCounterValue )
3236 threads.append( t )
3237 t.start()
3238
3239 for t in threads:
3240 t.join()
3241 pCounters.append( t.result )
3242 # Check that counter incremented numController times
3243 pCounterResults = True
3244 for i in addedPValues:
3245 tmpResult = i in pCounters
3246 pCounterResults = pCounterResults and tmpResult
3247 if not tmpResult:
3248 main.log.error( str( i ) + " is not in partitioned "
3249 "counter incremented results" )
3250 utilities.assert_equals( expect=True,
3251 actual=pCounterResults,
3252 onpass="Default counter incremented",
3253 onfail="Error incrementing default" +
3254 " counter" )
3255
3256 main.step( "Get then add 5 to a default counter on each node" )
3257 pCounters = []
3258 threads = []
3259 addedPValues = []
3260 for i in range( main.numCtrls ):
3261 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3262 name="counterIncrement-" + str( i ),
3263 args=[ pCounterName ],
3264 kwargs={ "delta": 5 } )
3265 addedPValues.append( pCounterValue )
3266 pCounterValue += 5
3267 threads.append( t )
3268 t.start()
3269
3270 for t in threads:
3271 t.join()
3272 pCounters.append( t.result )
3273 # Check that counter incremented numController times
3274 pCounterResults = True
3275 for i in addedPValues:
3276 tmpResult = i in pCounters
3277 pCounterResults = pCounterResults and tmpResult
3278 if not tmpResult:
3279 main.log.error( str( i ) + " is not in partitioned "
3280 "counter incremented results" )
3281 utilities.assert_equals( expect=True,
3282 actual=pCounterResults,
3283 onpass="Default counter incremented",
3284 onfail="Error incrementing default" +
3285 " counter" )
3286
3287 main.step( "Counters we added have the correct values" )
3288 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3289 utilities.assert_equals( expect=main.TRUE,
3290 actual=incrementCheck,
3291 onpass="Added counters are correct",
3292 onfail="Added counters are incorrect" )
3293
3294 # In-Memory counters
3295 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003296 iCounters = []
3297 addedIValues = []
3298 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003299 for i in range( main.numCtrls ):
3300 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003301 name="icounterIncrement-" + str( i ),
3302 args=[ iCounterName ],
3303 kwargs={ "inMemory": True } )
3304 iCounterValue += 1
3305 addedIValues.append( iCounterValue )
3306 threads.append( t )
3307 t.start()
3308
3309 for t in threads:
3310 t.join()
3311 iCounters.append( t.result )
3312 # Check that counter incremented numController times
3313 iCounterResults = True
3314 for i in addedIValues:
3315 tmpResult = i in iCounters
3316 iCounterResults = iCounterResults and tmpResult
3317 if not tmpResult:
3318 main.log.error( str( i ) + " is not in the in-memory "
3319 "counter incremented results" )
3320 utilities.assert_equals( expect=True,
3321 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003322 onpass="In-memory counter incremented",
3323 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003324 " counter" )
3325
Jon Halle1a3b752015-07-22 13:02:46 -07003326 main.step( "Get then Increment a in-memory counter on each node" )
3327 iCounters = []
3328 threads = []
3329 addedIValues = []
3330 for i in range( main.numCtrls ):
3331 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3332 name="counterGetAndAdd-" + str( i ),
3333 args=[ iCounterName ],
3334 kwargs={ "inMemory": True } )
3335 addedIValues.append( iCounterValue )
3336 iCounterValue += 1
3337 threads.append( t )
3338 t.start()
3339
3340 for t in threads:
3341 t.join()
3342 iCounters.append( t.result )
3343 # Check that counter incremented numController times
3344 iCounterResults = True
3345 for i in addedIValues:
3346 tmpResult = i in iCounters
3347 iCounterResults = iCounterResults and tmpResult
3348 if not tmpResult:
3349 main.log.error( str( i ) + " is not in in-memory "
3350 "counter incremented results" )
3351 utilities.assert_equals( expect=True,
3352 actual=iCounterResults,
3353 onpass="In-memory counter incremented",
3354 onfail="Error incrementing in-memory" +
3355 " counter" )
3356
3357 main.step( "Counters we added have the correct values" )
3358 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3359 utilities.assert_equals( expect=main.TRUE,
3360 actual=incrementCheck,
3361 onpass="Added counters are correct",
3362 onfail="Added counters are incorrect" )
3363
3364 main.step( "Add -8 to then get a in-memory counter on each node" )
3365 iCounters = []
3366 threads = []
3367 addedIValues = []
3368 for i in range( main.numCtrls ):
3369 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3370 name="counterIncrement-" + str( i ),
3371 args=[ iCounterName ],
3372 kwargs={ "delta": -8, "inMemory": True } )
3373 iCounterValue += -8
3374 addedIValues.append( iCounterValue )
3375 threads.append( t )
3376 t.start()
3377
3378 for t in threads:
3379 t.join()
3380 iCounters.append( t.result )
3381 # Check that counter incremented numController times
3382 iCounterResults = True
3383 for i in addedIValues:
3384 tmpResult = i in iCounters
3385 iCounterResults = iCounterResults and tmpResult
3386 if not tmpResult:
3387 main.log.error( str( i ) + " is not in in-memory "
3388 "counter incremented results" )
3389 utilities.assert_equals( expect=True,
3390 actual=pCounterResults,
3391 onpass="In-memory counter incremented",
3392 onfail="Error incrementing in-memory" +
3393 " counter" )
3394
3395 main.step( "Add 5 to then get a in-memory counter on each node" )
3396 iCounters = []
3397 threads = []
3398 addedIValues = []
3399 for i in range( main.numCtrls ):
3400 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3401 name="counterIncrement-" + str( i ),
3402 args=[ iCounterName ],
3403 kwargs={ "delta": 5, "inMemory": True } )
3404 iCounterValue += 5
3405 addedIValues.append( iCounterValue )
3406 threads.append( t )
3407 t.start()
3408
3409 for t in threads:
3410 t.join()
3411 iCounters.append( t.result )
3412 # Check that counter incremented numController times
3413 iCounterResults = True
3414 for i in addedIValues:
3415 tmpResult = i in iCounters
3416 iCounterResults = iCounterResults and tmpResult
3417 if not tmpResult:
3418 main.log.error( str( i ) + " is not in in-memory "
3419 "counter incremented results" )
3420 utilities.assert_equals( expect=True,
3421 actual=pCounterResults,
3422 onpass="In-memory counter incremented",
3423 onfail="Error incrementing in-memory" +
3424 " counter" )
3425
3426 main.step( "Get then add 5 to a in-memory counter on each node" )
3427 iCounters = []
3428 threads = []
3429 addedIValues = []
3430 for i in range( main.numCtrls ):
3431 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3432 name="counterIncrement-" + str( i ),
3433 args=[ iCounterName ],
3434 kwargs={ "delta": 5, "inMemory": True } )
3435 addedIValues.append( iCounterValue )
3436 iCounterValue += 5
3437 threads.append( t )
3438 t.start()
3439
3440 for t in threads:
3441 t.join()
3442 iCounters.append( t.result )
3443 # Check that counter incremented numController times
3444 iCounterResults = True
3445 for i in addedIValues:
3446 tmpResult = i in iCounters
3447 iCounterResults = iCounterResults and tmpResult
3448 if not tmpResult:
3449 main.log.error( str( i ) + " is not in in-memory "
3450 "counter incremented results" )
3451 utilities.assert_equals( expect=True,
3452 actual=iCounterResults,
3453 onpass="In-memory counter incremented",
3454 onfail="Error incrementing in-memory" +
3455 " counter" )
3456
3457 main.step( "Counters we added have the correct values" )
3458 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3459 utilities.assert_equals( expect=main.TRUE,
3460 actual=incrementCheck,
3461 onpass="Added counters are correct",
3462 onfail="Added counters are incorrect" )
3463
Jon Hall5cf14d52015-07-16 12:15:19 -07003464 main.step( "Check counters are consistant across nodes" )
3465 onosCounters = []
3466 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003467 for i in range( main.numCtrls ):
3468 t = main.Thread( target=main.CLIs[i].counters,
Jon Hall5cf14d52015-07-16 12:15:19 -07003469 name="counters-" + str( i ) )
3470 threads.append( t )
3471 t.start()
3472 for t in threads:
3473 t.join()
3474 onosCounters.append( t.result )
3475 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3476 if all( tmp ):
3477 main.log.info( "Counters are consistent across all nodes" )
3478 consistentCounterResults = main.TRUE
3479 else:
3480 main.log.error( "Counters are not consistent across all nodes" )
3481 consistentCounterResults = main.FALSE
3482 utilities.assert_equals( expect=main.TRUE,
3483 actual=consistentCounterResults,
3484 onpass="ONOS counters are consistent " +
3485 "across nodes",
3486 onfail="ONOS Counters are inconsistent " +
3487 "across nodes" )
3488
3489 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003490 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3491 incrementCheck = incrementCheck and \
3492 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003493 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003494 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003495 onpass="Added counters are correct",
3496 onfail="Added counters are incorrect" )
Jon Halle1a3b752015-07-22 13:02:46 -07003497
Jon Hall5cf14d52015-07-16 12:15:19 -07003498 # DISTRIBUTED SETS
3499 main.step( "Distributed Set get" )
3500 size = len( onosSet )
3501 getResponses = []
3502 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003503 for i in range( main.numCtrls ):
3504 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003505 name="setTestGet-" + str( i ),
3506 args=[ onosSetName ] )
3507 threads.append( t )
3508 t.start()
3509 for t in threads:
3510 t.join()
3511 getResponses.append( t.result )
3512
3513 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003514 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003515 if isinstance( getResponses[ i ], list):
3516 current = set( getResponses[ i ] )
3517 if len( current ) == len( getResponses[ i ] ):
3518 # no repeats
3519 if onosSet != current:
3520 main.log.error( "ONOS" + str( i + 1 ) +
3521 " has incorrect view" +
3522 " of set " + onosSetName + ":\n" +
3523 str( getResponses[ i ] ) )
3524 main.log.debug( "Expected: " + str( onosSet ) )
3525 main.log.debug( "Actual: " + str( current ) )
3526 getResults = main.FALSE
3527 else:
3528 # error, set is not a set
3529 main.log.error( "ONOS" + str( i + 1 ) +
3530 " has repeat elements in" +
3531 " set " + onosSetName + ":\n" +
3532 str( getResponses[ i ] ) )
3533 getResults = main.FALSE
3534 elif getResponses[ i ] == main.ERROR:
3535 getResults = main.FALSE
3536 utilities.assert_equals( expect=main.TRUE,
3537 actual=getResults,
3538 onpass="Set elements are correct",
3539 onfail="Set elements are incorrect" )
3540
3541 main.step( "Distributed Set size" )
3542 sizeResponses = []
3543 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003544 for i in range( main.numCtrls ):
3545 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003546 name="setTestSize-" + str( i ),
3547 args=[ onosSetName ] )
3548 threads.append( t )
3549 t.start()
3550 for t in threads:
3551 t.join()
3552 sizeResponses.append( t.result )
3553
3554 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003555 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003556 if size != sizeResponses[ i ]:
3557 sizeResults = main.FALSE
3558 main.log.error( "ONOS" + str( i + 1 ) +
3559 " expected a size of " + str( size ) +
3560 " for set " + onosSetName +
3561 " but got " + str( sizeResponses[ i ] ) )
3562 utilities.assert_equals( expect=main.TRUE,
3563 actual=sizeResults,
3564 onpass="Set sizes are correct",
3565 onfail="Set sizes are incorrect" )
3566
3567 main.step( "Distributed Set add()" )
3568 onosSet.add( addValue )
3569 addResponses = []
3570 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003571 for i in range( main.numCtrls ):
3572 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003573 name="setTestAdd-" + str( i ),
3574 args=[ onosSetName, addValue ] )
3575 threads.append( t )
3576 t.start()
3577 for t in threads:
3578 t.join()
3579 addResponses.append( t.result )
3580
3581 # main.TRUE = successfully changed the set
3582 # main.FALSE = action resulted in no change in set
3583 # main.ERROR - Some error in executing the function
3584 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003585 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003586 if addResponses[ i ] == main.TRUE:
3587 # All is well
3588 pass
3589 elif addResponses[ i ] == main.FALSE:
3590 # Already in set, probably fine
3591 pass
3592 elif addResponses[ i ] == main.ERROR:
3593 # Error in execution
3594 addResults = main.FALSE
3595 else:
3596 # unexpected result
3597 addResults = main.FALSE
3598 if addResults != main.TRUE:
3599 main.log.error( "Error executing set add" )
3600
3601 # Check if set is still correct
3602 size = len( onosSet )
3603 getResponses = []
3604 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003605 for i in range( main.numCtrls ):
3606 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003607 name="setTestGet-" + str( i ),
3608 args=[ onosSetName ] )
3609 threads.append( t )
3610 t.start()
3611 for t in threads:
3612 t.join()
3613 getResponses.append( t.result )
3614 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003615 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003616 if isinstance( getResponses[ i ], list):
3617 current = set( getResponses[ i ] )
3618 if len( current ) == len( getResponses[ i ] ):
3619 # no repeats
3620 if onosSet != current:
3621 main.log.error( "ONOS" + str( i + 1 ) +
3622 " has incorrect view" +
3623 " of set " + onosSetName + ":\n" +
3624 str( getResponses[ i ] ) )
3625 main.log.debug( "Expected: " + str( onosSet ) )
3626 main.log.debug( "Actual: " + str( current ) )
3627 getResults = main.FALSE
3628 else:
3629 # error, set is not a set
3630 main.log.error( "ONOS" + str( i + 1 ) +
3631 " has repeat elements in" +
3632 " set " + onosSetName + ":\n" +
3633 str( getResponses[ i ] ) )
3634 getResults = main.FALSE
3635 elif getResponses[ i ] == main.ERROR:
3636 getResults = main.FALSE
3637 sizeResponses = []
3638 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003639 for i in range( main.numCtrls ):
3640 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003641 name="setTestSize-" + str( i ),
3642 args=[ onosSetName ] )
3643 threads.append( t )
3644 t.start()
3645 for t in threads:
3646 t.join()
3647 sizeResponses.append( t.result )
3648 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003649 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003650 if size != sizeResponses[ i ]:
3651 sizeResults = main.FALSE
3652 main.log.error( "ONOS" + str( i + 1 ) +
3653 " expected a size of " + str( size ) +
3654 " for set " + onosSetName +
3655 " but got " + str( sizeResponses[ i ] ) )
3656 addResults = addResults and getResults and sizeResults
3657 utilities.assert_equals( expect=main.TRUE,
3658 actual=addResults,
3659 onpass="Set add correct",
3660 onfail="Set add was incorrect" )
3661
3662 main.step( "Distributed Set addAll()" )
3663 onosSet.update( addAllValue.split() )
3664 addResponses = []
3665 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003666 for i in range( main.numCtrls ):
3667 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003668 name="setTestAddAll-" + str( i ),
3669 args=[ onosSetName, addAllValue ] )
3670 threads.append( t )
3671 t.start()
3672 for t in threads:
3673 t.join()
3674 addResponses.append( t.result )
3675
3676 # main.TRUE = successfully changed the set
3677 # main.FALSE = action resulted in no change in set
3678 # main.ERROR - Some error in executing the function
3679 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003680 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003681 if addResponses[ i ] == main.TRUE:
3682 # All is well
3683 pass
3684 elif addResponses[ i ] == main.FALSE:
3685 # Already in set, probably fine
3686 pass
3687 elif addResponses[ i ] == main.ERROR:
3688 # Error in execution
3689 addAllResults = main.FALSE
3690 else:
3691 # unexpected result
3692 addAllResults = main.FALSE
3693 if addAllResults != main.TRUE:
3694 main.log.error( "Error executing set addAll" )
3695
3696 # Check if set is still correct
3697 size = len( onosSet )
3698 getResponses = []
3699 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003700 for i in range( main.numCtrls ):
3701 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003702 name="setTestGet-" + str( i ),
3703 args=[ onosSetName ] )
3704 threads.append( t )
3705 t.start()
3706 for t in threads:
3707 t.join()
3708 getResponses.append( t.result )
3709 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003710 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003711 if isinstance( getResponses[ i ], list):
3712 current = set( getResponses[ i ] )
3713 if len( current ) == len( getResponses[ i ] ):
3714 # no repeats
3715 if onosSet != current:
3716 main.log.error( "ONOS" + str( i + 1 ) +
3717 " has incorrect view" +
3718 " of set " + onosSetName + ":\n" +
3719 str( getResponses[ i ] ) )
3720 main.log.debug( "Expected: " + str( onosSet ) )
3721 main.log.debug( "Actual: " + str( current ) )
3722 getResults = main.FALSE
3723 else:
3724 # error, set is not a set
3725 main.log.error( "ONOS" + str( i + 1 ) +
3726 " has repeat elements in" +
3727 " set " + onosSetName + ":\n" +
3728 str( getResponses[ i ] ) )
3729 getResults = main.FALSE
3730 elif getResponses[ i ] == main.ERROR:
3731 getResults = main.FALSE
3732 sizeResponses = []
3733 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003734 for i in range( main.numCtrls ):
3735 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003736 name="setTestSize-" + str( i ),
3737 args=[ onosSetName ] )
3738 threads.append( t )
3739 t.start()
3740 for t in threads:
3741 t.join()
3742 sizeResponses.append( t.result )
3743 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003744 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003745 if size != sizeResponses[ i ]:
3746 sizeResults = main.FALSE
3747 main.log.error( "ONOS" + str( i + 1 ) +
3748 " expected a size of " + str( size ) +
3749 " for set " + onosSetName +
3750 " but got " + str( sizeResponses[ i ] ) )
3751 addAllResults = addAllResults and getResults and sizeResults
3752 utilities.assert_equals( expect=main.TRUE,
3753 actual=addAllResults,
3754 onpass="Set addAll correct",
3755 onfail="Set addAll was incorrect" )
3756
3757 main.step( "Distributed Set contains()" )
3758 containsResponses = []
3759 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003760 for i in range( main.numCtrls ):
3761 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003762 name="setContains-" + str( i ),
3763 args=[ onosSetName ],
3764 kwargs={ "values": addValue } )
3765 threads.append( t )
3766 t.start()
3767 for t in threads:
3768 t.join()
3769 # NOTE: This is the tuple
3770 containsResponses.append( t.result )
3771
3772 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003773 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003774 if containsResponses[ i ] == main.ERROR:
3775 containsResults = main.FALSE
3776 else:
3777 containsResults = containsResults and\
3778 containsResponses[ i ][ 1 ]
3779 utilities.assert_equals( expect=main.TRUE,
3780 actual=containsResults,
3781 onpass="Set contains is functional",
3782 onfail="Set contains failed" )
3783
3784 main.step( "Distributed Set containsAll()" )
3785 containsAllResponses = []
3786 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003787 for i in range( main.numCtrls ):
3788 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003789 name="setContainsAll-" + str( i ),
3790 args=[ onosSetName ],
3791 kwargs={ "values": addAllValue } )
3792 threads.append( t )
3793 t.start()
3794 for t in threads:
3795 t.join()
3796 # NOTE: This is the tuple
3797 containsAllResponses.append( t.result )
3798
3799 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003800 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003801 if containsResponses[ i ] == main.ERROR:
3802 containsResults = main.FALSE
3803 else:
3804 containsResults = containsResults and\
3805 containsResponses[ i ][ 1 ]
3806 utilities.assert_equals( expect=main.TRUE,
3807 actual=containsAllResults,
3808 onpass="Set containsAll is functional",
3809 onfail="Set containsAll failed" )
3810
3811 main.step( "Distributed Set remove()" )
3812 onosSet.remove( addValue )
3813 removeResponses = []
3814 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003815 for i in range( main.numCtrls ):
3816 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003817 name="setTestRemove-" + str( i ),
3818 args=[ onosSetName, addValue ] )
3819 threads.append( t )
3820 t.start()
3821 for t in threads:
3822 t.join()
3823 removeResponses.append( t.result )
3824
3825 # main.TRUE = successfully changed the set
3826 # main.FALSE = action resulted in no change in set
3827 # main.ERROR - Some error in executing the function
3828 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003829 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003830 if removeResponses[ i ] == main.TRUE:
3831 # All is well
3832 pass
3833 elif removeResponses[ i ] == main.FALSE:
3834 # not in set, probably fine
3835 pass
3836 elif removeResponses[ i ] == main.ERROR:
3837 # Error in execution
3838 removeResults = main.FALSE
3839 else:
3840 # unexpected result
3841 removeResults = main.FALSE
3842 if removeResults != main.TRUE:
3843 main.log.error( "Error executing set remove" )
3844
3845 # Check if set is still correct
3846 size = len( onosSet )
3847 getResponses = []
3848 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003849 for i in range( main.numCtrls ):
3850 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003851 name="setTestGet-" + str( i ),
3852 args=[ onosSetName ] )
3853 threads.append( t )
3854 t.start()
3855 for t in threads:
3856 t.join()
3857 getResponses.append( t.result )
3858 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003859 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003860 if isinstance( getResponses[ i ], list):
3861 current = set( getResponses[ i ] )
3862 if len( current ) == len( getResponses[ i ] ):
3863 # no repeats
3864 if onosSet != current:
3865 main.log.error( "ONOS" + str( i + 1 ) +
3866 " has incorrect view" +
3867 " of set " + onosSetName + ":\n" +
3868 str( getResponses[ i ] ) )
3869 main.log.debug( "Expected: " + str( onosSet ) )
3870 main.log.debug( "Actual: " + str( current ) )
3871 getResults = main.FALSE
3872 else:
3873 # error, set is not a set
3874 main.log.error( "ONOS" + str( i + 1 ) +
3875 " has repeat elements in" +
3876 " set " + onosSetName + ":\n" +
3877 str( getResponses[ i ] ) )
3878 getResults = main.FALSE
3879 elif getResponses[ i ] == main.ERROR:
3880 getResults = main.FALSE
3881 sizeResponses = []
3882 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003883 for i in range( main.numCtrls ):
3884 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003885 name="setTestSize-" + str( i ),
3886 args=[ onosSetName ] )
3887 threads.append( t )
3888 t.start()
3889 for t in threads:
3890 t.join()
3891 sizeResponses.append( t.result )
3892 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003893 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003894 if size != sizeResponses[ i ]:
3895 sizeResults = main.FALSE
3896 main.log.error( "ONOS" + str( i + 1 ) +
3897 " expected a size of " + str( size ) +
3898 " for set " + onosSetName +
3899 " but got " + str( sizeResponses[ i ] ) )
3900 removeResults = removeResults and getResults and sizeResults
3901 utilities.assert_equals( expect=main.TRUE,
3902 actual=removeResults,
3903 onpass="Set remove correct",
3904 onfail="Set remove was incorrect" )
3905
3906 main.step( "Distributed Set removeAll()" )
3907 onosSet.difference_update( addAllValue.split() )
3908 removeAllResponses = []
3909 threads = []
3910 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003911 for i in range( main.numCtrls ):
3912 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003913 name="setTestRemoveAll-" + str( i ),
3914 args=[ onosSetName, addAllValue ] )
3915 threads.append( t )
3916 t.start()
3917 for t in threads:
3918 t.join()
3919 removeAllResponses.append( t.result )
3920 except Exception, e:
3921 main.log.exception(e)
3922
3923 # main.TRUE = successfully changed the set
3924 # main.FALSE = action resulted in no change in set
3925 # main.ERROR - Some error in executing the function
3926 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003927 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003928 if removeAllResponses[ i ] == main.TRUE:
3929 # All is well
3930 pass
3931 elif removeAllResponses[ i ] == main.FALSE:
3932 # not in set, probably fine
3933 pass
3934 elif removeAllResponses[ i ] == main.ERROR:
3935 # Error in execution
3936 removeAllResults = main.FALSE
3937 else:
3938 # unexpected result
3939 removeAllResults = main.FALSE
3940 if removeAllResults != main.TRUE:
3941 main.log.error( "Error executing set removeAll" )
3942
3943 # Check if set is still correct
3944 size = len( onosSet )
3945 getResponses = []
3946 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003947 for i in range( main.numCtrls ):
3948 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003949 name="setTestGet-" + str( i ),
3950 args=[ onosSetName ] )
3951 threads.append( t )
3952 t.start()
3953 for t in threads:
3954 t.join()
3955 getResponses.append( t.result )
3956 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003957 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003958 if isinstance( getResponses[ i ], list):
3959 current = set( getResponses[ i ] )
3960 if len( current ) == len( getResponses[ i ] ):
3961 # no repeats
3962 if onosSet != current:
3963 main.log.error( "ONOS" + str( i + 1 ) +
3964 " has incorrect view" +
3965 " of set " + onosSetName + ":\n" +
3966 str( getResponses[ i ] ) )
3967 main.log.debug( "Expected: " + str( onosSet ) )
3968 main.log.debug( "Actual: " + str( current ) )
3969 getResults = main.FALSE
3970 else:
3971 # error, set is not a set
3972 main.log.error( "ONOS" + str( i + 1 ) +
3973 " has repeat elements in" +
3974 " set " + onosSetName + ":\n" +
3975 str( getResponses[ i ] ) )
3976 getResults = main.FALSE
3977 elif getResponses[ i ] == main.ERROR:
3978 getResults = main.FALSE
3979 sizeResponses = []
3980 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003981 for i in range( main.numCtrls ):
3982 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003983 name="setTestSize-" + str( i ),
3984 args=[ onosSetName ] )
3985 threads.append( t )
3986 t.start()
3987 for t in threads:
3988 t.join()
3989 sizeResponses.append( t.result )
3990 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003991 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003992 if size != sizeResponses[ i ]:
3993 sizeResults = main.FALSE
3994 main.log.error( "ONOS" + str( i + 1 ) +
3995 " expected a size of " + str( size ) +
3996 " for set " + onosSetName +
3997 " but got " + str( sizeResponses[ i ] ) )
3998 removeAllResults = removeAllResults and getResults and sizeResults
3999 utilities.assert_equals( expect=main.TRUE,
4000 actual=removeAllResults,
4001 onpass="Set removeAll correct",
4002 onfail="Set removeAll was incorrect" )
4003
4004 main.step( "Distributed Set addAll()" )
4005 onosSet.update( addAllValue.split() )
4006 addResponses = []
4007 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004008 for i in range( main.numCtrls ):
4009 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004010 name="setTestAddAll-" + str( i ),
4011 args=[ onosSetName, addAllValue ] )
4012 threads.append( t )
4013 t.start()
4014 for t in threads:
4015 t.join()
4016 addResponses.append( t.result )
4017
4018 # main.TRUE = successfully changed the set
4019 # main.FALSE = action resulted in no change in set
4020 # main.ERROR - Some error in executing the function
4021 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004022 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004023 if addResponses[ i ] == main.TRUE:
4024 # All is well
4025 pass
4026 elif addResponses[ i ] == main.FALSE:
4027 # Already in set, probably fine
4028 pass
4029 elif addResponses[ i ] == main.ERROR:
4030 # Error in execution
4031 addAllResults = main.FALSE
4032 else:
4033 # unexpected result
4034 addAllResults = main.FALSE
4035 if addAllResults != main.TRUE:
4036 main.log.error( "Error executing set addAll" )
4037
4038 # Check if set is still correct
4039 size = len( onosSet )
4040 getResponses = []
4041 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004042 for i in range( main.numCtrls ):
4043 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004044 name="setTestGet-" + str( i ),
4045 args=[ onosSetName ] )
4046 threads.append( t )
4047 t.start()
4048 for t in threads:
4049 t.join()
4050 getResponses.append( t.result )
4051 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004052 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004053 if isinstance( getResponses[ i ], list):
4054 current = set( getResponses[ i ] )
4055 if len( current ) == len( getResponses[ i ] ):
4056 # no repeats
4057 if onosSet != current:
4058 main.log.error( "ONOS" + str( i + 1 ) +
4059 " has incorrect view" +
4060 " of set " + onosSetName + ":\n" +
4061 str( getResponses[ i ] ) )
4062 main.log.debug( "Expected: " + str( onosSet ) )
4063 main.log.debug( "Actual: " + str( current ) )
4064 getResults = main.FALSE
4065 else:
4066 # error, set is not a set
4067 main.log.error( "ONOS" + str( i + 1 ) +
4068 " has repeat elements in" +
4069 " set " + onosSetName + ":\n" +
4070 str( getResponses[ i ] ) )
4071 getResults = main.FALSE
4072 elif getResponses[ i ] == main.ERROR:
4073 getResults = main.FALSE
4074 sizeResponses = []
4075 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004076 for i in range( main.numCtrls ):
4077 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004078 name="setTestSize-" + str( i ),
4079 args=[ onosSetName ] )
4080 threads.append( t )
4081 t.start()
4082 for t in threads:
4083 t.join()
4084 sizeResponses.append( t.result )
4085 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004086 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004087 if size != sizeResponses[ i ]:
4088 sizeResults = main.FALSE
4089 main.log.error( "ONOS" + str( i + 1 ) +
4090 " expected a size of " + str( size ) +
4091 " for set " + onosSetName +
4092 " but got " + str( sizeResponses[ i ] ) )
4093 addAllResults = addAllResults and getResults and sizeResults
4094 utilities.assert_equals( expect=main.TRUE,
4095 actual=addAllResults,
4096 onpass="Set addAll correct",
4097 onfail="Set addAll was incorrect" )
4098
4099 main.step( "Distributed Set clear()" )
4100 onosSet.clear()
4101 clearResponses = []
4102 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004103 for i in range( main.numCtrls ):
4104 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004105 name="setTestClear-" + str( i ),
4106 args=[ onosSetName, " "], # Values doesn't matter
4107 kwargs={ "clear": True } )
4108 threads.append( t )
4109 t.start()
4110 for t in threads:
4111 t.join()
4112 clearResponses.append( t.result )
4113
4114 # main.TRUE = successfully changed the set
4115 # main.FALSE = action resulted in no change in set
4116 # main.ERROR - Some error in executing the function
4117 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004118 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004119 if clearResponses[ i ] == main.TRUE:
4120 # All is well
4121 pass
4122 elif clearResponses[ i ] == main.FALSE:
4123 # Nothing set, probably fine
4124 pass
4125 elif clearResponses[ i ] == main.ERROR:
4126 # Error in execution
4127 clearResults = main.FALSE
4128 else:
4129 # unexpected result
4130 clearResults = main.FALSE
4131 if clearResults != main.TRUE:
4132 main.log.error( "Error executing set clear" )
4133
4134 # Check if set is still correct
4135 size = len( onosSet )
4136 getResponses = []
4137 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004138 for i in range( main.numCtrls ):
4139 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004140 name="setTestGet-" + str( i ),
4141 args=[ onosSetName ] )
4142 threads.append( t )
4143 t.start()
4144 for t in threads:
4145 t.join()
4146 getResponses.append( t.result )
4147 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004148 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004149 if isinstance( getResponses[ i ], list):
4150 current = set( getResponses[ i ] )
4151 if len( current ) == len( getResponses[ i ] ):
4152 # no repeats
4153 if onosSet != current:
4154 main.log.error( "ONOS" + str( i + 1 ) +
4155 " has incorrect view" +
4156 " of set " + onosSetName + ":\n" +
4157 str( getResponses[ i ] ) )
4158 main.log.debug( "Expected: " + str( onosSet ) )
4159 main.log.debug( "Actual: " + str( current ) )
4160 getResults = main.FALSE
4161 else:
4162 # error, set is not a set
4163 main.log.error( "ONOS" + str( i + 1 ) +
4164 " has repeat elements in" +
4165 " set " + onosSetName + ":\n" +
4166 str( getResponses[ i ] ) )
4167 getResults = main.FALSE
4168 elif getResponses[ i ] == main.ERROR:
4169 getResults = main.FALSE
4170 sizeResponses = []
4171 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004172 for i in range( main.numCtrls ):
4173 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004174 name="setTestSize-" + str( i ),
4175 args=[ onosSetName ] )
4176 threads.append( t )
4177 t.start()
4178 for t in threads:
4179 t.join()
4180 sizeResponses.append( t.result )
4181 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004182 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004183 if size != sizeResponses[ i ]:
4184 sizeResults = main.FALSE
4185 main.log.error( "ONOS" + str( i + 1 ) +
4186 " expected a size of " + str( size ) +
4187 " for set " + onosSetName +
4188 " but got " + str( sizeResponses[ i ] ) )
4189 clearResults = clearResults and getResults and sizeResults
4190 utilities.assert_equals( expect=main.TRUE,
4191 actual=clearResults,
4192 onpass="Set clear correct",
4193 onfail="Set clear was incorrect" )
4194
4195 main.step( "Distributed Set addAll()" )
4196 onosSet.update( addAllValue.split() )
4197 addResponses = []
4198 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004199 for i in range( main.numCtrls ):
4200 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004201 name="setTestAddAll-" + str( i ),
4202 args=[ onosSetName, addAllValue ] )
4203 threads.append( t )
4204 t.start()
4205 for t in threads:
4206 t.join()
4207 addResponses.append( t.result )
4208
4209 # main.TRUE = successfully changed the set
4210 # main.FALSE = action resulted in no change in set
4211 # main.ERROR - Some error in executing the function
4212 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004213 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004214 if addResponses[ i ] == main.TRUE:
4215 # All is well
4216 pass
4217 elif addResponses[ i ] == main.FALSE:
4218 # Already in set, probably fine
4219 pass
4220 elif addResponses[ i ] == main.ERROR:
4221 # Error in execution
4222 addAllResults = main.FALSE
4223 else:
4224 # unexpected result
4225 addAllResults = main.FALSE
4226 if addAllResults != main.TRUE:
4227 main.log.error( "Error executing set addAll" )
4228
4229 # Check if set is still correct
4230 size = len( onosSet )
4231 getResponses = []
4232 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004233 for i in range( main.numCtrls ):
4234 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004235 name="setTestGet-" + str( i ),
4236 args=[ onosSetName ] )
4237 threads.append( t )
4238 t.start()
4239 for t in threads:
4240 t.join()
4241 getResponses.append( t.result )
4242 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004243 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004244 if isinstance( getResponses[ i ], list):
4245 current = set( getResponses[ i ] )
4246 if len( current ) == len( getResponses[ i ] ):
4247 # no repeats
4248 if onosSet != current:
4249 main.log.error( "ONOS" + str( i + 1 ) +
4250 " has incorrect view" +
4251 " of set " + onosSetName + ":\n" +
4252 str( getResponses[ i ] ) )
4253 main.log.debug( "Expected: " + str( onosSet ) )
4254 main.log.debug( "Actual: " + str( current ) )
4255 getResults = main.FALSE
4256 else:
4257 # error, set is not a set
4258 main.log.error( "ONOS" + str( i + 1 ) +
4259 " has repeat elements in" +
4260 " set " + onosSetName + ":\n" +
4261 str( getResponses[ i ] ) )
4262 getResults = main.FALSE
4263 elif getResponses[ i ] == main.ERROR:
4264 getResults = main.FALSE
4265 sizeResponses = []
4266 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004267 for i in range( main.numCtrls ):
4268 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004269 name="setTestSize-" + str( i ),
4270 args=[ onosSetName ] )
4271 threads.append( t )
4272 t.start()
4273 for t in threads:
4274 t.join()
4275 sizeResponses.append( t.result )
4276 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004277 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004278 if size != sizeResponses[ i ]:
4279 sizeResults = main.FALSE
4280 main.log.error( "ONOS" + str( i + 1 ) +
4281 " expected a size of " + str( size ) +
4282 " for set " + onosSetName +
4283 " but got " + str( sizeResponses[ i ] ) )
4284 addAllResults = addAllResults and getResults and sizeResults
4285 utilities.assert_equals( expect=main.TRUE,
4286 actual=addAllResults,
4287 onpass="Set addAll correct",
4288 onfail="Set addAll was incorrect" )
4289
4290 main.step( "Distributed Set retain()" )
4291 onosSet.intersection_update( retainValue.split() )
4292 retainResponses = []
4293 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004294 for i in range( main.numCtrls ):
4295 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004296 name="setTestRetain-" + str( i ),
4297 args=[ onosSetName, retainValue ],
4298 kwargs={ "retain": True } )
4299 threads.append( t )
4300 t.start()
4301 for t in threads:
4302 t.join()
4303 retainResponses.append( t.result )
4304
4305 # main.TRUE = successfully changed the set
4306 # main.FALSE = action resulted in no change in set
4307 # main.ERROR - Some error in executing the function
4308 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004309 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004310 if retainResponses[ i ] == main.TRUE:
4311 # All is well
4312 pass
4313 elif retainResponses[ i ] == main.FALSE:
4314 # Already in set, probably fine
4315 pass
4316 elif retainResponses[ i ] == main.ERROR:
4317 # Error in execution
4318 retainResults = main.FALSE
4319 else:
4320 # unexpected result
4321 retainResults = main.FALSE
4322 if retainResults != main.TRUE:
4323 main.log.error( "Error executing set retain" )
4324
4325 # Check if set is still correct
4326 size = len( onosSet )
4327 getResponses = []
4328 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004329 for i in range( main.numCtrls ):
4330 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004331 name="setTestGet-" + str( i ),
4332 args=[ onosSetName ] )
4333 threads.append( t )
4334 t.start()
4335 for t in threads:
4336 t.join()
4337 getResponses.append( t.result )
4338 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004339 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004340 if isinstance( getResponses[ i ], list):
4341 current = set( getResponses[ i ] )
4342 if len( current ) == len( getResponses[ i ] ):
4343 # no repeats
4344 if onosSet != current:
4345 main.log.error( "ONOS" + str( i + 1 ) +
4346 " has incorrect view" +
4347 " of set " + onosSetName + ":\n" +
4348 str( getResponses[ i ] ) )
4349 main.log.debug( "Expected: " + str( onosSet ) )
4350 main.log.debug( "Actual: " + str( current ) )
4351 getResults = main.FALSE
4352 else:
4353 # error, set is not a set
4354 main.log.error( "ONOS" + str( i + 1 ) +
4355 " has repeat elements in" +
4356 " set " + onosSetName + ":\n" +
4357 str( getResponses[ i ] ) )
4358 getResults = main.FALSE
4359 elif getResponses[ i ] == main.ERROR:
4360 getResults = main.FALSE
4361 sizeResponses = []
4362 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004363 for i in range( main.numCtrls ):
4364 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004365 name="setTestSize-" + str( i ),
4366 args=[ onosSetName ] )
4367 threads.append( t )
4368 t.start()
4369 for t in threads:
4370 t.join()
4371 sizeResponses.append( t.result )
4372 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004373 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004374 if size != sizeResponses[ i ]:
4375 sizeResults = main.FALSE
4376 main.log.error( "ONOS" + str( i + 1 ) +
4377 " expected a size of " +
4378 str( size ) + " for set " + onosSetName +
4379 " but got " + str( sizeResponses[ i ] ) )
4380 retainResults = retainResults and getResults and sizeResults
4381 utilities.assert_equals( expect=main.TRUE,
4382 actual=retainResults,
4383 onpass="Set retain correct",
4384 onfail="Set retain was incorrect" )
4385
Jon Hall2a5002c2015-08-21 16:49:11 -07004386 # Transactional maps
4387 main.step( "Partitioned Transactional maps put" )
4388 tMapValue = "Testing"
4389 numKeys = 100
4390 putResult = True
4391 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4392 if len( putResponses ) == 100:
4393 for i in putResponses:
4394 if putResponses[ i ][ 'value' ] != tMapValue:
4395 putResult = False
4396 else:
4397 putResult = False
4398 if not putResult:
4399 main.log.debug( "Put response values: " + str( putResponses ) )
4400 utilities.assert_equals( expect=True,
4401 actual=putResult,
4402 onpass="Partitioned Transactional Map put successful",
4403 onfail="Partitioned Transactional Map put values are incorrect" )
4404
4405 main.step( "Partitioned Transactional maps get" )
4406 getCheck = True
4407 for n in range( 1, numKeys + 1 ):
4408 getResponses = []
4409 threads = []
4410 valueCheck = True
4411 for i in range( main.numCtrls ):
4412 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4413 name="TMap-get-" + str( i ),
4414 args=[ "Key" + str ( n ) ] )
4415 threads.append( t )
4416 t.start()
4417 for t in threads:
4418 t.join()
4419 getResponses.append( t.result )
4420 for node in getResponses:
4421 if node != tMapValue:
4422 valueCheck = False
4423 if not valueCheck:
4424 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4425 main.log.warn( getResponses )
4426 getCheck = getCheck and valueCheck
4427 utilities.assert_equals( expect=True,
4428 actual=getCheck,
4429 onpass="Partitioned Transactional Map get values were correct",
4430 onfail="Partitioned Transactional Map values incorrect" )
4431
4432 main.step( "In-memory Transactional maps put" )
4433 tMapValue = "Testing"
4434 numKeys = 100
4435 putResult = True
4436 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4437 if len( putResponses ) == 100:
4438 for i in putResponses:
4439 if putResponses[ i ][ 'value' ] != tMapValue:
4440 putResult = False
4441 else:
4442 putResult = False
4443 if not putResult:
4444 main.log.debug( "Put response values: " + str( putResponses ) )
4445 utilities.assert_equals( expect=True,
4446 actual=putResult,
4447 onpass="In-Memory Transactional Map put successful",
4448 onfail="In-Memory Transactional Map put values are incorrect" )
4449
4450 main.step( "In-Memory Transactional maps get" )
4451 getCheck = True
4452 for n in range( 1, numKeys + 1 ):
4453 getResponses = []
4454 threads = []
4455 valueCheck = True
4456 for i in range( main.numCtrls ):
4457 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4458 name="TMap-get-" + str( i ),
4459 args=[ "Key" + str ( n ) ],
4460 kwargs={ "inMemory": True } )
4461 threads.append( t )
4462 t.start()
4463 for t in threads:
4464 t.join()
4465 getResponses.append( t.result )
4466 for node in getResponses:
4467 if node != tMapValue:
4468 valueCheck = False
4469 if not valueCheck:
4470 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4471 main.log.warn( getResponses )
4472 getCheck = getCheck and valueCheck
4473 utilities.assert_equals( expect=True,
4474 actual=getCheck,
4475 onpass="In-Memory Transactional Map get values were correct",
4476 onfail="In-Memory Transactional Map values incorrect" )