blob: d175fd77027d5a088c31e203bd07779555f3a3f2 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAminorityRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hall5cf14d52015-07-16 12:15:19 -070051 main.log.info( "ONOS HA test: Restart minority of ONOS nodes - " +
52 "initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
70 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070071 global ONOS1Port
72 global ONOS2Port
73 global ONOS3Port
74 global ONOS4Port
75 global ONOS5Port
76 global ONOS6Port
77 global ONOS7Port
78
79 # FIXME: just get controller port from params?
80 # TODO: do we really need all these?
81 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
82 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
83 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
84 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
85 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
86 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
87 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
88
Jon Halle1a3b752015-07-22 13:02:46 -070089 try:
90 fileName = "Counters"
91 # TODO: Maybe make a library folder somewhere?
92 path = main.params[ 'imports' ][ 'path' ]
93 main.Counters = imp.load_source( fileName,
94 path + fileName + ".py" )
95 except Exception as e:
96 main.log.exception( e )
97 main.cleanup()
98 main.exit()
99
100 main.CLIs = []
101 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700102 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700103 for i in range( 1, main.numCtrls + 1 ):
104 try:
105 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
106 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
107 ipList.append( main.nodes[ -1 ].ip_address )
108 except AttributeError:
109 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700110
111 main.step( "Create cell file" )
112 cellAppString = main.params[ 'ENV' ][ 'appString' ]
113 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
114 main.Mininet1.ip_address,
115 cellAppString, ipList )
116 main.step( "Applying cell variable to environment" )
117 cellResult = main.ONOSbench.setCell( cellName )
118 verifyResult = main.ONOSbench.verifyCell()
119
120 # FIXME:this is short term fix
121 main.log.info( "Removing raft logs" )
122 main.ONOSbench.onosRemoveRaftLogs()
123
124 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700125 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700126 main.ONOSbench.onosUninstall( node.ip_address )
127
128 # Make sure ONOS is DEAD
129 main.log.info( "Killing any ONOS processes" )
130 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700131 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700132 killed = main.ONOSbench.onosKill( node.ip_address )
133 killResults = killResults and killed
134
135 cleanInstallResult = main.TRUE
136 gitPullResult = main.TRUE
137
138 main.step( "Starting Mininet" )
139 # scp topo file to mininet
140 # TODO: move to params?
141 topoName = "obelisk.py"
142 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700143 main.ONOSbench.scp( main.Mininet1,
144 filePath + topoName,
145 main.Mininet1.home,
146 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700147 mnResult = main.Mininet1.startNet( )
148 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
149 onpass="Mininet Started",
150 onfail="Error starting Mininet" )
151
152 main.step( "Git checkout and pull " + gitBranch )
153 if PULLCODE:
154 main.ONOSbench.gitCheckout( gitBranch )
155 gitPullResult = main.ONOSbench.gitPull()
156 # values of 1 or 3 are good
157 utilities.assert_lesser( expect=0, actual=gitPullResult,
158 onpass="Git pull successful",
159 onfail="Git pull failed" )
160 main.ONOSbench.getVersion( report=True )
161
162 main.step( "Using mvn clean install" )
163 cleanInstallResult = main.TRUE
164 if PULLCODE and gitPullResult == main.TRUE:
165 cleanInstallResult = main.ONOSbench.cleanInstall()
166 else:
167 main.log.warn( "Did not pull new code so skipping mvn " +
168 "clean install" )
169 utilities.assert_equals( expect=main.TRUE,
170 actual=cleanInstallResult,
171 onpass="MCI successful",
172 onfail="MCI failed" )
173 # GRAPHS
174 # NOTE: important params here:
175 # job = name of Jenkins job
176 # Plot Name = Plot-HA, only can be used if multiple plots
177 # index = The number of the graph under plot name
178 job = "HAminorityRestart"
179 plotName = "Plot-HA"
180 graphs = '<ac:structured-macro ac:name="html">\n'
181 graphs += '<ac:plain-text-body><![CDATA[\n'
182 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
183 '/plot/' + plotName + '/getPlot?index=0' +\
184 '&width=500&height=300"' +\
185 'noborder="0" width="500" height="300" scrolling="yes" ' +\
186 'seamless="seamless"></iframe>\n'
187 graphs += ']]></ac:plain-text-body>\n'
188 graphs += '</ac:structured-macro>\n'
189 main.log.wiki(graphs)
190
191 main.step( "Creating ONOS package" )
192 packageResult = main.ONOSbench.onosPackage()
193 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
194 onpass="ONOS package successful",
195 onfail="ONOS package failed" )
196
197 main.step( "Installing ONOS package" )
198 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700199 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700200 tmpResult = main.ONOSbench.onosInstall( options="-f",
201 node=node.ip_address )
202 onosInstallResult = onosInstallResult and tmpResult
203 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
204 onpass="ONOS install successful",
205 onfail="ONOS install failed" )
206
207 main.step( "Checking if ONOS is up yet" )
208 for i in range( 2 ):
209 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700210 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700211 started = main.ONOSbench.isup( node.ip_address )
212 if not started:
213 main.log.error( node.name + " didn't start!" )
214 main.ONOSbench.onosStop( node.ip_address )
215 main.ONOSbench.onosStart( node.ip_address )
216 onosIsupResult = onosIsupResult and started
217 if onosIsupResult == main.TRUE:
218 break
219 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
220 onpass="ONOS startup successful",
221 onfail="ONOS startup failed" )
222
223 main.log.step( "Starting ONOS CLI sessions" )
224 cliResults = main.TRUE
225 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700226 for i in range( main.numCtrls ):
227 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700228 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700229 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 threads.append( t )
231 t.start()
232
233 for t in threads:
234 t.join()
235 cliResults = cliResults and t.result
236 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
237 onpass="ONOS cli startup successful",
238 onfail="ONOS cli startup failed" )
239
240 if main.params[ 'tcpdump' ].lower() == "true":
241 main.step( "Start Packet Capture MN" )
242 main.Mininet2.startTcpdump(
243 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
244 + "-MN.pcap",
245 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
246 port=main.params[ 'MNtcpdump' ][ 'port' ] )
247
248 main.step( "App Ids check" )
249 appCheck = main.TRUE
250 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700251 for i in range( main.numCtrls ):
252 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700253 name="appToIDCheck-" + str( i ),
254 args=[] )
255 threads.append( t )
256 t.start()
257
258 for t in threads:
259 t.join()
260 appCheck = appCheck and t.result
261 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700262 main.log.warn( main.CLIs[0].apps() )
263 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700264 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
265 onpass="App Ids seem to be correct",
266 onfail="Something is wrong with app Ids" )
267
268 if cliResults == main.FALSE:
269 main.log.error( "Failed to start ONOS, stopping test" )
270 main.cleanup()
271 main.exit()
272
273 def CASE2( self, main ):
274 """
275 Assign devices to controllers
276 """
277 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700278 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700279 assert main, "main not defined"
280 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700281 assert main.CLIs, "main.CLIs not defined"
282 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700283 assert ONOS1Port, "ONOS1Port not defined"
284 assert ONOS2Port, "ONOS2Port not defined"
285 assert ONOS3Port, "ONOS3Port not defined"
286 assert ONOS4Port, "ONOS4Port not defined"
287 assert ONOS5Port, "ONOS5Port not defined"
288 assert ONOS6Port, "ONOS6Port not defined"
289 assert ONOS7Port, "ONOS7Port not defined"
290
291 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700292 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700293 "and check that an ONOS node becomes the " +\
294 "master of the device."
295 main.step( "Assign switches to controllers" )
296
297 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700298 for i in range( main.numCtrls ):
299 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700300 swList = []
301 for i in range( 1, 29 ):
302 swList.append( "s" + str( i ) )
303 main.Mininet1.assignSwController( sw=swList, ip=ipList )
304
305 mastershipCheck = main.TRUE
306 for i in range( 1, 29 ):
307 response = main.Mininet1.getSwController( "s" + str( i ) )
308 try:
309 main.log.info( str( response ) )
310 except Exception:
311 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700312 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700313 if re.search( "tcp:" + node.ip_address, response ):
314 mastershipCheck = mastershipCheck and main.TRUE
315 else:
316 main.log.error( "Error, node " + node.ip_address + " is " +
317 "not in the list of controllers s" +
318 str( i ) + " is connecting to." )
319 mastershipCheck = main.FALSE
320 utilities.assert_equals(
321 expect=main.TRUE,
322 actual=mastershipCheck,
323 onpass="Switch mastership assigned correctly",
324 onfail="Switches not assigned correctly to controllers" )
325
326 def CASE21( self, main ):
327 """
328 Assign mastership to controllers
329 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700330 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700331 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700332 assert main, "main not defined"
333 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700334 assert main.CLIs, "main.CLIs not defined"
335 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700336 assert ONOS1Port, "ONOS1Port not defined"
337 assert ONOS2Port, "ONOS2Port not defined"
338 assert ONOS3Port, "ONOS3Port not defined"
339 assert ONOS4Port, "ONOS4Port not defined"
340 assert ONOS5Port, "ONOS5Port not defined"
341 assert ONOS6Port, "ONOS6Port not defined"
342 assert ONOS7Port, "ONOS7Port not defined"
343
344 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700345 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700346 "device. Then manually assign" +\
347 " mastership to specific ONOS nodes using" +\
348 " 'device-role'"
349 main.step( "Assign mastership of switches to specific controllers" )
350 # Manually assign mastership to the controller we want
351 roleCall = main.TRUE
352
353 ipList = [ ]
354 deviceList = []
355 try:
356 # Assign mastership to specific controllers. This assignment was
357 # determined for a 7 node cluser, but will work with any sized
358 # cluster
359 for i in range( 1, 29 ): # switches 1 through 28
360 # set up correct variables:
361 if i == 1:
362 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700363 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700364 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
365 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700366 c = 1 % main.numCtrls
367 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
369 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700370 c = 1 % main.numCtrls
371 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
373 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700374 c = 3 % main.numCtrls
375 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700376 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
377 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700378 c = 2 % main.numCtrls
379 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700380 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
381 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700382 c = 2 % main.numCtrls
383 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700384 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
385 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700386 c = 5 % main.numCtrls
387 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700388 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
389 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700390 c = 4 % main.numCtrls
391 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700392 dpid = '3' + str( i ).zfill( 3 )
393 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
394 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700395 c = 6 % main.numCtrls
396 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700397 dpid = '6' + str( i ).zfill( 3 )
398 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
399 elif i == 28:
400 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700401 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700402 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
403 else:
404 main.log.error( "You didn't write an else statement for " +
405 "switch s" + str( i ) )
406 roleCall = main.FALSE
407 # Assign switch
408 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
409 # TODO: make this controller dynamic
410 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
411 ip )
412 ipList.append( ip )
413 deviceList.append( deviceId )
414 except ( AttributeError, AssertionError ):
415 main.log.exception( "Something is wrong with ONOS device view" )
416 main.log.info( main.ONOScli1.devices() )
417 utilities.assert_equals(
418 expect=main.TRUE,
419 actual=roleCall,
420 onpass="Re-assigned switch mastership to designated controller",
421 onfail="Something wrong with deviceRole calls" )
422
423 main.step( "Check mastership was correctly assigned" )
424 roleCheck = main.TRUE
425 # NOTE: This is due to the fact that device mastership change is not
426 # atomic and is actually a multi step process
427 time.sleep( 5 )
428 for i in range( len( ipList ) ):
429 ip = ipList[i]
430 deviceId = deviceList[i]
431 # Check assignment
432 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
433 if ip in master:
434 roleCheck = roleCheck and main.TRUE
435 else:
436 roleCheck = roleCheck and main.FALSE
437 main.log.error( "Error, controller " + ip + " is not" +
438 " master " + "of device " +
439 str( deviceId ) + ". Master is " +
440 repr( master ) + "." )
441 utilities.assert_equals(
442 expect=main.TRUE,
443 actual=roleCheck,
444 onpass="Switches were successfully reassigned to designated " +
445 "controller",
446 onfail="Switches were not successfully reassigned" )
447
448 def CASE3( self, main ):
449 """
450 Assign intents
451 """
452 import time
453 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700454 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700455 assert main, "main not defined"
456 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700457 assert main.CLIs, "main.CLIs not defined"
458 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700460 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700461 "assign predetermined host-to-host intents." +\
462 " After installation, check that the intent" +\
463 " is distributed to all nodes and the state" +\
464 " is INSTALLED"
465
466 # install onos-app-fwd
467 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700468 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700469 utilities.assert_equals( expect=main.TRUE, actual=installResults,
470 onpass="Install fwd successful",
471 onfail="Install fwd failed" )
472
473 main.step( "Check app ids" )
474 appCheck = main.TRUE
475 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700476 for i in range( main.numCtrls ):
477 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700478 name="appToIDCheck-" + str( i ),
479 args=[] )
480 threads.append( t )
481 t.start()
482
483 for t in threads:
484 t.join()
485 appCheck = appCheck and t.result
486 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700487 main.log.warn( main.CLIs[0].apps() )
488 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700489 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
490 onpass="App Ids seem to be correct",
491 onfail="Something is wrong with app Ids" )
492
493 main.step( "Discovering Hosts( Via pingall for now )" )
494 # FIXME: Once we have a host discovery mechanism, use that instead
495 # REACTIVE FWD test
496 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700497 passMsg = "Reactive Pingall test passed"
498 time1 = time.time()
499 pingResult = main.Mininet1.pingall()
500 time2 = time.time()
501 if not pingResult:
502 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700503 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700504 passMsg += " on the second try"
505 utilities.assert_equals(
506 expect=main.TRUE,
507 actual=pingResult,
508 onpass= passMsg,
509 onfail="Reactive Pingall failed, " +
510 "one or more ping pairs failed" )
511 main.log.info( "Time for pingall: %2f seconds" %
512 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700513 # timeout for fwd flows
514 time.sleep( 11 )
515 # uninstall onos-app-fwd
516 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700517 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700518 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
519 onpass="Uninstall fwd successful",
520 onfail="Uninstall fwd failed" )
521
522 main.step( "Check app ids" )
523 threads = []
524 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700525 for i in range( main.numCtrls ):
526 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700527 name="appToIDCheck-" + str( i ),
528 args=[] )
529 threads.append( t )
530 t.start()
531
532 for t in threads:
533 t.join()
534 appCheck2 = appCheck2 and t.result
535 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700536 main.log.warn( main.CLIs[0].apps() )
537 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700538 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
539 onpass="App Ids seem to be correct",
540 onfail="Something is wrong with app Ids" )
541
542 main.step( "Add host intents via cli" )
543 intentIds = []
544 # TODO: move the host numbers to params
545 # Maybe look at all the paths we ping?
546 intentAddResult = True
547 hostResult = main.TRUE
548 for i in range( 8, 18 ):
549 main.log.info( "Adding host intent between h" + str( i ) +
550 " and h" + str( i + 10 ) )
551 host1 = "00:00:00:00:00:" + \
552 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
553 host2 = "00:00:00:00:00:" + \
554 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
555 # NOTE: getHost can return None
556 host1Dict = main.ONOScli1.getHost( host1 )
557 host2Dict = main.ONOScli1.getHost( host2 )
558 host1Id = None
559 host2Id = None
560 if host1Dict and host2Dict:
561 host1Id = host1Dict.get( 'id', None )
562 host2Id = host2Dict.get( 'id', None )
563 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700564 nodeNum = ( i % main.numCtrls )
565 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700566 if tmpId:
567 main.log.info( "Added intent with id: " + tmpId )
568 intentIds.append( tmpId )
569 else:
570 main.log.error( "addHostIntent returned: " +
571 repr( tmpId ) )
572 else:
573 main.log.error( "Error, getHost() failed for h" + str( i ) +
574 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700575 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700576 main.log.warn( "Hosts output: " )
577 try:
578 main.log.warn( json.dumps( json.loads( hosts ),
579 sort_keys=True,
580 indent=4,
581 separators=( ',', ': ' ) ) )
582 except ( ValueError, TypeError ):
583 main.log.warn( repr( hosts ) )
584 hostResult = main.FALSE
585 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
586 onpass="Found a host id for each host",
587 onfail="Error looking up host ids" )
588
589 intentStart = time.time()
590 onosIds = main.ONOScli1.getAllIntentsId()
591 main.log.info( "Submitted intents: " + str( intentIds ) )
592 main.log.info( "Intents in ONOS: " + str( onosIds ) )
593 for intent in intentIds:
594 if intent in onosIds:
595 pass # intent submitted is in onos
596 else:
597 intentAddResult = False
598 if intentAddResult:
599 intentStop = time.time()
600 else:
601 intentStop = None
602 # Print the intent states
603 intents = main.ONOScli1.intents()
604 intentStates = []
605 installedCheck = True
606 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
607 count = 0
608 try:
609 for intent in json.loads( intents ):
610 state = intent.get( 'state', None )
611 if "INSTALLED" not in state:
612 installedCheck = False
613 intentId = intent.get( 'id', None )
614 intentStates.append( ( intentId, state ) )
615 except ( ValueError, TypeError ):
616 main.log.exception( "Error parsing intents" )
617 # add submitted intents not in the store
618 tmplist = [ i for i, s in intentStates ]
619 missingIntents = False
620 for i in intentIds:
621 if i not in tmplist:
622 intentStates.append( ( i, " - " ) )
623 missingIntents = True
624 intentStates.sort()
625 for i, s in intentStates:
626 count += 1
627 main.log.info( "%-6s%-15s%-15s" %
628 ( str( count ), str( i ), str( s ) ) )
629 leaders = main.ONOScli1.leaders()
630 try:
631 missing = False
632 if leaders:
633 parsedLeaders = json.loads( leaders )
634 main.log.warn( json.dumps( parsedLeaders,
635 sort_keys=True,
636 indent=4,
637 separators=( ',', ': ' ) ) )
638 # check for all intent partitions
639 topics = []
640 for i in range( 14 ):
641 topics.append( "intent-partition-" + str( i ) )
642 main.log.debug( topics )
643 ONOStopics = [ j['topic'] for j in parsedLeaders ]
644 for topic in topics:
645 if topic not in ONOStopics:
646 main.log.error( "Error: " + topic +
647 " not in leaders" )
648 missing = True
649 else:
650 main.log.error( "leaders() returned None" )
651 except ( ValueError, TypeError ):
652 main.log.exception( "Error parsing leaders" )
653 main.log.error( repr( leaders ) )
654 # Check all nodes
655 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700656 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700657 response = node.leaders( jsonFormat=False)
658 main.log.warn( str( node.name ) + " leaders output: \n" +
659 str( response ) )
660
661 partitions = main.ONOScli1.partitions()
662 try:
663 if partitions :
664 parsedPartitions = json.loads( partitions )
665 main.log.warn( json.dumps( parsedPartitions,
666 sort_keys=True,
667 indent=4,
668 separators=( ',', ': ' ) ) )
669 # TODO check for a leader in all paritions
670 # TODO check for consistency among nodes
671 else:
672 main.log.error( "partitions() returned None" )
673 except ( ValueError, TypeError ):
674 main.log.exception( "Error parsing partitions" )
675 main.log.error( repr( partitions ) )
676 pendingMap = main.ONOScli1.pendingMap()
677 try:
678 if pendingMap :
679 parsedPending = json.loads( pendingMap )
680 main.log.warn( json.dumps( parsedPending,
681 sort_keys=True,
682 indent=4,
683 separators=( ',', ': ' ) ) )
684 # TODO check something here?
685 else:
686 main.log.error( "pendingMap() returned None" )
687 except ( ValueError, TypeError ):
688 main.log.exception( "Error parsing pending map" )
689 main.log.error( repr( pendingMap ) )
690
691 intentAddResult = bool( intentAddResult and not missingIntents and
692 installedCheck )
693 if not intentAddResult:
694 main.log.error( "Error in pushing host intents to ONOS" )
695
696 main.step( "Intent Anti-Entropy dispersion" )
697 for i in range(100):
698 correct = True
699 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700700 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700701 onosIds = []
702 ids = cli.getAllIntentsId()
703 onosIds.append( ids )
704 main.log.debug( "Intents in " + cli.name + ": " +
705 str( sorted( onosIds ) ) )
706 if sorted( ids ) != sorted( intentIds ):
707 main.log.warn( "Set of intent IDs doesn't match" )
708 correct = False
709 break
710 else:
711 intents = json.loads( cli.intents() )
712 for intent in intents:
713 if intent[ 'state' ] != "INSTALLED":
714 main.log.warn( "Intent " + intent[ 'id' ] +
715 " is " + intent[ 'state' ] )
716 correct = False
717 break
718 if correct:
719 break
720 else:
721 time.sleep(1)
722 if not intentStop:
723 intentStop = time.time()
724 global gossipTime
725 gossipTime = intentStop - intentStart
726 main.log.info( "It took about " + str( gossipTime ) +
727 " seconds for all intents to appear in each node" )
728 # FIXME: make this time configurable/calculate based off of number of
729 # nodes and gossip rounds
730 utilities.assert_greater_equals(
731 expect=40, actual=gossipTime,
732 onpass="ECM anti-entropy for intents worked within " +
733 "expected time",
734 onfail="Intent ECM anti-entropy took too long" )
735 if gossipTime <= 40:
736 intentAddResult = True
737
738 if not intentAddResult or "key" in pendingMap:
739 import time
740 installedCheck = True
741 main.log.info( "Sleeping 60 seconds to see if intents are found" )
742 time.sleep( 60 )
743 onosIds = main.ONOScli1.getAllIntentsId()
744 main.log.info( "Submitted intents: " + str( intentIds ) )
745 main.log.info( "Intents in ONOS: " + str( onosIds ) )
746 # Print the intent states
747 intents = main.ONOScli1.intents()
748 intentStates = []
749 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
750 count = 0
751 try:
752 for intent in json.loads( intents ):
753 # Iter through intents of a node
754 state = intent.get( 'state', None )
755 if "INSTALLED" not in state:
756 installedCheck = False
757 intentId = intent.get( 'id', None )
758 intentStates.append( ( intentId, state ) )
759 except ( ValueError, TypeError ):
760 main.log.exception( "Error parsing intents" )
761 # add submitted intents not in the store
762 tmplist = [ i for i, s in intentStates ]
763 for i in intentIds:
764 if i not in tmplist:
765 intentStates.append( ( i, " - " ) )
766 intentStates.sort()
767 for i, s in intentStates:
768 count += 1
769 main.log.info( "%-6s%-15s%-15s" %
770 ( str( count ), str( i ), str( s ) ) )
771 leaders = main.ONOScli1.leaders()
772 try:
773 missing = False
774 if leaders:
775 parsedLeaders = json.loads( leaders )
776 main.log.warn( json.dumps( parsedLeaders,
777 sort_keys=True,
778 indent=4,
779 separators=( ',', ': ' ) ) )
780 # check for all intent partitions
781 # check for election
782 topics = []
783 for i in range( 14 ):
784 topics.append( "intent-partition-" + str( i ) )
785 # FIXME: this should only be after we start the app
786 topics.append( "org.onosproject.election" )
787 main.log.debug( topics )
788 ONOStopics = [ j['topic'] for j in parsedLeaders ]
789 for topic in topics:
790 if topic not in ONOStopics:
791 main.log.error( "Error: " + topic +
792 " not in leaders" )
793 missing = True
794 else:
795 main.log.error( "leaders() returned None" )
796 except ( ValueError, TypeError ):
797 main.log.exception( "Error parsing leaders" )
798 main.log.error( repr( leaders ) )
799 # Check all nodes
800 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700801 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700802 response = node.leaders( jsonFormat=False)
803 main.log.warn( str( node.name ) + " leaders output: \n" +
804 str( response ) )
805
806 partitions = main.ONOScli1.partitions()
807 try:
808 if partitions :
809 parsedPartitions = json.loads( partitions )
810 main.log.warn( json.dumps( parsedPartitions,
811 sort_keys=True,
812 indent=4,
813 separators=( ',', ': ' ) ) )
814 # TODO check for a leader in all paritions
815 # TODO check for consistency among nodes
816 else:
817 main.log.error( "partitions() returned None" )
818 except ( ValueError, TypeError ):
819 main.log.exception( "Error parsing partitions" )
820 main.log.error( repr( partitions ) )
821 pendingMap = main.ONOScli1.pendingMap()
822 try:
823 if pendingMap :
824 parsedPending = json.loads( pendingMap )
825 main.log.warn( json.dumps( parsedPending,
826 sort_keys=True,
827 indent=4,
828 separators=( ',', ': ' ) ) )
829 # TODO check something here?
830 else:
831 main.log.error( "pendingMap() returned None" )
832 except ( ValueError, TypeError ):
833 main.log.exception( "Error parsing pending map" )
834 main.log.error( repr( pendingMap ) )
835
836 def CASE4( self, main ):
837 """
838 Ping across added host intents
839 """
840 import json
841 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700842 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700843 assert main, "main not defined"
844 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700845 assert main.CLIs, "main.CLIs not defined"
846 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700847 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700848 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700849 "functionality and check the state of " +\
850 "the intent"
851 main.step( "Ping across added host intents" )
852 PingResult = main.TRUE
853 for i in range( 8, 18 ):
854 ping = main.Mininet1.pingHost( src="h" + str( i ),
855 target="h" + str( i + 10 ) )
856 PingResult = PingResult and ping
857 if ping == main.FALSE:
858 main.log.warn( "Ping failed between h" + str( i ) +
859 " and h" + str( i + 10 ) )
860 elif ping == main.TRUE:
861 main.log.info( "Ping test passed!" )
862 # Don't set PingResult or you'd override failures
863 if PingResult == main.FALSE:
864 main.log.error(
865 "Intents have not been installed correctly, pings failed." )
866 # TODO: pretty print
867 main.log.warn( "ONOS1 intents: " )
868 try:
869 tmpIntents = main.ONOScli1.intents()
870 main.log.warn( json.dumps( json.loads( tmpIntents ),
871 sort_keys=True,
872 indent=4,
873 separators=( ',', ': ' ) ) )
874 except ( ValueError, TypeError ):
875 main.log.warn( repr( tmpIntents ) )
876 utilities.assert_equals(
877 expect=main.TRUE,
878 actual=PingResult,
879 onpass="Intents have been installed correctly and pings work",
880 onfail="Intents have not been installed correctly, pings failed." )
881
882 main.step( "Check Intent state" )
883 installedCheck = False
884 loopCount = 0
885 while not installedCheck and loopCount < 40:
886 installedCheck = True
887 # Print the intent states
888 intents = main.ONOScli1.intents()
889 intentStates = []
890 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
891 count = 0
892 # Iter through intents of a node
893 try:
894 for intent in json.loads( intents ):
895 state = intent.get( 'state', None )
896 if "INSTALLED" not in state:
897 installedCheck = False
898 intentId = intent.get( 'id', None )
899 intentStates.append( ( intentId, state ) )
900 except ( ValueError, TypeError ):
901 main.log.exception( "Error parsing intents." )
902 # Print states
903 intentStates.sort()
904 for i, s in intentStates:
905 count += 1
906 main.log.info( "%-6s%-15s%-15s" %
907 ( str( count ), str( i ), str( s ) ) )
908 if not installedCheck:
909 time.sleep( 1 )
910 loopCount += 1
911 utilities.assert_equals( expect=True, actual=installedCheck,
912 onpass="Intents are all INSTALLED",
913 onfail="Intents are not all in " +
914 "INSTALLED state" )
915
916 main.step( "Check leadership of topics" )
917 leaders = main.ONOScli1.leaders()
918 topicCheck = main.TRUE
919 try:
920 if leaders:
921 parsedLeaders = json.loads( leaders )
922 main.log.warn( json.dumps( parsedLeaders,
923 sort_keys=True,
924 indent=4,
925 separators=( ',', ': ' ) ) )
926 # check for all intent partitions
927 # check for election
928 # TODO: Look at Devices as topics now that it uses this system
929 topics = []
930 for i in range( 14 ):
931 topics.append( "intent-partition-" + str( i ) )
932 # FIXME: this should only be after we start the app
933 # FIXME: topics.append( "org.onosproject.election" )
934 # Print leaders output
935 main.log.debug( topics )
936 ONOStopics = [ j['topic'] for j in parsedLeaders ]
937 for topic in topics:
938 if topic not in ONOStopics:
939 main.log.error( "Error: " + topic +
940 " not in leaders" )
941 topicCheck = main.FALSE
942 else:
943 main.log.error( "leaders() returned None" )
944 topicCheck = main.FALSE
945 except ( ValueError, TypeError ):
946 topicCheck = main.FALSE
947 main.log.exception( "Error parsing leaders" )
948 main.log.error( repr( leaders ) )
949 # TODO: Check for a leader of these topics
950 # Check all nodes
951 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700952 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700953 response = node.leaders( jsonFormat=False)
954 main.log.warn( str( node.name ) + " leaders output: \n" +
955 str( response ) )
956
957 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
958 onpass="intent Partitions is in leaders",
959 onfail="Some topics were lost " )
960 # Print partitions
961 partitions = main.ONOScli1.partitions()
962 try:
963 if partitions :
964 parsedPartitions = json.loads( partitions )
965 main.log.warn( json.dumps( parsedPartitions,
966 sort_keys=True,
967 indent=4,
968 separators=( ',', ': ' ) ) )
969 # TODO check for a leader in all paritions
970 # TODO check for consistency among nodes
971 else:
972 main.log.error( "partitions() returned None" )
973 except ( ValueError, TypeError ):
974 main.log.exception( "Error parsing partitions" )
975 main.log.error( repr( partitions ) )
976 # Print Pending Map
977 pendingMap = main.ONOScli1.pendingMap()
978 try:
979 if pendingMap :
980 parsedPending = json.loads( pendingMap )
981 main.log.warn( json.dumps( parsedPending,
982 sort_keys=True,
983 indent=4,
984 separators=( ',', ': ' ) ) )
985 # TODO check something here?
986 else:
987 main.log.error( "pendingMap() returned None" )
988 except ( ValueError, TypeError ):
989 main.log.exception( "Error parsing pending map" )
990 main.log.error( repr( pendingMap ) )
991
992 if not installedCheck:
993 main.log.info( "Waiting 60 seconds to see if the state of " +
994 "intents change" )
995 time.sleep( 60 )
996 # Print the intent states
997 intents = main.ONOScli1.intents()
998 intentStates = []
999 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1000 count = 0
1001 # Iter through intents of a node
1002 try:
1003 for intent in json.loads( intents ):
1004 state = intent.get( 'state', None )
1005 if "INSTALLED" not in state:
1006 installedCheck = False
1007 intentId = intent.get( 'id', None )
1008 intentStates.append( ( intentId, state ) )
1009 except ( ValueError, TypeError ):
1010 main.log.exception( "Error parsing intents." )
1011 intentStates.sort()
1012 for i, s in intentStates:
1013 count += 1
1014 main.log.info( "%-6s%-15s%-15s" %
1015 ( str( count ), str( i ), str( s ) ) )
1016 leaders = main.ONOScli1.leaders()
1017 try:
1018 missing = False
1019 if leaders:
1020 parsedLeaders = json.loads( leaders )
1021 main.log.warn( json.dumps( parsedLeaders,
1022 sort_keys=True,
1023 indent=4,
1024 separators=( ',', ': ' ) ) )
1025 # check for all intent partitions
1026 # check for election
1027 topics = []
1028 for i in range( 14 ):
1029 topics.append( "intent-partition-" + str( i ) )
1030 # FIXME: this should only be after we start the app
1031 topics.append( "org.onosproject.election" )
1032 main.log.debug( topics )
1033 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1034 for topic in topics:
1035 if topic not in ONOStopics:
1036 main.log.error( "Error: " + topic +
1037 " not in leaders" )
1038 missing = True
1039 else:
1040 main.log.error( "leaders() returned None" )
1041 except ( ValueError, TypeError ):
1042 main.log.exception( "Error parsing leaders" )
1043 main.log.error( repr( leaders ) )
1044 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001045 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001046 response = node.leaders( jsonFormat=False)
1047 main.log.warn( str( node.name ) + " leaders output: \n" +
1048 str( response ) )
1049
1050 partitions = main.ONOScli1.partitions()
1051 try:
1052 if partitions :
1053 parsedPartitions = json.loads( partitions )
1054 main.log.warn( json.dumps( parsedPartitions,
1055 sort_keys=True,
1056 indent=4,
1057 separators=( ',', ': ' ) ) )
1058 # TODO check for a leader in all paritions
1059 # TODO check for consistency among nodes
1060 else:
1061 main.log.error( "partitions() returned None" )
1062 except ( ValueError, TypeError ):
1063 main.log.exception( "Error parsing partitions" )
1064 main.log.error( repr( partitions ) )
1065 pendingMap = main.ONOScli1.pendingMap()
1066 try:
1067 if pendingMap :
1068 parsedPending = json.loads( pendingMap )
1069 main.log.warn( json.dumps( parsedPending,
1070 sort_keys=True,
1071 indent=4,
1072 separators=( ',', ': ' ) ) )
1073 # TODO check something here?
1074 else:
1075 main.log.error( "pendingMap() returned None" )
1076 except ( ValueError, TypeError ):
1077 main.log.exception( "Error parsing pending map" )
1078 main.log.error( repr( pendingMap ) )
1079 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001080 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001081 main.step( "Wait a minute then ping again" )
1082 # the wait is above
1083 PingResult = main.TRUE
1084 for i in range( 8, 18 ):
1085 ping = main.Mininet1.pingHost( src="h" + str( i ),
1086 target="h" + str( i + 10 ) )
1087 PingResult = PingResult and ping
1088 if ping == main.FALSE:
1089 main.log.warn( "Ping failed between h" + str( i ) +
1090 " and h" + str( i + 10 ) )
1091 elif ping == main.TRUE:
1092 main.log.info( "Ping test passed!" )
1093 # Don't set PingResult or you'd override failures
1094 if PingResult == main.FALSE:
1095 main.log.error(
1096 "Intents have not been installed correctly, pings failed." )
1097 # TODO: pretty print
1098 main.log.warn( "ONOS1 intents: " )
1099 try:
1100 tmpIntents = main.ONOScli1.intents()
1101 main.log.warn( json.dumps( json.loads( tmpIntents ),
1102 sort_keys=True,
1103 indent=4,
1104 separators=( ',', ': ' ) ) )
1105 except ( ValueError, TypeError ):
1106 main.log.warn( repr( tmpIntents ) )
1107 utilities.assert_equals(
1108 expect=main.TRUE,
1109 actual=PingResult,
1110 onpass="Intents have been installed correctly and pings work",
1111 onfail="Intents have not been installed correctly, pings failed." )
1112
1113 def CASE5( self, main ):
1114 """
1115 Reading state of ONOS
1116 """
1117 import json
1118 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001119 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001120 assert main, "main not defined"
1121 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001122 assert main.CLIs, "main.CLIs not defined"
1123 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001124
1125 main.case( "Setting up and gathering data for current state" )
1126 # The general idea for this test case is to pull the state of
1127 # ( intents,flows, topology,... ) from each ONOS node
1128 # We can then compare them with each other and also with past states
1129
1130 main.step( "Check that each switch has a master" )
1131 global mastershipState
1132 mastershipState = '[]'
1133
1134 # Assert that each device has a master
1135 rolesNotNull = main.TRUE
1136 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001137 for i in range( main.numCtrls ):
1138 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001139 name="rolesNotNull-" + str( i ),
1140 args=[] )
1141 threads.append( t )
1142 t.start()
1143
1144 for t in threads:
1145 t.join()
1146 rolesNotNull = rolesNotNull and t.result
1147 utilities.assert_equals(
1148 expect=main.TRUE,
1149 actual=rolesNotNull,
1150 onpass="Each device has a master",
1151 onfail="Some devices don't have a master assigned" )
1152
1153 main.step( "Get the Mastership of each switch from each controller" )
1154 ONOSMastership = []
1155 mastershipCheck = main.FALSE
1156 consistentMastership = True
1157 rolesResults = True
1158 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001159 for i in range( main.numCtrls ):
1160 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001161 name="roles-" + str( i ),
1162 args=[] )
1163 threads.append( t )
1164 t.start()
1165
1166 for t in threads:
1167 t.join()
1168 ONOSMastership.append( t.result )
1169
Jon Halle1a3b752015-07-22 13:02:46 -07001170 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001171 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1172 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1173 " roles" )
1174 main.log.warn(
1175 "ONOS" + str( i + 1 ) + " mastership response: " +
1176 repr( ONOSMastership[i] ) )
1177 rolesResults = False
1178 utilities.assert_equals(
1179 expect=True,
1180 actual=rolesResults,
1181 onpass="No error in reading roles output",
1182 onfail="Error in reading roles from ONOS" )
1183
1184 main.step( "Check for consistency in roles from each controller" )
1185 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1186 main.log.info(
1187 "Switch roles are consistent across all ONOS nodes" )
1188 else:
1189 consistentMastership = False
1190 utilities.assert_equals(
1191 expect=True,
1192 actual=consistentMastership,
1193 onpass="Switch roles are consistent across all ONOS nodes",
1194 onfail="ONOS nodes have different views of switch roles" )
1195
1196 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001197 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001198 try:
1199 main.log.warn(
1200 "ONOS" + str( i + 1 ) + " roles: ",
1201 json.dumps(
1202 json.loads( ONOSMastership[ i ] ),
1203 sort_keys=True,
1204 indent=4,
1205 separators=( ',', ': ' ) ) )
1206 except ( ValueError, TypeError ):
1207 main.log.warn( repr( ONOSMastership[ i ] ) )
1208 elif rolesResults and consistentMastership:
1209 mastershipCheck = main.TRUE
1210 mastershipState = ONOSMastership[ 0 ]
1211
1212 main.step( "Get the intents from each controller" )
1213 global intentState
1214 intentState = []
1215 ONOSIntents = []
1216 intentCheck = main.FALSE
1217 consistentIntents = True
1218 intentsResults = True
1219 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001220 for i in range( main.numCtrls ):
1221 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001222 name="intents-" + str( i ),
1223 args=[],
1224 kwargs={ 'jsonFormat': True } )
1225 threads.append( t )
1226 t.start()
1227
1228 for t in threads:
1229 t.join()
1230 ONOSIntents.append( t.result )
1231
Jon Halle1a3b752015-07-22 13:02:46 -07001232 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001233 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1234 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1235 " intents" )
1236 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1237 repr( ONOSIntents[ i ] ) )
1238 intentsResults = False
1239 utilities.assert_equals(
1240 expect=True,
1241 actual=intentsResults,
1242 onpass="No error in reading intents output",
1243 onfail="Error in reading intents from ONOS" )
1244
1245 main.step( "Check for consistency in Intents from each controller" )
1246 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1247 main.log.info( "Intents are consistent across all ONOS " +
1248 "nodes" )
1249 else:
1250 consistentIntents = False
1251 main.log.error( "Intents not consistent" )
1252 utilities.assert_equals(
1253 expect=True,
1254 actual=consistentIntents,
1255 onpass="Intents are consistent across all ONOS nodes",
1256 onfail="ONOS nodes have different views of intents" )
1257
1258 if intentsResults:
1259 # Try to make it easy to figure out what is happening
1260 #
1261 # Intent ONOS1 ONOS2 ...
1262 # 0x01 INSTALLED INSTALLING
1263 # ... ... ...
1264 # ... ... ...
1265 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001266 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001267 title += " " * 10 + "ONOS" + str( n + 1 )
1268 main.log.warn( title )
1269 # get all intent keys in the cluster
1270 keys = []
1271 for nodeStr in ONOSIntents:
1272 node = json.loads( nodeStr )
1273 for intent in node:
1274 keys.append( intent.get( 'id' ) )
1275 keys = set( keys )
1276 for key in keys:
1277 row = "%-13s" % key
1278 for nodeStr in ONOSIntents:
1279 node = json.loads( nodeStr )
1280 for intent in node:
1281 if intent.get( 'id', "Error" ) == key:
1282 row += "%-15s" % intent.get( 'state' )
1283 main.log.warn( row )
1284 # End table view
1285
1286 if intentsResults and not consistentIntents:
1287 # print the json objects
1288 n = len(ONOSIntents)
1289 main.log.debug( "ONOS" + str( n ) + " intents: " )
1290 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1291 sort_keys=True,
1292 indent=4,
1293 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001294 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001295 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1296 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1297 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1298 sort_keys=True,
1299 indent=4,
1300 separators=( ',', ': ' ) ) )
1301 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001302 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001303 str( n ) + " intents" )
1304 elif intentsResults and consistentIntents:
1305 intentCheck = main.TRUE
1306 intentState = ONOSIntents[ 0 ]
1307
1308 main.step( "Get the flows from each controller" )
1309 global flowState
1310 flowState = []
1311 ONOSFlows = []
1312 ONOSFlowsJson = []
1313 flowCheck = main.FALSE
1314 consistentFlows = True
1315 flowsResults = True
1316 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001317 for i in range( main.numCtrls ):
1318 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001319 name="flows-" + str( i ),
1320 args=[],
1321 kwargs={ 'jsonFormat': True } )
1322 threads.append( t )
1323 t.start()
1324
1325 # NOTE: Flows command can take some time to run
1326 time.sleep(30)
1327 for t in threads:
1328 t.join()
1329 result = t.result
1330 ONOSFlows.append( result )
1331
Jon Halle1a3b752015-07-22 13:02:46 -07001332 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001333 num = str( i + 1 )
1334 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1335 main.log.error( "Error in getting ONOS" + num + " flows" )
1336 main.log.warn( "ONOS" + num + " flows response: " +
1337 repr( ONOSFlows[ i ] ) )
1338 flowsResults = False
1339 ONOSFlowsJson.append( None )
1340 else:
1341 try:
1342 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1343 except ( ValueError, TypeError ):
1344 # FIXME: change this to log.error?
1345 main.log.exception( "Error in parsing ONOS" + num +
1346 " response as json." )
1347 main.log.error( repr( ONOSFlows[ i ] ) )
1348 ONOSFlowsJson.append( None )
1349 flowsResults = False
1350 utilities.assert_equals(
1351 expect=True,
1352 actual=flowsResults,
1353 onpass="No error in reading flows output",
1354 onfail="Error in reading flows from ONOS" )
1355
1356 main.step( "Check for consistency in Flows from each controller" )
1357 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1358 if all( tmp ):
1359 main.log.info( "Flow count is consistent across all ONOS nodes" )
1360 else:
1361 consistentFlows = False
1362 utilities.assert_equals(
1363 expect=True,
1364 actual=consistentFlows,
1365 onpass="The flow count is consistent across all ONOS nodes",
1366 onfail="ONOS nodes have different flow counts" )
1367
1368 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001369 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001370 try:
1371 main.log.warn(
1372 "ONOS" + str( i + 1 ) + " flows: " +
1373 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1374 indent=4, separators=( ',', ': ' ) ) )
1375 except ( ValueError, TypeError ):
1376 main.log.warn(
1377 "ONOS" + str( i + 1 ) + " flows: " +
1378 repr( ONOSFlows[ i ] ) )
1379 elif flowsResults and consistentFlows:
1380 flowCheck = main.TRUE
1381 flowState = ONOSFlows[ 0 ]
1382
1383 main.step( "Get the OF Table entries" )
1384 global flows
1385 flows = []
1386 for i in range( 1, 29 ):
Jon Hall9043c902015-07-30 14:23:44 -07001387 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001388 if flowCheck == main.FALSE:
1389 for table in flows:
1390 main.log.warn( table )
1391 # TODO: Compare switch flow tables with ONOS flow tables
1392
1393 main.step( "Start continuous pings" )
1394 main.Mininet2.pingLong(
1395 src=main.params[ 'PING' ][ 'source1' ],
1396 target=main.params[ 'PING' ][ 'target1' ],
1397 pingTime=500 )
1398 main.Mininet2.pingLong(
1399 src=main.params[ 'PING' ][ 'source2' ],
1400 target=main.params[ 'PING' ][ 'target2' ],
1401 pingTime=500 )
1402 main.Mininet2.pingLong(
1403 src=main.params[ 'PING' ][ 'source3' ],
1404 target=main.params[ 'PING' ][ 'target3' ],
1405 pingTime=500 )
1406 main.Mininet2.pingLong(
1407 src=main.params[ 'PING' ][ 'source4' ],
1408 target=main.params[ 'PING' ][ 'target4' ],
1409 pingTime=500 )
1410 main.Mininet2.pingLong(
1411 src=main.params[ 'PING' ][ 'source5' ],
1412 target=main.params[ 'PING' ][ 'target5' ],
1413 pingTime=500 )
1414 main.Mininet2.pingLong(
1415 src=main.params[ 'PING' ][ 'source6' ],
1416 target=main.params[ 'PING' ][ 'target6' ],
1417 pingTime=500 )
1418 main.Mininet2.pingLong(
1419 src=main.params[ 'PING' ][ 'source7' ],
1420 target=main.params[ 'PING' ][ 'target7' ],
1421 pingTime=500 )
1422 main.Mininet2.pingLong(
1423 src=main.params[ 'PING' ][ 'source8' ],
1424 target=main.params[ 'PING' ][ 'target8' ],
1425 pingTime=500 )
1426 main.Mininet2.pingLong(
1427 src=main.params[ 'PING' ][ 'source9' ],
1428 target=main.params[ 'PING' ][ 'target9' ],
1429 pingTime=500 )
1430 main.Mininet2.pingLong(
1431 src=main.params[ 'PING' ][ 'source10' ],
1432 target=main.params[ 'PING' ][ 'target10' ],
1433 pingTime=500 )
1434
1435 main.step( "Collecting topology information from ONOS" )
1436 devices = []
1437 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001438 for i in range( main.numCtrls ):
1439 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001440 name="devices-" + str( i ),
1441 args=[ ] )
1442 threads.append( t )
1443 t.start()
1444
1445 for t in threads:
1446 t.join()
1447 devices.append( t.result )
1448 hosts = []
1449 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001450 for i in range( main.numCtrls ):
1451 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001452 name="hosts-" + str( i ),
1453 args=[ ] )
1454 threads.append( t )
1455 t.start()
1456
1457 for t in threads:
1458 t.join()
1459 try:
1460 hosts.append( json.loads( t.result ) )
1461 except ( ValueError, TypeError ):
1462 # FIXME: better handling of this, print which node
1463 # Maybe use thread name?
1464 main.log.exception( "Error parsing json output of hosts" )
1465 # FIXME: should this be an empty json object instead?
1466 hosts.append( None )
1467
1468 ports = []
1469 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001470 for i in range( main.numCtrls ):
1471 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001472 name="ports-" + str( i ),
1473 args=[ ] )
1474 threads.append( t )
1475 t.start()
1476
1477 for t in threads:
1478 t.join()
1479 ports.append( t.result )
1480 links = []
1481 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001482 for i in range( main.numCtrls ):
1483 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001484 name="links-" + str( i ),
1485 args=[ ] )
1486 threads.append( t )
1487 t.start()
1488
1489 for t in threads:
1490 t.join()
1491 links.append( t.result )
1492 clusters = []
1493 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001494 for i in range( main.numCtrls ):
1495 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001496 name="clusters-" + str( i ),
1497 args=[ ] )
1498 threads.append( t )
1499 t.start()
1500
1501 for t in threads:
1502 t.join()
1503 clusters.append( t.result )
1504 # Compare json objects for hosts and dataplane clusters
1505
1506 # hosts
1507 main.step( "Host view is consistent across ONOS nodes" )
1508 consistentHostsResult = main.TRUE
1509 for controller in range( len( hosts ) ):
1510 controllerStr = str( controller + 1 )
1511 if "Error" not in hosts[ controller ]:
1512 if hosts[ controller ] == hosts[ 0 ]:
1513 continue
1514 else: # hosts not consistent
1515 main.log.error( "hosts from ONOS" +
1516 controllerStr +
1517 " is inconsistent with ONOS1" )
1518 main.log.warn( repr( hosts[ controller ] ) )
1519 consistentHostsResult = main.FALSE
1520
1521 else:
1522 main.log.error( "Error in getting ONOS hosts from ONOS" +
1523 controllerStr )
1524 consistentHostsResult = main.FALSE
1525 main.log.warn( "ONOS" + controllerStr +
1526 " hosts response: " +
1527 repr( hosts[ controller ] ) )
1528 utilities.assert_equals(
1529 expect=main.TRUE,
1530 actual=consistentHostsResult,
1531 onpass="Hosts view is consistent across all ONOS nodes",
1532 onfail="ONOS nodes have different views of hosts" )
1533
1534 main.step( "Each host has an IP address" )
1535 ipResult = main.TRUE
1536 for controller in range( 0, len( hosts ) ):
1537 controllerStr = str( controller + 1 )
1538 for host in hosts[ controller ]:
1539 if not host.get( 'ipAddresses', [ ] ):
1540 main.log.error( "DEBUG:Error with host ips on controller" +
1541 controllerStr + ": " + str( host ) )
1542 ipResult = main.FALSE
1543 utilities.assert_equals(
1544 expect=main.TRUE,
1545 actual=ipResult,
1546 onpass="The ips of the hosts aren't empty",
1547 onfail="The ip of at least one host is missing" )
1548
1549 # Strongly connected clusters of devices
1550 main.step( "Cluster view is consistent across ONOS nodes" )
1551 consistentClustersResult = main.TRUE
1552 for controller in range( len( clusters ) ):
1553 controllerStr = str( controller + 1 )
1554 if "Error" not in clusters[ controller ]:
1555 if clusters[ controller ] == clusters[ 0 ]:
1556 continue
1557 else: # clusters not consistent
1558 main.log.error( "clusters from ONOS" + controllerStr +
1559 " is inconsistent with ONOS1" )
1560 consistentClustersResult = main.FALSE
1561
1562 else:
1563 main.log.error( "Error in getting dataplane clusters " +
1564 "from ONOS" + controllerStr )
1565 consistentClustersResult = main.FALSE
1566 main.log.warn( "ONOS" + controllerStr +
1567 " clusters response: " +
1568 repr( clusters[ controller ] ) )
1569 utilities.assert_equals(
1570 expect=main.TRUE,
1571 actual=consistentClustersResult,
1572 onpass="Clusters view is consistent across all ONOS nodes",
1573 onfail="ONOS nodes have different views of clusters" )
1574 # there should always only be one cluster
1575 main.step( "Cluster view correct across ONOS nodes" )
1576 try:
1577 numClusters = len( json.loads( clusters[ 0 ] ) )
1578 except ( ValueError, TypeError ):
1579 main.log.exception( "Error parsing clusters[0]: " +
1580 repr( clusters[ 0 ] ) )
1581 clusterResults = main.FALSE
1582 if numClusters == 1:
1583 clusterResults = main.TRUE
1584 utilities.assert_equals(
1585 expect=1,
1586 actual=numClusters,
1587 onpass="ONOS shows 1 SCC",
1588 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1589
1590 main.step( "Comparing ONOS topology to MN" )
1591 devicesResults = main.TRUE
1592 linksResults = main.TRUE
1593 hostsResults = main.TRUE
1594 mnSwitches = main.Mininet1.getSwitches()
1595 mnLinks = main.Mininet1.getLinks()
1596 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001597 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001598 controllerStr = str( controller + 1 )
1599 if devices[ controller ] and ports[ controller ] and\
1600 "Error" not in devices[ controller ] and\
1601 "Error" not in ports[ controller ]:
1602
1603 currentDevicesResult = main.Mininet1.compareSwitches(
1604 mnSwitches,
1605 json.loads( devices[ controller ] ),
1606 json.loads( ports[ controller ] ) )
1607 else:
1608 currentDevicesResult = main.FALSE
1609 utilities.assert_equals( expect=main.TRUE,
1610 actual=currentDevicesResult,
1611 onpass="ONOS" + controllerStr +
1612 " Switches view is correct",
1613 onfail="ONOS" + controllerStr +
1614 " Switches view is incorrect" )
1615 if links[ controller ] and "Error" not in links[ controller ]:
1616 currentLinksResult = main.Mininet1.compareLinks(
1617 mnSwitches, mnLinks,
1618 json.loads( links[ controller ] ) )
1619 else:
1620 currentLinksResult = main.FALSE
1621 utilities.assert_equals( expect=main.TRUE,
1622 actual=currentLinksResult,
1623 onpass="ONOS" + controllerStr +
1624 " links view is correct",
1625 onfail="ONOS" + controllerStr +
1626 " links view is incorrect" )
1627
1628 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1629 currentHostsResult = main.Mininet1.compareHosts(
1630 mnHosts,
1631 hosts[ controller ] )
1632 else:
1633 currentHostsResult = main.FALSE
1634 utilities.assert_equals( expect=main.TRUE,
1635 actual=currentHostsResult,
1636 onpass="ONOS" + controllerStr +
1637 " hosts exist in Mininet",
1638 onfail="ONOS" + controllerStr +
1639 " hosts don't match Mininet" )
1640
1641 devicesResults = devicesResults and currentDevicesResult
1642 linksResults = linksResults and currentLinksResult
1643 hostsResults = hostsResults and currentHostsResult
1644
1645 main.step( "Device information is correct" )
1646 utilities.assert_equals(
1647 expect=main.TRUE,
1648 actual=devicesResults,
1649 onpass="Device information is correct",
1650 onfail="Device information is incorrect" )
1651
1652 main.step( "Links are correct" )
1653 utilities.assert_equals(
1654 expect=main.TRUE,
1655 actual=linksResults,
1656 onpass="Link are correct",
1657 onfail="Links are incorrect" )
1658
1659 main.step( "Hosts are correct" )
1660 utilities.assert_equals(
1661 expect=main.TRUE,
1662 actual=hostsResults,
1663 onpass="Hosts are correct",
1664 onfail="Hosts are incorrect" )
1665
1666 def CASE6( self, main ):
1667 """
1668 The Failure case.
1669 """
1670 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001671 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001672 assert main, "main not defined"
1673 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001674 assert main.CLIs, "main.CLIs not defined"
1675 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001676 main.case( "Restart minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -07001677
1678 main.step( "Checking ONOS Logs for errors" )
1679 for node in main.nodes:
1680 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1681 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1682
Jon Hall5cf14d52015-07-16 12:15:19 -07001683 main.step( "Killing 3 ONOS nodes" )
1684 killTime = time.time()
1685 # TODO: Randomize these nodes or base this on partitions
1686 # TODO: use threads in this case
Jon Halle1a3b752015-07-22 13:02:46 -07001687 killResults = main.ONOSbench.onosKill( main.nodes[0].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001688 time.sleep( 10 )
1689 killResults = killResults and\
Jon Halle1a3b752015-07-22 13:02:46 -07001690 main.ONOSbench.onosKill( main.nodes[1].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001691 time.sleep( 10 )
1692 killResults = killResults and\
Jon Halle1a3b752015-07-22 13:02:46 -07001693 main.ONOSbench.onosKill( main.nodes[2].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001694 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1695 onpass="ONOS Killed successfully",
1696 onfail="ONOS kill NOT successful" )
1697
1698 main.step( "Checking if ONOS is up yet" )
1699 count = 0
1700 onosIsupResult = main.FALSE
1701 while onosIsupResult == main.FALSE and count < 10:
Jon Halle1a3b752015-07-22 13:02:46 -07001702 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
1703 onos2Isup = main.ONOSbench.isup( main.nodes[1].ip_address )
1704 onos3Isup = main.ONOSbench.isup( main.nodes[2].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001705 onosIsupResult = onos1Isup and onos2Isup and onos3Isup
1706 count = count + 1
1707 # TODO: if it becomes an issue, we can retry this step a few times
1708 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1709 onpass="ONOS restarted successfully",
1710 onfail="ONOS restart NOT successful" )
1711
Jon Halle1a3b752015-07-22 13:02:46 -07001712 main.step( "Restarting ONOS main.CLIs" )
1713 cliResult1 = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
1714 cliResult2 = main.ONOScli2.startOnosCli( main.nodes[1].ip_address )
1715 cliResult3 = main.ONOScli3.startOnosCli( main.nodes[2].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001716 cliResults = cliResult1 and cliResult2 and cliResult3
1717 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1718 onpass="ONOS cli restarted",
1719 onfail="ONOS cli did not restart" )
1720
1721 # Grab the time of restart so we chan check how long the gossip
1722 # protocol has had time to work
1723 main.restartTime = time.time() - killTime
1724 main.log.debug( "Restart time: " + str( main.restartTime ) )
1725 '''
1726 # FIXME: revisit test plan for election with madan
1727 # Rerun for election on restarted nodes
Jon Halle1a3b752015-07-22 13:02:46 -07001728 run1 = main.CLIs[0].electionTestRun()
1729 run2 = main.CLIs[1].electionTestRun()
1730 run3 = main.CLIs[2].electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001731 runResults = run1 and run2 and run3
1732 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1733 onpass="Reran for election",
1734 onfail="Failed to rerun for election" )
1735 '''
1736 # TODO: MAke this configurable. Also, we are breaking the above timer
1737 time.sleep( 60 )
Jon Halle1a3b752015-07-22 13:02:46 -07001738 main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) )
1739 main.log.debug( main.CLIs[0].leaders( jsonFormat=False ) )
1740 main.log.debug( main.CLIs[0].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001741
1742 def CASE7( self, main ):
1743 """
1744 Check state after ONOS failure
1745 """
1746 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001747 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001748 assert main, "main not defined"
1749 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001750 assert main.CLIs, "main.CLIs not defined"
1751 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001752 main.case( "Running ONOS Constant State Tests" )
1753
1754 main.step( "Check that each switch has a master" )
1755 # Assert that each device has a master
1756 rolesNotNull = main.TRUE
1757 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001758 for i in range( main.numCtrls ):
1759 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001760 name="rolesNotNull-" + str( i ),
1761 args=[ ] )
1762 threads.append( t )
1763 t.start()
1764
1765 for t in threads:
1766 t.join()
1767 rolesNotNull = rolesNotNull and t.result
1768 utilities.assert_equals(
1769 expect=main.TRUE,
1770 actual=rolesNotNull,
1771 onpass="Each device has a master",
1772 onfail="Some devices don't have a master assigned" )
1773
1774 main.step( "Read device roles from ONOS" )
1775 ONOSMastership = []
1776 consistentMastership = True
1777 rolesResults = True
1778 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001779 for i in range( main.numCtrls ):
1780 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001781 name="roles-" + str( i ),
1782 args=[] )
1783 threads.append( t )
1784 t.start()
1785
1786 for t in threads:
1787 t.join()
1788 ONOSMastership.append( t.result )
1789
Jon Halle1a3b752015-07-22 13:02:46 -07001790 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001791 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1792 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1793 " roles" )
1794 main.log.warn(
1795 "ONOS" + str( i + 1 ) + " mastership response: " +
1796 repr( ONOSMastership[i] ) )
1797 rolesResults = False
1798 utilities.assert_equals(
1799 expect=True,
1800 actual=rolesResults,
1801 onpass="No error in reading roles output",
1802 onfail="Error in reading roles from ONOS" )
1803
1804 main.step( "Check for consistency in roles from each controller" )
1805 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1806 main.log.info(
1807 "Switch roles are consistent across all ONOS nodes" )
1808 else:
1809 consistentMastership = False
1810 utilities.assert_equals(
1811 expect=True,
1812 actual=consistentMastership,
1813 onpass="Switch roles are consistent across all ONOS nodes",
1814 onfail="ONOS nodes have different views of switch roles" )
1815
1816 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001817 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001818 main.log.warn(
1819 "ONOS" + str( i + 1 ) + " roles: ",
1820 json.dumps(
1821 json.loads( ONOSMastership[ i ] ),
1822 sort_keys=True,
1823 indent=4,
1824 separators=( ',', ': ' ) ) )
1825
1826 # NOTE: we expect mastership to change on controller failure
1827 '''
1828 description2 = "Compare switch roles from before failure"
1829 main.step( description2 )
1830 try:
1831 currentJson = json.loads( ONOSMastership[0] )
1832 oldJson = json.loads( mastershipState )
1833 except ( ValueError, TypeError ):
1834 main.log.exception( "Something is wrong with parsing " +
1835 "ONOSMastership[0] or mastershipState" )
1836 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1837 main.log.error( "mastershipState" + repr( mastershipState ) )
1838 main.cleanup()
1839 main.exit()
1840 mastershipCheck = main.TRUE
1841 for i in range( 1, 29 ):
1842 switchDPID = str(
1843 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1844 current = [ switch[ 'master' ] for switch in currentJson
1845 if switchDPID in switch[ 'id' ] ]
1846 old = [ switch[ 'master' ] for switch in oldJson
1847 if switchDPID in switch[ 'id' ] ]
1848 if current == old:
1849 mastershipCheck = mastershipCheck and main.TRUE
1850 else:
1851 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1852 mastershipCheck = main.FALSE
1853 utilities.assert_equals(
1854 expect=main.TRUE,
1855 actual=mastershipCheck,
1856 onpass="Mastership of Switches was not changed",
1857 onfail="Mastership of some switches changed" )
1858 '''
1859
1860 main.step( "Get the intents and compare across all nodes" )
1861 ONOSIntents = []
1862 intentCheck = main.FALSE
1863 consistentIntents = True
1864 intentsResults = True
1865 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001866 for i in range( main.numCtrls ):
1867 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001868 name="intents-" + str( i ),
1869 args=[],
1870 kwargs={ 'jsonFormat': True } )
1871 threads.append( t )
1872 t.start()
1873
1874 for t in threads:
1875 t.join()
1876 ONOSIntents.append( t.result )
1877
Jon Halle1a3b752015-07-22 13:02:46 -07001878 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001879 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1880 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1881 " intents" )
1882 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1883 repr( ONOSIntents[ i ] ) )
1884 intentsResults = False
1885 utilities.assert_equals(
1886 expect=True,
1887 actual=intentsResults,
1888 onpass="No error in reading intents output",
1889 onfail="Error in reading intents from ONOS" )
1890
1891 main.step( "Check for consistency in Intents from each controller" )
1892 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1893 main.log.info( "Intents are consistent across all ONOS " +
1894 "nodes" )
1895 else:
1896 consistentIntents = False
1897
1898 # Try to make it easy to figure out what is happening
1899 #
1900 # Intent ONOS1 ONOS2 ...
1901 # 0x01 INSTALLED INSTALLING
1902 # ... ... ...
1903 # ... ... ...
1904 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001905 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001906 title += " " * 10 + "ONOS" + str( n + 1 )
1907 main.log.warn( title )
1908 # get all intent keys in the cluster
1909 keys = []
1910 for nodeStr in ONOSIntents:
1911 node = json.loads( nodeStr )
1912 for intent in node:
1913 keys.append( intent.get( 'id' ) )
1914 keys = set( keys )
1915 for key in keys:
1916 row = "%-13s" % key
1917 for nodeStr in ONOSIntents:
1918 node = json.loads( nodeStr )
1919 for intent in node:
1920 if intent.get( 'id' ) == key:
1921 row += "%-15s" % intent.get( 'state' )
1922 main.log.warn( row )
1923 # End table view
1924
1925 utilities.assert_equals(
1926 expect=True,
1927 actual=consistentIntents,
1928 onpass="Intents are consistent across all ONOS nodes",
1929 onfail="ONOS nodes have different views of intents" )
1930 intentStates = []
1931 for node in ONOSIntents: # Iter through ONOS nodes
1932 nodeStates = []
1933 # Iter through intents of a node
1934 try:
1935 for intent in json.loads( node ):
1936 nodeStates.append( intent[ 'state' ] )
1937 except ( ValueError, TypeError ):
1938 main.log.exception( "Error in parsing intents" )
1939 main.log.error( repr( node ) )
1940 intentStates.append( nodeStates )
1941 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1942 main.log.info( dict( out ) )
1943
1944 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001945 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001946 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1947 main.log.warn( json.dumps(
1948 json.loads( ONOSIntents[ i ] ),
1949 sort_keys=True,
1950 indent=4,
1951 separators=( ',', ': ' ) ) )
1952 elif intentsResults and consistentIntents:
1953 intentCheck = main.TRUE
1954
1955 # NOTE: Store has no durability, so intents are lost across system
1956 # restarts
1957 main.step( "Compare current intents with intents before the failure" )
1958 # NOTE: this requires case 5 to pass for intentState to be set.
1959 # maybe we should stop the test if that fails?
1960 sameIntents = main.FALSE
1961 if intentState and intentState == ONOSIntents[ 0 ]:
1962 sameIntents = main.TRUE
1963 main.log.info( "Intents are consistent with before failure" )
1964 # TODO: possibly the states have changed? we may need to figure out
1965 # what the acceptable states are
1966 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1967 sameIntents = main.TRUE
1968 try:
1969 before = json.loads( intentState )
1970 after = json.loads( ONOSIntents[ 0 ] )
1971 for intent in before:
1972 if intent not in after:
1973 sameIntents = main.FALSE
1974 main.log.debug( "Intent is not currently in ONOS " +
1975 "(at least in the same form):" )
1976 main.log.debug( json.dumps( intent ) )
1977 except ( ValueError, TypeError ):
1978 main.log.exception( "Exception printing intents" )
1979 main.log.debug( repr( ONOSIntents[0] ) )
1980 main.log.debug( repr( intentState ) )
1981 if sameIntents == main.FALSE:
1982 try:
1983 main.log.debug( "ONOS intents before: " )
1984 main.log.debug( json.dumps( json.loads( intentState ),
1985 sort_keys=True, indent=4,
1986 separators=( ',', ': ' ) ) )
1987 main.log.debug( "Current ONOS intents: " )
1988 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1989 sort_keys=True, indent=4,
1990 separators=( ',', ': ' ) ) )
1991 except ( ValueError, TypeError ):
1992 main.log.exception( "Exception printing intents" )
1993 main.log.debug( repr( ONOSIntents[0] ) )
1994 main.log.debug( repr( intentState ) )
1995 utilities.assert_equals(
1996 expect=main.TRUE,
1997 actual=sameIntents,
1998 onpass="Intents are consistent with before failure",
1999 onfail="The Intents changed during failure" )
2000 intentCheck = intentCheck and sameIntents
2001
2002 main.step( "Get the OF Table entries and compare to before " +
2003 "component failure" )
2004 FlowTables = main.TRUE
2005 flows2 = []
2006 for i in range( 28 ):
2007 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall9043c902015-07-30 14:23:44 -07002008 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002009 flows2.append( tmpFlows )
Jon Hall9043c902015-07-30 14:23:44 -07002010 tempResult = main.Mininet1.flowComp(
Jon Hall5cf14d52015-07-16 12:15:19 -07002011 flow1=flows[ i ],
2012 flow2=tmpFlows )
2013 FlowTables = FlowTables and tempResult
2014 if FlowTables == main.FALSE:
2015 main.log.info( "Differences in flow table for switch: s" +
2016 str( i + 1 ) )
2017 utilities.assert_equals(
2018 expect=main.TRUE,
2019 actual=FlowTables,
2020 onpass="No changes were found in the flow tables",
2021 onfail="Changes were found in the flow tables" )
2022
2023 main.Mininet2.pingLongKill()
2024 '''
2025 main.step( "Check the continuous pings to ensure that no packets " +
2026 "were dropped during component failure" )
2027 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2028 main.params[ 'TESTONIP' ] )
2029 LossInPings = main.FALSE
2030 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2031 for i in range( 8, 18 ):
2032 main.log.info(
2033 "Checking for a loss in pings along flow from s" +
2034 str( i ) )
2035 LossInPings = main.Mininet2.checkForLoss(
2036 "/tmp/ping.h" +
2037 str( i ) ) or LossInPings
2038 if LossInPings == main.TRUE:
2039 main.log.info( "Loss in ping detected" )
2040 elif LossInPings == main.ERROR:
2041 main.log.info( "There are multiple mininet process running" )
2042 elif LossInPings == main.FALSE:
2043 main.log.info( "No Loss in the pings" )
2044 main.log.info( "No loss of dataplane connectivity" )
2045 utilities.assert_equals(
2046 expect=main.FALSE,
2047 actual=LossInPings,
2048 onpass="No Loss of connectivity",
2049 onfail="Loss of dataplane connectivity detected" )
2050 '''
2051
2052 main.step( "Leadership Election is still functional" )
2053 # Test of LeadershipElection
2054 leaderList = []
2055 # FIXME: make sure this matches nodes that were restarted
Jon Halle1a3b752015-07-22 13:02:46 -07002056 restarted = [ main.nodes[0].ip_address, main.nodes[1].ip_address,
2057 main.nodes[2].ip_address ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002058
2059 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002060 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002061 leaderN = cli.electionTestLeader()
2062 leaderList.append( leaderN )
2063 if leaderN == main.FALSE:
2064 # error in response
2065 main.log.error( "Something is wrong with " +
2066 "electionTestLeader function, check the" +
2067 " error logs" )
2068 leaderResult = main.FALSE
2069 elif leaderN is None:
2070 main.log.error( cli.name +
2071 " shows no leader for the election-app was" +
2072 " elected after the old one died" )
2073 leaderResult = main.FALSE
2074 elif leaderN in restarted:
2075 main.log.error( cli.name + " shows " + str( leaderN ) +
2076 " as leader for the election-app, but it " +
2077 "was restarted" )
2078 leaderResult = main.FALSE
2079 if len( set( leaderList ) ) != 1:
2080 leaderResult = main.FALSE
2081 main.log.error(
2082 "Inconsistent view of leader for the election test app" )
2083 # TODO: print the list
2084 utilities.assert_equals(
2085 expect=main.TRUE,
2086 actual=leaderResult,
2087 onpass="Leadership election passed",
2088 onfail="Something went wrong with Leadership election" )
2089
2090 def CASE8( self, main ):
2091 """
2092 Compare topo
2093 """
2094 import json
2095 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002096 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002097 assert main, "main not defined"
2098 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002099 assert main.CLIs, "main.CLIs not defined"
2100 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002101
2102 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002103 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002104 " and ONOS"
2105
2106 main.step( "Comparing ONOS topology to MN" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002107 topoResult = main.FALSE
2108 elapsed = 0
2109 count = 0
2110 main.step( "Collecting topology information from ONOS" )
2111 startTime = time.time()
2112 # Give time for Gossip to work
2113 while topoResult == main.FALSE and elapsed < 60:
Jon Hall96091e62015-09-21 17:34:17 -07002114 devicesResults = main.TRUE
2115 linksResults = main.TRUE
2116 hostsResults = main.TRUE
2117 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002118 count += 1
2119 cliStart = time.time()
2120 devices = []
2121 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002122 for i in range( main.numCtrls ):
2123 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002124 name="devices-" + str( i ),
2125 args=[ ] )
2126 threads.append( t )
2127 t.start()
2128
2129 for t in threads:
2130 t.join()
2131 devices.append( t.result )
2132 hosts = []
2133 ipResult = main.TRUE
2134 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002135 for i in range( main.numCtrls ):
2136 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07002137 name="hosts-" + str( i ),
2138 args=[ ] )
2139 threads.append( t )
2140 t.start()
2141
2142 for t in threads:
2143 t.join()
2144 try:
2145 hosts.append( json.loads( t.result ) )
2146 except ( ValueError, TypeError ):
2147 main.log.exception( "Error parsing hosts results" )
2148 main.log.error( repr( t.result ) )
2149 for controller in range( 0, len( hosts ) ):
2150 controllerStr = str( controller + 1 )
2151 for host in hosts[ controller ]:
2152 if host is None or host.get( 'ipAddresses', [] ) == []:
2153 main.log.error(
2154 "DEBUG:Error with host ipAddresses on controller" +
2155 controllerStr + ": " + str( host ) )
2156 ipResult = main.FALSE
2157 ports = []
2158 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002159 for i in range( main.numCtrls ):
2160 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002161 name="ports-" + str( i ),
2162 args=[ ] )
2163 threads.append( t )
2164 t.start()
2165
2166 for t in threads:
2167 t.join()
2168 ports.append( t.result )
2169 links = []
2170 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002171 for i in range( main.numCtrls ):
2172 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002173 name="links-" + str( i ),
2174 args=[ ] )
2175 threads.append( t )
2176 t.start()
2177
2178 for t in threads:
2179 t.join()
2180 links.append( t.result )
2181 clusters = []
2182 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002183 for i in range( main.numCtrls ):
2184 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002185 name="clusters-" + str( i ),
2186 args=[ ] )
2187 threads.append( t )
2188 t.start()
2189
2190 for t in threads:
2191 t.join()
2192 clusters.append( t.result )
2193
2194 elapsed = time.time() - startTime
2195 cliTime = time.time() - cliStart
2196 print "Elapsed time: " + str( elapsed )
2197 print "CLI time: " + str( cliTime )
2198
2199 mnSwitches = main.Mininet1.getSwitches()
2200 mnLinks = main.Mininet1.getLinks()
2201 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002202 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002203 controllerStr = str( controller + 1 )
2204 if devices[ controller ] and ports[ controller ] and\
2205 "Error" not in devices[ controller ] and\
2206 "Error" not in ports[ controller ]:
2207
2208 currentDevicesResult = main.Mininet1.compareSwitches(
2209 mnSwitches,
2210 json.loads( devices[ controller ] ),
2211 json.loads( ports[ controller ] ) )
2212 else:
2213 currentDevicesResult = main.FALSE
2214 utilities.assert_equals( expect=main.TRUE,
2215 actual=currentDevicesResult,
2216 onpass="ONOS" + controllerStr +
2217 " Switches view is correct",
2218 onfail="ONOS" + controllerStr +
2219 " Switches view is incorrect" )
2220
2221 if links[ controller ] and "Error" not in links[ controller ]:
2222 currentLinksResult = main.Mininet1.compareLinks(
2223 mnSwitches, mnLinks,
2224 json.loads( links[ controller ] ) )
2225 else:
2226 currentLinksResult = main.FALSE
2227 utilities.assert_equals( expect=main.TRUE,
2228 actual=currentLinksResult,
2229 onpass="ONOS" + controllerStr +
2230 " links view is correct",
2231 onfail="ONOS" + controllerStr +
2232 " links view is incorrect" )
2233
2234 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2235 currentHostsResult = main.Mininet1.compareHosts(
2236 mnHosts,
2237 hosts[ controller ] )
2238 else:
2239 currentHostsResult = main.FALSE
2240 utilities.assert_equals( expect=main.TRUE,
2241 actual=currentHostsResult,
2242 onpass="ONOS" + controllerStr +
2243 " hosts exist in Mininet",
2244 onfail="ONOS" + controllerStr +
2245 " hosts don't match Mininet" )
2246 # CHECKING HOST ATTACHMENT POINTS
2247 hostAttachment = True
2248 zeroHosts = False
2249 # FIXME: topo-HA/obelisk specific mappings:
2250 # key is mac and value is dpid
2251 mappings = {}
2252 for i in range( 1, 29 ): # hosts 1 through 28
2253 # set up correct variables:
2254 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2255 if i == 1:
2256 deviceId = "1000".zfill(16)
2257 elif i == 2:
2258 deviceId = "2000".zfill(16)
2259 elif i == 3:
2260 deviceId = "3000".zfill(16)
2261 elif i == 4:
2262 deviceId = "3004".zfill(16)
2263 elif i == 5:
2264 deviceId = "5000".zfill(16)
2265 elif i == 6:
2266 deviceId = "6000".zfill(16)
2267 elif i == 7:
2268 deviceId = "6007".zfill(16)
2269 elif i >= 8 and i <= 17:
2270 dpid = '3' + str( i ).zfill( 3 )
2271 deviceId = dpid.zfill(16)
2272 elif i >= 18 and i <= 27:
2273 dpid = '6' + str( i ).zfill( 3 )
2274 deviceId = dpid.zfill(16)
2275 elif i == 28:
2276 deviceId = "2800".zfill(16)
2277 mappings[ macId ] = deviceId
2278 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2279 if hosts[ controller ] == []:
2280 main.log.warn( "There are no hosts discovered" )
2281 zeroHosts = True
2282 else:
2283 for host in hosts[ controller ]:
2284 mac = None
2285 location = None
2286 device = None
2287 port = None
2288 try:
2289 mac = host.get( 'mac' )
2290 assert mac, "mac field could not be found for this host object"
2291
2292 location = host.get( 'location' )
2293 assert location, "location field could not be found for this host object"
2294
2295 # Trim the protocol identifier off deviceId
2296 device = str( location.get( 'elementId' ) ).split(':')[1]
2297 assert device, "elementId field could not be found for this host location object"
2298
2299 port = location.get( 'port' )
2300 assert port, "port field could not be found for this host location object"
2301
2302 # Now check if this matches where they should be
2303 if mac and device and port:
2304 if str( port ) != "1":
2305 main.log.error( "The attachment port is incorrect for " +
2306 "host " + str( mac ) +
2307 ". Expected: 1 Actual: " + str( port) )
2308 hostAttachment = False
2309 if device != mappings[ str( mac ) ]:
2310 main.log.error( "The attachment device is incorrect for " +
2311 "host " + str( mac ) +
2312 ". Expected: " + mappings[ str( mac ) ] +
2313 " Actual: " + device )
2314 hostAttachment = False
2315 else:
2316 hostAttachment = False
2317 except AssertionError:
2318 main.log.exception( "Json object not as expected" )
2319 main.log.error( repr( host ) )
2320 hostAttachment = False
2321 else:
2322 main.log.error( "No hosts json output or \"Error\"" +
2323 " in output. hosts = " +
2324 repr( hosts[ controller ] ) )
2325 if zeroHosts is False:
2326 hostAttachment = True
2327
2328 # END CHECKING HOST ATTACHMENT POINTS
2329 devicesResults = devicesResults and currentDevicesResult
2330 linksResults = linksResults and currentLinksResult
2331 hostsResults = hostsResults and currentHostsResult
2332 hostAttachmentResults = hostAttachmentResults and\
2333 hostAttachment
2334
2335 # Compare json objects for hosts and dataplane clusters
2336
2337 # hosts
2338 main.step( "Hosts view is consistent across all ONOS nodes" )
2339 consistentHostsResult = main.TRUE
2340 for controller in range( len( hosts ) ):
2341 controllerStr = str( controller + 1 )
2342 if "Error" not in hosts[ controller ]:
2343 if hosts[ controller ] == hosts[ 0 ]:
2344 continue
2345 else: # hosts not consistent
2346 main.log.error( "hosts from ONOS" + controllerStr +
2347 " is inconsistent with ONOS1" )
2348 main.log.warn( repr( hosts[ controller ] ) )
2349 consistentHostsResult = main.FALSE
2350
2351 else:
2352 main.log.error( "Error in getting ONOS hosts from ONOS" +
2353 controllerStr )
2354 consistentHostsResult = main.FALSE
2355 main.log.warn( "ONOS" + controllerStr +
2356 " hosts response: " +
2357 repr( hosts[ controller ] ) )
2358 utilities.assert_equals(
2359 expect=main.TRUE,
2360 actual=consistentHostsResult,
2361 onpass="Hosts view is consistent across all ONOS nodes",
2362 onfail="ONOS nodes have different views of hosts" )
2363
2364 main.step( "Hosts information is correct" )
2365 hostsResults = hostsResults and ipResult
2366 utilities.assert_equals(
2367 expect=main.TRUE,
2368 actual=hostsResults,
2369 onpass="Host information is correct",
2370 onfail="Host information is incorrect" )
2371
2372 main.step( "Host attachment points to the network" )
2373 utilities.assert_equals(
2374 expect=True,
2375 actual=hostAttachmentResults,
2376 onpass="Hosts are correctly attached to the network",
2377 onfail="ONOS did not correctly attach hosts to the network" )
2378
2379 # Strongly connected clusters of devices
2380 main.step( "Clusters view is consistent across all ONOS nodes" )
2381 consistentClustersResult = main.TRUE
2382 for controller in range( len( clusters ) ):
2383 controllerStr = str( controller + 1 )
2384 if "Error" not in clusters[ controller ]:
2385 if clusters[ controller ] == clusters[ 0 ]:
2386 continue
2387 else: # clusters not consistent
2388 main.log.error( "clusters from ONOS" +
2389 controllerStr +
2390 " is inconsistent with ONOS1" )
2391 consistentClustersResult = main.FALSE
2392
2393 else:
2394 main.log.error( "Error in getting dataplane clusters " +
2395 "from ONOS" + controllerStr )
2396 consistentClustersResult = main.FALSE
2397 main.log.warn( "ONOS" + controllerStr +
2398 " clusters response: " +
2399 repr( clusters[ controller ] ) )
2400 utilities.assert_equals(
2401 expect=main.TRUE,
2402 actual=consistentClustersResult,
2403 onpass="Clusters view is consistent across all ONOS nodes",
2404 onfail="ONOS nodes have different views of clusters" )
2405
2406 main.step( "There is only one SCC" )
2407 # there should always only be one cluster
2408 try:
2409 numClusters = len( json.loads( clusters[ 0 ] ) )
2410 except ( ValueError, TypeError ):
2411 main.log.exception( "Error parsing clusters[0]: " +
2412 repr( clusters[0] ) )
2413 clusterResults = main.FALSE
2414 if numClusters == 1:
2415 clusterResults = main.TRUE
2416 utilities.assert_equals(
2417 expect=1,
2418 actual=numClusters,
2419 onpass="ONOS shows 1 SCC",
2420 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2421
2422 topoResult = ( devicesResults and linksResults
2423 and hostsResults and consistentHostsResult
2424 and consistentClustersResult and clusterResults
2425 and ipResult and hostAttachmentResults )
2426
2427 topoResult = topoResult and int( count <= 2 )
2428 note = "note it takes about " + str( int( cliTime ) ) + \
2429 " seconds for the test to make all the cli calls to fetch " +\
2430 "the topology from each ONOS instance"
2431 main.log.info(
2432 "Very crass estimate for topology discovery/convergence( " +
2433 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2434 str( count ) + " tries" )
2435
2436 main.step( "Device information is correct" )
2437 utilities.assert_equals(
2438 expect=main.TRUE,
2439 actual=devicesResults,
2440 onpass="Device information is correct",
2441 onfail="Device information is incorrect" )
2442
2443 main.step( "Links are correct" )
2444 utilities.assert_equals(
2445 expect=main.TRUE,
2446 actual=linksResults,
2447 onpass="Link are correct",
2448 onfail="Links are incorrect" )
2449
2450 # FIXME: move this to an ONOS state case
2451 main.step( "Checking ONOS nodes" )
2452 nodesOutput = []
2453 nodeResults = main.TRUE
2454 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002455 for i in range( main.numCtrls ):
2456 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002457 name="nodes-" + str( i ),
2458 args=[ ] )
2459 threads.append( t )
2460 t.start()
2461
2462 for t in threads:
2463 t.join()
2464 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002465 ips = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002466 for i in nodesOutput:
2467 try:
2468 current = json.loads( i )
2469 for node in current:
2470 currentResult = main.FALSE
2471 if node['ip'] in ips: # node in nodes() output is in cell
2472 if node['state'] == 'ACTIVE':
2473 currentResult = main.TRUE
2474 else:
2475 main.log.error( "Error in ONOS node availability" )
2476 main.log.error(
2477 json.dumps( current,
2478 sort_keys=True,
2479 indent=4,
2480 separators=( ',', ': ' ) ) )
2481 break
2482 nodeResults = nodeResults and currentResult
2483 except ( ValueError, TypeError ):
2484 main.log.error( "Error parsing nodes output" )
2485 main.log.warn( repr( i ) )
2486 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2487 onpass="Nodes check successful",
2488 onfail="Nodes check NOT successful" )
2489
2490 def CASE9( self, main ):
2491 """
2492 Link s3-s28 down
2493 """
2494 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002495 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002496 assert main, "main not defined"
2497 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002498 assert main.CLIs, "main.CLIs not defined"
2499 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002500 # NOTE: You should probably run a topology check after this
2501
2502 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2503
2504 description = "Turn off a link to ensure that Link Discovery " +\
2505 "is working properly"
2506 main.case( description )
2507
2508 main.step( "Kill Link between s3 and s28" )
2509 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2510 main.log.info( "Waiting " + str( linkSleep ) +
2511 " seconds for link down to be discovered" )
2512 time.sleep( linkSleep )
2513 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2514 onpass="Link down successful",
2515 onfail="Failed to bring link down" )
2516 # TODO do some sort of check here
2517
2518 def CASE10( self, main ):
2519 """
2520 Link s3-s28 up
2521 """
2522 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002523 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002524 assert main, "main not defined"
2525 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002526 assert main.CLIs, "main.CLIs not defined"
2527 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002528 # NOTE: You should probably run a topology check after this
2529
2530 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2531
2532 description = "Restore a link to ensure that Link Discovery is " + \
2533 "working properly"
2534 main.case( description )
2535
2536 main.step( "Bring link between s3 and s28 back up" )
2537 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2538 main.log.info( "Waiting " + str( linkSleep ) +
2539 " seconds for link up to be discovered" )
2540 time.sleep( linkSleep )
2541 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2542 onpass="Link up successful",
2543 onfail="Failed to bring link up" )
2544 # TODO do some sort of check here
2545
2546 def CASE11( self, main ):
2547 """
2548 Switch Down
2549 """
2550 # NOTE: You should probably run a topology check after this
2551 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002552 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002553 assert main, "main not defined"
2554 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002555 assert main.CLIs, "main.CLIs not defined"
2556 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002557
2558 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2559
2560 description = "Killing a switch to ensure it is discovered correctly"
2561 main.case( description )
2562 switch = main.params[ 'kill' ][ 'switch' ]
2563 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2564
2565 # TODO: Make this switch parameterizable
2566 main.step( "Kill " + switch )
2567 main.log.info( "Deleting " + switch )
2568 main.Mininet1.delSwitch( switch )
2569 main.log.info( "Waiting " + str( switchSleep ) +
2570 " seconds for switch down to be discovered" )
2571 time.sleep( switchSleep )
2572 device = main.ONOScli1.getDevice( dpid=switchDPID )
2573 # Peek at the deleted switch
2574 main.log.warn( str( device ) )
2575 result = main.FALSE
2576 if device and device[ 'available' ] is False:
2577 result = main.TRUE
2578 utilities.assert_equals( expect=main.TRUE, actual=result,
2579 onpass="Kill switch successful",
2580 onfail="Failed to kill switch?" )
2581
2582 def CASE12( self, main ):
2583 """
2584 Switch Up
2585 """
2586 # NOTE: You should probably run a topology check after this
2587 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002588 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002589 assert main, "main not defined"
2590 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002591 assert main.CLIs, "main.CLIs not defined"
2592 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002593 assert ONOS1Port, "ONOS1Port not defined"
2594 assert ONOS2Port, "ONOS2Port not defined"
2595 assert ONOS3Port, "ONOS3Port not defined"
2596 assert ONOS4Port, "ONOS4Port not defined"
2597 assert ONOS5Port, "ONOS5Port not defined"
2598 assert ONOS6Port, "ONOS6Port not defined"
2599 assert ONOS7Port, "ONOS7Port not defined"
2600
2601 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2602 switch = main.params[ 'kill' ][ 'switch' ]
2603 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2604 links = main.params[ 'kill' ][ 'links' ].split()
2605 description = "Adding a switch to ensure it is discovered correctly"
2606 main.case( description )
2607
2608 main.step( "Add back " + switch )
2609 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2610 for peer in links:
2611 main.Mininet1.addLink( switch, peer )
2612 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002613 for i in range( main.numCtrls ):
2614 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002615 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2616 main.log.info( "Waiting " + str( switchSleep ) +
2617 " seconds for switch up to be discovered" )
2618 time.sleep( switchSleep )
2619 device = main.ONOScli1.getDevice( dpid=switchDPID )
2620 # Peek at the deleted switch
2621 main.log.warn( str( device ) )
2622 result = main.FALSE
2623 if device and device[ 'available' ]:
2624 result = main.TRUE
2625 utilities.assert_equals( expect=main.TRUE, actual=result,
2626 onpass="add switch successful",
2627 onfail="Failed to add switch?" )
2628
2629 def CASE13( self, main ):
2630 """
2631 Clean up
2632 """
2633 import os
2634 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002635 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002636 assert main, "main not defined"
2637 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002638 assert main.CLIs, "main.CLIs not defined"
2639 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002640
2641 # printing colors to terminal
2642 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2643 'blue': '\033[94m', 'green': '\033[92m',
2644 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2645 main.case( "Test Cleanup" )
2646 main.step( "Killing tcpdumps" )
2647 main.Mininet2.stopTcpdump()
2648
2649 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002650 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002651 main.step( "Copying MN pcap and ONOS log files to test station" )
2652 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2653 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002654 # NOTE: MN Pcap file is being saved to logdir.
2655 # We scp this file as MN and TestON aren't necessarily the same vm
2656
2657 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002658 # TODO: Load these from params
2659 # NOTE: must end in /
2660 logFolder = "/opt/onos/log/"
2661 logFiles = [ "karaf.log", "karaf.log.1" ]
2662 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002663 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002664 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002665 dstName = main.logdir + "/" + node.name + "-" + f
2666 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2667 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002668 # std*.log's
2669 # NOTE: must end in /
2670 logFolder = "/opt/onos/var/"
2671 logFiles = [ "stderr.log", "stdout.log" ]
2672 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002673 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002674 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002675 dstName = main.logdir + "/" + node.name + "-" + f
2676 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2677 logFolder + f, dstName )
2678 else:
2679 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002680
2681 main.step( "Stopping Mininet" )
2682 mnResult = main.Mininet1.stopNet()
2683 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2684 onpass="Mininet stopped",
2685 onfail="MN cleanup NOT successful" )
2686
2687 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002688 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002689 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2690 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002691
2692 try:
2693 timerLog = open( main.logdir + "/Timers.csv", 'w')
2694 # Overwrite with empty line and close
2695 labels = "Gossip Intents, Restart"
2696 data = str( gossipTime ) + ", " + str( main.restartTime )
2697 timerLog.write( labels + "\n" + data )
2698 timerLog.close()
2699 except NameError, e:
2700 main.log.exception(e)
2701
2702 def CASE14( self, main ):
2703 """
2704 start election app on all onos nodes
2705 """
Jon Halle1a3b752015-07-22 13:02:46 -07002706 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002707 assert main, "main not defined"
2708 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002709 assert main.CLIs, "main.CLIs not defined"
2710 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002711
2712 main.case("Start Leadership Election app")
2713 main.step( "Install leadership election app" )
2714 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2715 utilities.assert_equals(
2716 expect=main.TRUE,
2717 actual=appResult,
2718 onpass="Election app installed",
2719 onfail="Something went wrong with installing Leadership election" )
2720
2721 main.step( "Run for election on each node" )
2722 leaderResult = main.TRUE
2723 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002724 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002725 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002726 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002727 leader = cli.electionTestLeader()
2728 if leader is None or leader == main.FALSE:
2729 main.log.error( cli.name + ": Leader for the election app " +
2730 "should be an ONOS node, instead got '" +
2731 str( leader ) + "'" )
2732 leaderResult = main.FALSE
2733 leaders.append( leader )
2734 utilities.assert_equals(
2735 expect=main.TRUE,
2736 actual=leaderResult,
2737 onpass="Successfully ran for leadership",
2738 onfail="Failed to run for leadership" )
2739
2740 main.step( "Check that each node shows the same leader" )
2741 sameLeader = main.TRUE
2742 if len( set( leaders ) ) != 1:
2743 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002744 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002745 str( leaders ) )
2746 utilities.assert_equals(
2747 expect=main.TRUE,
2748 actual=sameLeader,
2749 onpass="Leadership is consistent for the election topic",
2750 onfail="Nodes have different leaders" )
2751
2752 def CASE15( self, main ):
2753 """
2754 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002755 15.1 Run election on each node
2756 15.2 Check that each node has the same leaders and candidates
2757 15.3 Find current leader and withdraw
2758 15.4 Check that a new node was elected leader
2759 15.5 Check that that new leader was the candidate of old leader
2760 15.6 Run for election on old leader
2761 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2762 15.8 Make sure that the old leader was added to the candidate list
2763
2764 old and new variable prefixes refer to data from before vs after
2765 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002766 """
2767 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002768 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002769 assert main, "main not defined"
2770 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002771 assert main.CLIs, "main.CLIs not defined"
2772 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002773
Jon Hall5cf14d52015-07-16 12:15:19 -07002774 description = "Check that Leadership Election is still functional"
2775 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07002776 # NOTE: Need to re-run since being a canidate is not persistant
2777 # TODO: add check for "Command not found:" in the driver, this
2778 # means the election test app isn't loaded
Jon Hall5cf14d52015-07-16 12:15:19 -07002779
acsmars71adceb2015-08-31 15:09:26 -07002780 oldLeaders = [] # leaders by node before withdrawl from candidates
2781 newLeaders = [] # leaders by node after withdrawl from candidates
2782 oldAllCandidates = [] # list of lists of each nodes' candidates before
2783 newAllCandidates = [] # list of lists of each nodes' candidates after
2784 oldCandidates = [] # list of candidates from node 0 before withdrawl
2785 newCandidates = [] # list of candidates from node 0 after withdrawl
2786 oldLeader = '' # the old leader from oldLeaders, None if not same
2787 newLeader = '' # the new leaders fron newLoeaders, None if not same
2788 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2789 expectNoLeader = False # True when there is only one leader
2790 if main.numCtrls == 1:
2791 expectNoLeader = True
2792
2793 main.step( "Run for election on each node" )
2794 electionResult = main.TRUE
2795
2796 for cli in main.CLIs: # run test election on each node
2797 if cli.electionTestRun() == main.FALSE:
2798 electionResult = main.FALSE
2799
Jon Hall5cf14d52015-07-16 12:15:19 -07002800 utilities.assert_equals(
2801 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002802 actual=electionResult,
2803 onpass="All nodes successfully ran for leadership",
2804 onfail="At least one node failed to run for leadership" )
2805
acsmars3a72bde2015-09-02 14:16:22 -07002806 if electionResult == main.FALSE:
2807 main.log.error(
2808 "Skipping Test Case because Election Test App isn't loaded" )
2809 main.skipCase()
2810
acsmars71adceb2015-08-31 15:09:26 -07002811 main.step( "Check that each node shows the same leader and candidates" )
2812 sameResult = main.TRUE
2813 failMessage = "Nodes have different leaders"
2814 for cli in main.CLIs:
2815 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2816 oldAllCandidates.append( node )
2817 oldLeaders.append( node[ 0 ] )
2818 oldCandidates = oldAllCandidates[ 0 ]
2819
2820 # Check that each node has the same leader. Defines oldLeader
2821 if len( set( oldLeaders ) ) != 1:
2822 sameResult = main.FALSE
2823 main.log.error( "More than one leader present:" + str( oldLeaders ) )
2824 oldLeader = None
2825 else:
2826 oldLeader = oldLeaders[ 0 ]
2827
2828 # Check that each node's candidate list is the same
2829 for candidates in oldAllCandidates:
2830 if set( candidates ) != set( oldCandidates ):
2831 sameResult = main.FALSE
2832 failMessage += "and candidates"
2833
2834 utilities.assert_equals(
2835 expect=main.TRUE,
2836 actual=sameResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002837 onpass="Leadership is consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002838 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002839
2840 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002841 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002842 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002843 if oldLeader is None:
2844 main.log.error( "Leadership isn't consistent." )
2845 withdrawResult = main.FALSE
2846 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002847 for i in range( len( main.CLIs ) ):
acsmars71adceb2015-08-31 15:09:26 -07002848 if oldLeader == main.nodes[ i ].ip_address:
2849 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002850 break
2851 else: # FOR/ELSE statement
2852 main.log.error( "Leader election, could not find current leader" )
2853 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002854 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002855 utilities.assert_equals(
2856 expect=main.TRUE,
2857 actual=withdrawResult,
2858 onpass="Node was withdrawn from election",
2859 onfail="Node was not withdrawn from election" )
2860
acsmars71adceb2015-08-31 15:09:26 -07002861 main.step( "Check that a new node was elected leader" )
2862
Jon Hall5cf14d52015-07-16 12:15:19 -07002863 # FIXME: use threads
acsmars71adceb2015-08-31 15:09:26 -07002864 newLeaderResult = main.TRUE
2865 failMessage = "Nodes have different leaders"
2866
2867 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002868 for cli in main.CLIs:
acsmars71adceb2015-08-31 15:09:26 -07002869 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2870 # elections might no have finished yet
2871 if node[ 0 ] == 'none' and not expectNoLeader:
2872 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2873 "sure elections are complete." )
2874 time.sleep(5)
2875 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2876 # election still isn't done or there is a problem
2877 if node[ 0 ] == 'none':
2878 main.log.error( "No leader was elected on at least 1 node" )
2879 newLeaderResult = main.FALSE
2880 newAllCandidates.append( node )
2881 newLeaders.append( node[ 0 ] )
2882 newCandidates = newAllCandidates[ 0 ]
2883
2884 # Check that each node has the same leader. Defines newLeader
2885 if len( set( newLeaders ) ) != 1:
2886 newLeaderResult = main.FALSE
2887 main.log.error( "Nodes have different leaders: " +
2888 str( newLeaders ) )
2889 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002890 else:
acsmars71adceb2015-08-31 15:09:26 -07002891 newLeader = newLeaders[ 0 ]
2892
2893 # Check that each node's candidate list is the same
2894 for candidates in newAllCandidates:
2895 if set( candidates ) != set( newCandidates ):
2896 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002897 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002898
2899 # Check that the new leader is not the older leader, which was withdrawn
2900 if newLeader == oldLeader:
2901 newLeaderResult = main.FALSE
2902 main.log.error( "All nodes still see old leader: " + oldLeader +
2903 " as the current leader" )
2904
Jon Hall5cf14d52015-07-16 12:15:19 -07002905 utilities.assert_equals(
2906 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002907 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002908 onpass="Leadership election passed",
2909 onfail="Something went wrong with Leadership election" )
2910
acsmars71adceb2015-08-31 15:09:26 -07002911 main.step( "Check that that new leader was the candidate of old leader")
2912 # candidates[ 2 ] should be come the top candidate after withdrawl
2913 correctCandidateResult = main.TRUE
2914 if expectNoLeader:
2915 if newLeader == 'none':
2916 main.log.info( "No leader expected. None found. Pass" )
2917 correctCandidateResult = main.TRUE
2918 else:
2919 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2920 correctCandidateResult = main.FALSE
2921 elif newLeader != oldCandidates[ 2 ]:
2922 correctCandidateResult = main.FALSE
2923 main.log.error( "Candidate " + newLeader + " was elected. " +
2924 oldCandidates[ 2 ] + " should have had priority." )
2925
2926 utilities.assert_equals(
2927 expect=main.TRUE,
2928 actual=correctCandidateResult,
2929 onpass="Correct Candidate Elected",
2930 onfail="Incorrect Candidate Elected" )
2931
Jon Hall5cf14d52015-07-16 12:15:19 -07002932 main.step( "Run for election on old leader( just so everyone " +
2933 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002934 if oldLeaderCLI is not None:
2935 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002936 else:
acsmars71adceb2015-08-31 15:09:26 -07002937 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002938 runResult = main.FALSE
2939 utilities.assert_equals(
2940 expect=main.TRUE,
2941 actual=runResult,
2942 onpass="App re-ran for election",
2943 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002944 main.step(
2945 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002946 # verify leader didn't just change
acsmars71adceb2015-08-31 15:09:26 -07002947 positionResult = main.TRUE
2948 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
2949
2950 # Reset and reuse the new candidate and leaders lists
2951 newAllCandidates = []
2952 newCandidates = []
2953 newLeaders = []
2954 for cli in main.CLIs:
2955 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2956 if oldLeader not in node: # election might no have finished yet
2957 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2958 "be sure elections are complete" )
2959 time.sleep(5)
2960 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2961 if oldLeader not in node: # election still isn't done, errors
2962 main.log.error(
2963 "Old leader was not elected on at least one node" )
2964 positionResult = main.FALSE
2965 newAllCandidates.append( node )
2966 newLeaders.append( node[ 0 ] )
2967 newCandidates = newAllCandidates[ 0 ]
2968
2969 # Check that each node has the same leader. Defines newLeader
2970 if len( set( newLeaders ) ) != 1:
2971 positionResult = main.FALSE
2972 main.log.error( "Nodes have different leaders: " +
2973 str( newLeaders ) )
2974 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002975 else:
acsmars71adceb2015-08-31 15:09:26 -07002976 newLeader = newLeaders[ 0 ]
2977
2978 # Check that each node's candidate list is the same
2979 for candidates in newAllCandidates:
2980 if set( candidates ) != set( newCandidates ):
2981 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002982 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002983
2984 # Check that the re-elected node is last on the candidate List
2985 if oldLeader != newCandidates[ -1 ]:
2986 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
2987 str( newCandidates ) )
2988 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002989
2990 utilities.assert_equals(
2991 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002992 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002993 onpass="Old leader successfully re-ran for election",
2994 onfail="Something went wrong with Leadership election after " +
2995 "the old leader re-ran for election" )
2996
2997 def CASE16( self, main ):
2998 """
2999 Install Distributed Primitives app
3000 """
3001 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003002 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003003 assert main, "main not defined"
3004 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003005 assert main.CLIs, "main.CLIs not defined"
3006 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003007
3008 # Variables for the distributed primitives tests
3009 global pCounterName
3010 global iCounterName
3011 global pCounterValue
3012 global iCounterValue
3013 global onosSet
3014 global onosSetName
3015 pCounterName = "TestON-Partitions"
3016 iCounterName = "TestON-inMemory"
3017 pCounterValue = 0
3018 iCounterValue = 0
3019 onosSet = set([])
3020 onosSetName = "TestON-set"
3021
3022 description = "Install Primitives app"
3023 main.case( description )
3024 main.step( "Install Primitives app" )
3025 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07003026 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003027 utilities.assert_equals( expect=main.TRUE,
3028 actual=appResults,
3029 onpass="Primitives app activated",
3030 onfail="Primitives app not activated" )
3031 time.sleep( 5 ) # To allow all nodes to activate
3032
3033 def CASE17( self, main ):
3034 """
3035 Check for basic functionality with distributed primitives
3036 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003037 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003038 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003039 assert main, "main not defined"
3040 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003041 assert main.CLIs, "main.CLIs not defined"
3042 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003043 assert pCounterName, "pCounterName not defined"
3044 assert iCounterName, "iCounterName not defined"
3045 assert onosSetName, "onosSetName not defined"
3046 # NOTE: assert fails if value is 0/None/Empty/False
3047 try:
3048 pCounterValue
3049 except NameError:
3050 main.log.error( "pCounterValue not defined, setting to 0" )
3051 pCounterValue = 0
3052 try:
3053 iCounterValue
3054 except NameError:
3055 main.log.error( "iCounterValue not defined, setting to 0" )
3056 iCounterValue = 0
3057 try:
3058 onosSet
3059 except NameError:
3060 main.log.error( "onosSet not defined, setting to empty Set" )
3061 onosSet = set([])
3062 # Variables for the distributed primitives tests. These are local only
3063 addValue = "a"
3064 addAllValue = "a b c d e f"
3065 retainValue = "c d e f"
3066
3067 description = "Check for basic functionality with distributed " +\
3068 "primitives"
3069 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003070 main.caseExplanation = "Test the methods of the distributed " +\
3071 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003072 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003073 # Partitioned counters
3074 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003075 pCounters = []
3076 threads = []
3077 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003078 for i in range( main.numCtrls ):
3079 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3080 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003081 args=[ pCounterName ] )
3082 pCounterValue += 1
3083 addedPValues.append( pCounterValue )
3084 threads.append( t )
3085 t.start()
3086
3087 for t in threads:
3088 t.join()
3089 pCounters.append( t.result )
3090 # Check that counter incremented numController times
3091 pCounterResults = True
3092 for i in addedPValues:
3093 tmpResult = i in pCounters
3094 pCounterResults = pCounterResults and tmpResult
3095 if not tmpResult:
3096 main.log.error( str( i ) + " is not in partitioned "
3097 "counter incremented results" )
3098 utilities.assert_equals( expect=True,
3099 actual=pCounterResults,
3100 onpass="Default counter incremented",
3101 onfail="Error incrementing default" +
3102 " counter" )
3103
Jon Halle1a3b752015-07-22 13:02:46 -07003104 main.step( "Get then Increment a default counter on each node" )
3105 pCounters = []
3106 threads = []
3107 addedPValues = []
3108 for i in range( main.numCtrls ):
3109 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3110 name="counterGetAndAdd-" + str( i ),
3111 args=[ pCounterName ] )
3112 addedPValues.append( pCounterValue )
3113 pCounterValue += 1
3114 threads.append( t )
3115 t.start()
3116
3117 for t in threads:
3118 t.join()
3119 pCounters.append( t.result )
3120 # Check that counter incremented numController times
3121 pCounterResults = True
3122 for i in addedPValues:
3123 tmpResult = i in pCounters
3124 pCounterResults = pCounterResults and tmpResult
3125 if not tmpResult:
3126 main.log.error( str( i ) + " is not in partitioned "
3127 "counter incremented results" )
3128 utilities.assert_equals( expect=True,
3129 actual=pCounterResults,
3130 onpass="Default counter incremented",
3131 onfail="Error incrementing default" +
3132 " counter" )
3133
3134 main.step( "Counters we added have the correct values" )
3135 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3136 utilities.assert_equals( expect=main.TRUE,
3137 actual=incrementCheck,
3138 onpass="Added counters are correct",
3139 onfail="Added counters are incorrect" )
3140
3141 main.step( "Add -8 to then get a default counter on each node" )
3142 pCounters = []
3143 threads = []
3144 addedPValues = []
3145 for i in range( main.numCtrls ):
3146 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3147 name="counterIncrement-" + str( i ),
3148 args=[ pCounterName ],
3149 kwargs={ "delta": -8 } )
3150 pCounterValue += -8
3151 addedPValues.append( pCounterValue )
3152 threads.append( t )
3153 t.start()
3154
3155 for t in threads:
3156 t.join()
3157 pCounters.append( t.result )
3158 # Check that counter incremented numController times
3159 pCounterResults = True
3160 for i in addedPValues:
3161 tmpResult = i in pCounters
3162 pCounterResults = pCounterResults and tmpResult
3163 if not tmpResult:
3164 main.log.error( str( i ) + " is not in partitioned "
3165 "counter incremented results" )
3166 utilities.assert_equals( expect=True,
3167 actual=pCounterResults,
3168 onpass="Default counter incremented",
3169 onfail="Error incrementing default" +
3170 " counter" )
3171
3172 main.step( "Add 5 to then get a default counter on each node" )
3173 pCounters = []
3174 threads = []
3175 addedPValues = []
3176 for i in range( main.numCtrls ):
3177 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3178 name="counterIncrement-" + str( i ),
3179 args=[ pCounterName ],
3180 kwargs={ "delta": 5 } )
3181 pCounterValue += 5
3182 addedPValues.append( pCounterValue )
3183 threads.append( t )
3184 t.start()
3185
3186 for t in threads:
3187 t.join()
3188 pCounters.append( t.result )
3189 # Check that counter incremented numController times
3190 pCounterResults = True
3191 for i in addedPValues:
3192 tmpResult = i in pCounters
3193 pCounterResults = pCounterResults and tmpResult
3194 if not tmpResult:
3195 main.log.error( str( i ) + " is not in partitioned "
3196 "counter incremented results" )
3197 utilities.assert_equals( expect=True,
3198 actual=pCounterResults,
3199 onpass="Default counter incremented",
3200 onfail="Error incrementing default" +
3201 " counter" )
3202
3203 main.step( "Get then add 5 to a default counter on each node" )
3204 pCounters = []
3205 threads = []
3206 addedPValues = []
3207 for i in range( main.numCtrls ):
3208 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3209 name="counterIncrement-" + str( i ),
3210 args=[ pCounterName ],
3211 kwargs={ "delta": 5 } )
3212 addedPValues.append( pCounterValue )
3213 pCounterValue += 5
3214 threads.append( t )
3215 t.start()
3216
3217 for t in threads:
3218 t.join()
3219 pCounters.append( t.result )
3220 # Check that counter incremented numController times
3221 pCounterResults = True
3222 for i in addedPValues:
3223 tmpResult = i in pCounters
3224 pCounterResults = pCounterResults and tmpResult
3225 if not tmpResult:
3226 main.log.error( str( i ) + " is not in partitioned "
3227 "counter incremented results" )
3228 utilities.assert_equals( expect=True,
3229 actual=pCounterResults,
3230 onpass="Default counter incremented",
3231 onfail="Error incrementing default" +
3232 " counter" )
3233
3234 main.step( "Counters we added have the correct values" )
3235 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3236 utilities.assert_equals( expect=main.TRUE,
3237 actual=incrementCheck,
3238 onpass="Added counters are correct",
3239 onfail="Added counters are incorrect" )
3240
3241 # In-Memory counters
3242 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003243 iCounters = []
3244 addedIValues = []
3245 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003246 for i in range( main.numCtrls ):
3247 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003248 name="icounterIncrement-" + str( i ),
3249 args=[ iCounterName ],
3250 kwargs={ "inMemory": True } )
3251 iCounterValue += 1
3252 addedIValues.append( iCounterValue )
3253 threads.append( t )
3254 t.start()
3255
3256 for t in threads:
3257 t.join()
3258 iCounters.append( t.result )
3259 # Check that counter incremented numController times
3260 iCounterResults = True
3261 for i in addedIValues:
3262 tmpResult = i in iCounters
3263 iCounterResults = iCounterResults and tmpResult
3264 if not tmpResult:
3265 main.log.error( str( i ) + " is not in the in-memory "
3266 "counter incremented results" )
3267 utilities.assert_equals( expect=True,
3268 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003269 onpass="In-memory counter incremented",
3270 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003271 " counter" )
3272
Jon Halle1a3b752015-07-22 13:02:46 -07003273 main.step( "Get then Increment a in-memory counter on each node" )
3274 iCounters = []
3275 threads = []
3276 addedIValues = []
3277 for i in range( main.numCtrls ):
3278 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3279 name="counterGetAndAdd-" + str( i ),
3280 args=[ iCounterName ],
3281 kwargs={ "inMemory": True } )
3282 addedIValues.append( iCounterValue )
3283 iCounterValue += 1
3284 threads.append( t )
3285 t.start()
3286
3287 for t in threads:
3288 t.join()
3289 iCounters.append( t.result )
3290 # Check that counter incremented numController times
3291 iCounterResults = True
3292 for i in addedIValues:
3293 tmpResult = i in iCounters
3294 iCounterResults = iCounterResults and tmpResult
3295 if not tmpResult:
3296 main.log.error( str( i ) + " is not in in-memory "
3297 "counter incremented results" )
3298 utilities.assert_equals( expect=True,
3299 actual=iCounterResults,
3300 onpass="In-memory counter incremented",
3301 onfail="Error incrementing in-memory" +
3302 " counter" )
3303
3304 main.step( "Counters we added have the correct values" )
3305 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3306 utilities.assert_equals( expect=main.TRUE,
3307 actual=incrementCheck,
3308 onpass="Added counters are correct",
3309 onfail="Added counters are incorrect" )
3310
3311 main.step( "Add -8 to then get a in-memory counter on each node" )
3312 iCounters = []
3313 threads = []
3314 addedIValues = []
3315 for i in range( main.numCtrls ):
3316 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3317 name="counterIncrement-" + str( i ),
3318 args=[ iCounterName ],
3319 kwargs={ "delta": -8, "inMemory": True } )
3320 iCounterValue += -8
3321 addedIValues.append( iCounterValue )
3322 threads.append( t )
3323 t.start()
3324
3325 for t in threads:
3326 t.join()
3327 iCounters.append( t.result )
3328 # Check that counter incremented numController times
3329 iCounterResults = True
3330 for i in addedIValues:
3331 tmpResult = i in iCounters
3332 iCounterResults = iCounterResults and tmpResult
3333 if not tmpResult:
3334 main.log.error( str( i ) + " is not in in-memory "
3335 "counter incremented results" )
3336 utilities.assert_equals( expect=True,
3337 actual=pCounterResults,
3338 onpass="In-memory counter incremented",
3339 onfail="Error incrementing in-memory" +
3340 " counter" )
3341
3342 main.step( "Add 5 to then get a in-memory counter on each node" )
3343 iCounters = []
3344 threads = []
3345 addedIValues = []
3346 for i in range( main.numCtrls ):
3347 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3348 name="counterIncrement-" + str( i ),
3349 args=[ iCounterName ],
3350 kwargs={ "delta": 5, "inMemory": True } )
3351 iCounterValue += 5
3352 addedIValues.append( iCounterValue )
3353 threads.append( t )
3354 t.start()
3355
3356 for t in threads:
3357 t.join()
3358 iCounters.append( t.result )
3359 # Check that counter incremented numController times
3360 iCounterResults = True
3361 for i in addedIValues:
3362 tmpResult = i in iCounters
3363 iCounterResults = iCounterResults and tmpResult
3364 if not tmpResult:
3365 main.log.error( str( i ) + " is not in in-memory "
3366 "counter incremented results" )
3367 utilities.assert_equals( expect=True,
3368 actual=pCounterResults,
3369 onpass="In-memory counter incremented",
3370 onfail="Error incrementing in-memory" +
3371 " counter" )
3372
3373 main.step( "Get then add 5 to a in-memory counter on each node" )
3374 iCounters = []
3375 threads = []
3376 addedIValues = []
3377 for i in range( main.numCtrls ):
3378 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3379 name="counterIncrement-" + str( i ),
3380 args=[ iCounterName ],
3381 kwargs={ "delta": 5, "inMemory": True } )
3382 addedIValues.append( iCounterValue )
3383 iCounterValue += 5
3384 threads.append( t )
3385 t.start()
3386
3387 for t in threads:
3388 t.join()
3389 iCounters.append( t.result )
3390 # Check that counter incremented numController times
3391 iCounterResults = True
3392 for i in addedIValues:
3393 tmpResult = i in iCounters
3394 iCounterResults = iCounterResults and tmpResult
3395 if not tmpResult:
3396 main.log.error( str( i ) + " is not in in-memory "
3397 "counter incremented results" )
3398 utilities.assert_equals( expect=True,
3399 actual=iCounterResults,
3400 onpass="In-memory counter incremented",
3401 onfail="Error incrementing in-memory" +
3402 " counter" )
3403
3404 main.step( "Counters we added have the correct values" )
3405 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3406 utilities.assert_equals( expect=main.TRUE,
3407 actual=incrementCheck,
3408 onpass="Added counters are correct",
3409 onfail="Added counters are incorrect" )
3410
Jon Hall5cf14d52015-07-16 12:15:19 -07003411 main.step( "Check counters are consistant across nodes" )
3412 onosCounters = []
3413 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003414 for i in range( main.numCtrls ):
3415 t = main.Thread( target=main.CLIs[i].counters,
Jon Hall5cf14d52015-07-16 12:15:19 -07003416 name="counters-" + str( i ) )
3417 threads.append( t )
3418 t.start()
3419 for t in threads:
3420 t.join()
3421 onosCounters.append( t.result )
3422 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3423 if all( tmp ):
3424 main.log.info( "Counters are consistent across all nodes" )
3425 consistentCounterResults = main.TRUE
3426 else:
3427 main.log.error( "Counters are not consistent across all nodes" )
3428 consistentCounterResults = main.FALSE
3429 utilities.assert_equals( expect=main.TRUE,
3430 actual=consistentCounterResults,
3431 onpass="ONOS counters are consistent " +
3432 "across nodes",
3433 onfail="ONOS Counters are inconsistent " +
3434 "across nodes" )
3435
3436 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003437 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3438 incrementCheck = incrementCheck and \
3439 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003440 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003441 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003442 onpass="Added counters are correct",
3443 onfail="Added counters are incorrect" )
Jon Halle1a3b752015-07-22 13:02:46 -07003444
Jon Hall5cf14d52015-07-16 12:15:19 -07003445 # DISTRIBUTED SETS
3446 main.step( "Distributed Set get" )
3447 size = len( onosSet )
3448 getResponses = []
3449 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003450 for i in range( main.numCtrls ):
3451 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003452 name="setTestGet-" + str( i ),
3453 args=[ onosSetName ] )
3454 threads.append( t )
3455 t.start()
3456 for t in threads:
3457 t.join()
3458 getResponses.append( t.result )
3459
3460 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003461 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003462 if isinstance( getResponses[ i ], list):
3463 current = set( getResponses[ i ] )
3464 if len( current ) == len( getResponses[ i ] ):
3465 # no repeats
3466 if onosSet != current:
3467 main.log.error( "ONOS" + str( i + 1 ) +
3468 " has incorrect view" +
3469 " of set " + onosSetName + ":\n" +
3470 str( getResponses[ i ] ) )
3471 main.log.debug( "Expected: " + str( onosSet ) )
3472 main.log.debug( "Actual: " + str( current ) )
3473 getResults = main.FALSE
3474 else:
3475 # error, set is not a set
3476 main.log.error( "ONOS" + str( i + 1 ) +
3477 " has repeat elements in" +
3478 " set " + onosSetName + ":\n" +
3479 str( getResponses[ i ] ) )
3480 getResults = main.FALSE
3481 elif getResponses[ i ] == main.ERROR:
3482 getResults = main.FALSE
3483 utilities.assert_equals( expect=main.TRUE,
3484 actual=getResults,
3485 onpass="Set elements are correct",
3486 onfail="Set elements are incorrect" )
3487
3488 main.step( "Distributed Set size" )
3489 sizeResponses = []
3490 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003491 for i in range( main.numCtrls ):
3492 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003493 name="setTestSize-" + str( i ),
3494 args=[ onosSetName ] )
3495 threads.append( t )
3496 t.start()
3497 for t in threads:
3498 t.join()
3499 sizeResponses.append( t.result )
3500
3501 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003502 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003503 if size != sizeResponses[ i ]:
3504 sizeResults = main.FALSE
3505 main.log.error( "ONOS" + str( i + 1 ) +
3506 " expected a size of " + str( size ) +
3507 " for set " + onosSetName +
3508 " but got " + str( sizeResponses[ i ] ) )
3509 utilities.assert_equals( expect=main.TRUE,
3510 actual=sizeResults,
3511 onpass="Set sizes are correct",
3512 onfail="Set sizes are incorrect" )
3513
3514 main.step( "Distributed Set add()" )
3515 onosSet.add( addValue )
3516 addResponses = []
3517 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003518 for i in range( main.numCtrls ):
3519 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003520 name="setTestAdd-" + str( i ),
3521 args=[ onosSetName, addValue ] )
3522 threads.append( t )
3523 t.start()
3524 for t in threads:
3525 t.join()
3526 addResponses.append( t.result )
3527
3528 # main.TRUE = successfully changed the set
3529 # main.FALSE = action resulted in no change in set
3530 # main.ERROR - Some error in executing the function
3531 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003532 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003533 if addResponses[ i ] == main.TRUE:
3534 # All is well
3535 pass
3536 elif addResponses[ i ] == main.FALSE:
3537 # Already in set, probably fine
3538 pass
3539 elif addResponses[ i ] == main.ERROR:
3540 # Error in execution
3541 addResults = main.FALSE
3542 else:
3543 # unexpected result
3544 addResults = main.FALSE
3545 if addResults != main.TRUE:
3546 main.log.error( "Error executing set add" )
3547
3548 # Check if set is still correct
3549 size = len( onosSet )
3550 getResponses = []
3551 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003552 for i in range( main.numCtrls ):
3553 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003554 name="setTestGet-" + str( i ),
3555 args=[ onosSetName ] )
3556 threads.append( t )
3557 t.start()
3558 for t in threads:
3559 t.join()
3560 getResponses.append( t.result )
3561 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003562 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003563 if isinstance( getResponses[ i ], list):
3564 current = set( getResponses[ i ] )
3565 if len( current ) == len( getResponses[ i ] ):
3566 # no repeats
3567 if onosSet != current:
3568 main.log.error( "ONOS" + str( i + 1 ) +
3569 " has incorrect view" +
3570 " of set " + onosSetName + ":\n" +
3571 str( getResponses[ i ] ) )
3572 main.log.debug( "Expected: " + str( onosSet ) )
3573 main.log.debug( "Actual: " + str( current ) )
3574 getResults = main.FALSE
3575 else:
3576 # error, set is not a set
3577 main.log.error( "ONOS" + str( i + 1 ) +
3578 " has repeat elements in" +
3579 " set " + onosSetName + ":\n" +
3580 str( getResponses[ i ] ) )
3581 getResults = main.FALSE
3582 elif getResponses[ i ] == main.ERROR:
3583 getResults = main.FALSE
3584 sizeResponses = []
3585 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003586 for i in range( main.numCtrls ):
3587 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003588 name="setTestSize-" + str( i ),
3589 args=[ onosSetName ] )
3590 threads.append( t )
3591 t.start()
3592 for t in threads:
3593 t.join()
3594 sizeResponses.append( t.result )
3595 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003596 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003597 if size != sizeResponses[ i ]:
3598 sizeResults = main.FALSE
3599 main.log.error( "ONOS" + str( i + 1 ) +
3600 " expected a size of " + str( size ) +
3601 " for set " + onosSetName +
3602 " but got " + str( sizeResponses[ i ] ) )
3603 addResults = addResults and getResults and sizeResults
3604 utilities.assert_equals( expect=main.TRUE,
3605 actual=addResults,
3606 onpass="Set add correct",
3607 onfail="Set add was incorrect" )
3608
3609 main.step( "Distributed Set addAll()" )
3610 onosSet.update( addAllValue.split() )
3611 addResponses = []
3612 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003613 for i in range( main.numCtrls ):
3614 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003615 name="setTestAddAll-" + str( i ),
3616 args=[ onosSetName, addAllValue ] )
3617 threads.append( t )
3618 t.start()
3619 for t in threads:
3620 t.join()
3621 addResponses.append( t.result )
3622
3623 # main.TRUE = successfully changed the set
3624 # main.FALSE = action resulted in no change in set
3625 # main.ERROR - Some error in executing the function
3626 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003627 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003628 if addResponses[ i ] == main.TRUE:
3629 # All is well
3630 pass
3631 elif addResponses[ i ] == main.FALSE:
3632 # Already in set, probably fine
3633 pass
3634 elif addResponses[ i ] == main.ERROR:
3635 # Error in execution
3636 addAllResults = main.FALSE
3637 else:
3638 # unexpected result
3639 addAllResults = main.FALSE
3640 if addAllResults != main.TRUE:
3641 main.log.error( "Error executing set addAll" )
3642
3643 # Check if set is still correct
3644 size = len( onosSet )
3645 getResponses = []
3646 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003647 for i in range( main.numCtrls ):
3648 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003649 name="setTestGet-" + str( i ),
3650 args=[ onosSetName ] )
3651 threads.append( t )
3652 t.start()
3653 for t in threads:
3654 t.join()
3655 getResponses.append( t.result )
3656 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003657 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003658 if isinstance( getResponses[ i ], list):
3659 current = set( getResponses[ i ] )
3660 if len( current ) == len( getResponses[ i ] ):
3661 # no repeats
3662 if onosSet != current:
3663 main.log.error( "ONOS" + str( i + 1 ) +
3664 " has incorrect view" +
3665 " of set " + onosSetName + ":\n" +
3666 str( getResponses[ i ] ) )
3667 main.log.debug( "Expected: " + str( onosSet ) )
3668 main.log.debug( "Actual: " + str( current ) )
3669 getResults = main.FALSE
3670 else:
3671 # error, set is not a set
3672 main.log.error( "ONOS" + str( i + 1 ) +
3673 " has repeat elements in" +
3674 " set " + onosSetName + ":\n" +
3675 str( getResponses[ i ] ) )
3676 getResults = main.FALSE
3677 elif getResponses[ i ] == main.ERROR:
3678 getResults = main.FALSE
3679 sizeResponses = []
3680 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003681 for i in range( main.numCtrls ):
3682 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003683 name="setTestSize-" + str( i ),
3684 args=[ onosSetName ] )
3685 threads.append( t )
3686 t.start()
3687 for t in threads:
3688 t.join()
3689 sizeResponses.append( t.result )
3690 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003691 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003692 if size != sizeResponses[ i ]:
3693 sizeResults = main.FALSE
3694 main.log.error( "ONOS" + str( i + 1 ) +
3695 " expected a size of " + str( size ) +
3696 " for set " + onosSetName +
3697 " but got " + str( sizeResponses[ i ] ) )
3698 addAllResults = addAllResults and getResults and sizeResults
3699 utilities.assert_equals( expect=main.TRUE,
3700 actual=addAllResults,
3701 onpass="Set addAll correct",
3702 onfail="Set addAll was incorrect" )
3703
3704 main.step( "Distributed Set contains()" )
3705 containsResponses = []
3706 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003707 for i in range( main.numCtrls ):
3708 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003709 name="setContains-" + str( i ),
3710 args=[ onosSetName ],
3711 kwargs={ "values": addValue } )
3712 threads.append( t )
3713 t.start()
3714 for t in threads:
3715 t.join()
3716 # NOTE: This is the tuple
3717 containsResponses.append( t.result )
3718
3719 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003720 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003721 if containsResponses[ i ] == main.ERROR:
3722 containsResults = main.FALSE
3723 else:
3724 containsResults = containsResults and\
3725 containsResponses[ i ][ 1 ]
3726 utilities.assert_equals( expect=main.TRUE,
3727 actual=containsResults,
3728 onpass="Set contains is functional",
3729 onfail="Set contains failed" )
3730
3731 main.step( "Distributed Set containsAll()" )
3732 containsAllResponses = []
3733 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003734 for i in range( main.numCtrls ):
3735 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003736 name="setContainsAll-" + str( i ),
3737 args=[ onosSetName ],
3738 kwargs={ "values": addAllValue } )
3739 threads.append( t )
3740 t.start()
3741 for t in threads:
3742 t.join()
3743 # NOTE: This is the tuple
3744 containsAllResponses.append( t.result )
3745
3746 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003747 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003748 if containsResponses[ i ] == main.ERROR:
3749 containsResults = main.FALSE
3750 else:
3751 containsResults = containsResults and\
3752 containsResponses[ i ][ 1 ]
3753 utilities.assert_equals( expect=main.TRUE,
3754 actual=containsAllResults,
3755 onpass="Set containsAll is functional",
3756 onfail="Set containsAll failed" )
3757
3758 main.step( "Distributed Set remove()" )
3759 onosSet.remove( addValue )
3760 removeResponses = []
3761 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003762 for i in range( main.numCtrls ):
3763 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003764 name="setTestRemove-" + str( i ),
3765 args=[ onosSetName, addValue ] )
3766 threads.append( t )
3767 t.start()
3768 for t in threads:
3769 t.join()
3770 removeResponses.append( t.result )
3771
3772 # main.TRUE = successfully changed the set
3773 # main.FALSE = action resulted in no change in set
3774 # main.ERROR - Some error in executing the function
3775 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003776 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003777 if removeResponses[ i ] == main.TRUE:
3778 # All is well
3779 pass
3780 elif removeResponses[ i ] == main.FALSE:
3781 # not in set, probably fine
3782 pass
3783 elif removeResponses[ i ] == main.ERROR:
3784 # Error in execution
3785 removeResults = main.FALSE
3786 else:
3787 # unexpected result
3788 removeResults = main.FALSE
3789 if removeResults != main.TRUE:
3790 main.log.error( "Error executing set remove" )
3791
3792 # Check if set is still correct
3793 size = len( onosSet )
3794 getResponses = []
3795 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003796 for i in range( main.numCtrls ):
3797 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003798 name="setTestGet-" + str( i ),
3799 args=[ onosSetName ] )
3800 threads.append( t )
3801 t.start()
3802 for t in threads:
3803 t.join()
3804 getResponses.append( t.result )
3805 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003806 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003807 if isinstance( getResponses[ i ], list):
3808 current = set( getResponses[ i ] )
3809 if len( current ) == len( getResponses[ i ] ):
3810 # no repeats
3811 if onosSet != current:
3812 main.log.error( "ONOS" + str( i + 1 ) +
3813 " has incorrect view" +
3814 " of set " + onosSetName + ":\n" +
3815 str( getResponses[ i ] ) )
3816 main.log.debug( "Expected: " + str( onosSet ) )
3817 main.log.debug( "Actual: " + str( current ) )
3818 getResults = main.FALSE
3819 else:
3820 # error, set is not a set
3821 main.log.error( "ONOS" + str( i + 1 ) +
3822 " has repeat elements in" +
3823 " set " + onosSetName + ":\n" +
3824 str( getResponses[ i ] ) )
3825 getResults = main.FALSE
3826 elif getResponses[ i ] == main.ERROR:
3827 getResults = main.FALSE
3828 sizeResponses = []
3829 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003830 for i in range( main.numCtrls ):
3831 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003832 name="setTestSize-" + str( i ),
3833 args=[ onosSetName ] )
3834 threads.append( t )
3835 t.start()
3836 for t in threads:
3837 t.join()
3838 sizeResponses.append( t.result )
3839 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003840 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003841 if size != sizeResponses[ i ]:
3842 sizeResults = main.FALSE
3843 main.log.error( "ONOS" + str( i + 1 ) +
3844 " expected a size of " + str( size ) +
3845 " for set " + onosSetName +
3846 " but got " + str( sizeResponses[ i ] ) )
3847 removeResults = removeResults and getResults and sizeResults
3848 utilities.assert_equals( expect=main.TRUE,
3849 actual=removeResults,
3850 onpass="Set remove correct",
3851 onfail="Set remove was incorrect" )
3852
3853 main.step( "Distributed Set removeAll()" )
3854 onosSet.difference_update( addAllValue.split() )
3855 removeAllResponses = []
3856 threads = []
3857 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003858 for i in range( main.numCtrls ):
3859 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003860 name="setTestRemoveAll-" + str( i ),
3861 args=[ onosSetName, addAllValue ] )
3862 threads.append( t )
3863 t.start()
3864 for t in threads:
3865 t.join()
3866 removeAllResponses.append( t.result )
3867 except Exception, e:
3868 main.log.exception(e)
3869
3870 # main.TRUE = successfully changed the set
3871 # main.FALSE = action resulted in no change in set
3872 # main.ERROR - Some error in executing the function
3873 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003874 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003875 if removeAllResponses[ i ] == main.TRUE:
3876 # All is well
3877 pass
3878 elif removeAllResponses[ i ] == main.FALSE:
3879 # not in set, probably fine
3880 pass
3881 elif removeAllResponses[ i ] == main.ERROR:
3882 # Error in execution
3883 removeAllResults = main.FALSE
3884 else:
3885 # unexpected result
3886 removeAllResults = main.FALSE
3887 if removeAllResults != main.TRUE:
3888 main.log.error( "Error executing set removeAll" )
3889
3890 # Check if set is still correct
3891 size = len( onosSet )
3892 getResponses = []
3893 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003894 for i in range( main.numCtrls ):
3895 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003896 name="setTestGet-" + str( i ),
3897 args=[ onosSetName ] )
3898 threads.append( t )
3899 t.start()
3900 for t in threads:
3901 t.join()
3902 getResponses.append( t.result )
3903 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003904 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003905 if isinstance( getResponses[ i ], list):
3906 current = set( getResponses[ i ] )
3907 if len( current ) == len( getResponses[ i ] ):
3908 # no repeats
3909 if onosSet != current:
3910 main.log.error( "ONOS" + str( i + 1 ) +
3911 " has incorrect view" +
3912 " of set " + onosSetName + ":\n" +
3913 str( getResponses[ i ] ) )
3914 main.log.debug( "Expected: " + str( onosSet ) )
3915 main.log.debug( "Actual: " + str( current ) )
3916 getResults = main.FALSE
3917 else:
3918 # error, set is not a set
3919 main.log.error( "ONOS" + str( i + 1 ) +
3920 " has repeat elements in" +
3921 " set " + onosSetName + ":\n" +
3922 str( getResponses[ i ] ) )
3923 getResults = main.FALSE
3924 elif getResponses[ i ] == main.ERROR:
3925 getResults = main.FALSE
3926 sizeResponses = []
3927 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003928 for i in range( main.numCtrls ):
3929 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003930 name="setTestSize-" + str( i ),
3931 args=[ onosSetName ] )
3932 threads.append( t )
3933 t.start()
3934 for t in threads:
3935 t.join()
3936 sizeResponses.append( t.result )
3937 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003938 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003939 if size != sizeResponses[ i ]:
3940 sizeResults = main.FALSE
3941 main.log.error( "ONOS" + str( i + 1 ) +
3942 " expected a size of " + str( size ) +
3943 " for set " + onosSetName +
3944 " but got " + str( sizeResponses[ i ] ) )
3945 removeAllResults = removeAllResults and getResults and sizeResults
3946 utilities.assert_equals( expect=main.TRUE,
3947 actual=removeAllResults,
3948 onpass="Set removeAll correct",
3949 onfail="Set removeAll was incorrect" )
3950
3951 main.step( "Distributed Set addAll()" )
3952 onosSet.update( addAllValue.split() )
3953 addResponses = []
3954 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003955 for i in range( main.numCtrls ):
3956 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003957 name="setTestAddAll-" + str( i ),
3958 args=[ onosSetName, addAllValue ] )
3959 threads.append( t )
3960 t.start()
3961 for t in threads:
3962 t.join()
3963 addResponses.append( t.result )
3964
3965 # main.TRUE = successfully changed the set
3966 # main.FALSE = action resulted in no change in set
3967 # main.ERROR - Some error in executing the function
3968 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003969 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003970 if addResponses[ i ] == main.TRUE:
3971 # All is well
3972 pass
3973 elif addResponses[ i ] == main.FALSE:
3974 # Already in set, probably fine
3975 pass
3976 elif addResponses[ i ] == main.ERROR:
3977 # Error in execution
3978 addAllResults = main.FALSE
3979 else:
3980 # unexpected result
3981 addAllResults = main.FALSE
3982 if addAllResults != main.TRUE:
3983 main.log.error( "Error executing set addAll" )
3984
3985 # Check if set is still correct
3986 size = len( onosSet )
3987 getResponses = []
3988 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003989 for i in range( main.numCtrls ):
3990 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003991 name="setTestGet-" + str( i ),
3992 args=[ onosSetName ] )
3993 threads.append( t )
3994 t.start()
3995 for t in threads:
3996 t.join()
3997 getResponses.append( t.result )
3998 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003999 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004000 if isinstance( getResponses[ i ], list):
4001 current = set( getResponses[ i ] )
4002 if len( current ) == len( getResponses[ i ] ):
4003 # no repeats
4004 if onosSet != current:
4005 main.log.error( "ONOS" + str( i + 1 ) +
4006 " has incorrect view" +
4007 " of set " + onosSetName + ":\n" +
4008 str( getResponses[ i ] ) )
4009 main.log.debug( "Expected: " + str( onosSet ) )
4010 main.log.debug( "Actual: " + str( current ) )
4011 getResults = main.FALSE
4012 else:
4013 # error, set is not a set
4014 main.log.error( "ONOS" + str( i + 1 ) +
4015 " has repeat elements in" +
4016 " set " + onosSetName + ":\n" +
4017 str( getResponses[ i ] ) )
4018 getResults = main.FALSE
4019 elif getResponses[ i ] == main.ERROR:
4020 getResults = main.FALSE
4021 sizeResponses = []
4022 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004023 for i in range( main.numCtrls ):
4024 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004025 name="setTestSize-" + str( i ),
4026 args=[ onosSetName ] )
4027 threads.append( t )
4028 t.start()
4029 for t in threads:
4030 t.join()
4031 sizeResponses.append( t.result )
4032 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004033 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004034 if size != sizeResponses[ i ]:
4035 sizeResults = main.FALSE
4036 main.log.error( "ONOS" + str( i + 1 ) +
4037 " expected a size of " + str( size ) +
4038 " for set " + onosSetName +
4039 " but got " + str( sizeResponses[ i ] ) )
4040 addAllResults = addAllResults and getResults and sizeResults
4041 utilities.assert_equals( expect=main.TRUE,
4042 actual=addAllResults,
4043 onpass="Set addAll correct",
4044 onfail="Set addAll was incorrect" )
4045
4046 main.step( "Distributed Set clear()" )
4047 onosSet.clear()
4048 clearResponses = []
4049 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004050 for i in range( main.numCtrls ):
4051 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004052 name="setTestClear-" + str( i ),
4053 args=[ onosSetName, " "], # Values doesn't matter
4054 kwargs={ "clear": True } )
4055 threads.append( t )
4056 t.start()
4057 for t in threads:
4058 t.join()
4059 clearResponses.append( t.result )
4060
4061 # main.TRUE = successfully changed the set
4062 # main.FALSE = action resulted in no change in set
4063 # main.ERROR - Some error in executing the function
4064 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004065 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004066 if clearResponses[ i ] == main.TRUE:
4067 # All is well
4068 pass
4069 elif clearResponses[ i ] == main.FALSE:
4070 # Nothing set, probably fine
4071 pass
4072 elif clearResponses[ i ] == main.ERROR:
4073 # Error in execution
4074 clearResults = main.FALSE
4075 else:
4076 # unexpected result
4077 clearResults = main.FALSE
4078 if clearResults != main.TRUE:
4079 main.log.error( "Error executing set clear" )
4080
4081 # Check if set is still correct
4082 size = len( onosSet )
4083 getResponses = []
4084 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004085 for i in range( main.numCtrls ):
4086 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004087 name="setTestGet-" + str( i ),
4088 args=[ onosSetName ] )
4089 threads.append( t )
4090 t.start()
4091 for t in threads:
4092 t.join()
4093 getResponses.append( t.result )
4094 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004095 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004096 if isinstance( getResponses[ i ], list):
4097 current = set( getResponses[ i ] )
4098 if len( current ) == len( getResponses[ i ] ):
4099 # no repeats
4100 if onosSet != current:
4101 main.log.error( "ONOS" + str( i + 1 ) +
4102 " has incorrect view" +
4103 " of set " + onosSetName + ":\n" +
4104 str( getResponses[ i ] ) )
4105 main.log.debug( "Expected: " + str( onosSet ) )
4106 main.log.debug( "Actual: " + str( current ) )
4107 getResults = main.FALSE
4108 else:
4109 # error, set is not a set
4110 main.log.error( "ONOS" + str( i + 1 ) +
4111 " has repeat elements in" +
4112 " set " + onosSetName + ":\n" +
4113 str( getResponses[ i ] ) )
4114 getResults = main.FALSE
4115 elif getResponses[ i ] == main.ERROR:
4116 getResults = main.FALSE
4117 sizeResponses = []
4118 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004119 for i in range( main.numCtrls ):
4120 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004121 name="setTestSize-" + str( i ),
4122 args=[ onosSetName ] )
4123 threads.append( t )
4124 t.start()
4125 for t in threads:
4126 t.join()
4127 sizeResponses.append( t.result )
4128 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004129 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004130 if size != sizeResponses[ i ]:
4131 sizeResults = main.FALSE
4132 main.log.error( "ONOS" + str( i + 1 ) +
4133 " expected a size of " + str( size ) +
4134 " for set " + onosSetName +
4135 " but got " + str( sizeResponses[ i ] ) )
4136 clearResults = clearResults and getResults and sizeResults
4137 utilities.assert_equals( expect=main.TRUE,
4138 actual=clearResults,
4139 onpass="Set clear correct",
4140 onfail="Set clear was incorrect" )
4141
4142 main.step( "Distributed Set addAll()" )
4143 onosSet.update( addAllValue.split() )
4144 addResponses = []
4145 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004146 for i in range( main.numCtrls ):
4147 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004148 name="setTestAddAll-" + str( i ),
4149 args=[ onosSetName, addAllValue ] )
4150 threads.append( t )
4151 t.start()
4152 for t in threads:
4153 t.join()
4154 addResponses.append( t.result )
4155
4156 # main.TRUE = successfully changed the set
4157 # main.FALSE = action resulted in no change in set
4158 # main.ERROR - Some error in executing the function
4159 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004160 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004161 if addResponses[ i ] == main.TRUE:
4162 # All is well
4163 pass
4164 elif addResponses[ i ] == main.FALSE:
4165 # Already in set, probably fine
4166 pass
4167 elif addResponses[ i ] == main.ERROR:
4168 # Error in execution
4169 addAllResults = main.FALSE
4170 else:
4171 # unexpected result
4172 addAllResults = main.FALSE
4173 if addAllResults != main.TRUE:
4174 main.log.error( "Error executing set addAll" )
4175
4176 # Check if set is still correct
4177 size = len( onosSet )
4178 getResponses = []
4179 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004180 for i in range( main.numCtrls ):
4181 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004182 name="setTestGet-" + str( i ),
4183 args=[ onosSetName ] )
4184 threads.append( t )
4185 t.start()
4186 for t in threads:
4187 t.join()
4188 getResponses.append( t.result )
4189 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004190 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004191 if isinstance( getResponses[ i ], list):
4192 current = set( getResponses[ i ] )
4193 if len( current ) == len( getResponses[ i ] ):
4194 # no repeats
4195 if onosSet != current:
4196 main.log.error( "ONOS" + str( i + 1 ) +
4197 " has incorrect view" +
4198 " of set " + onosSetName + ":\n" +
4199 str( getResponses[ i ] ) )
4200 main.log.debug( "Expected: " + str( onosSet ) )
4201 main.log.debug( "Actual: " + str( current ) )
4202 getResults = main.FALSE
4203 else:
4204 # error, set is not a set
4205 main.log.error( "ONOS" + str( i + 1 ) +
4206 " has repeat elements in" +
4207 " set " + onosSetName + ":\n" +
4208 str( getResponses[ i ] ) )
4209 getResults = main.FALSE
4210 elif getResponses[ i ] == main.ERROR:
4211 getResults = main.FALSE
4212 sizeResponses = []
4213 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004214 for i in range( main.numCtrls ):
4215 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004216 name="setTestSize-" + str( i ),
4217 args=[ onosSetName ] )
4218 threads.append( t )
4219 t.start()
4220 for t in threads:
4221 t.join()
4222 sizeResponses.append( t.result )
4223 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004224 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004225 if size != sizeResponses[ i ]:
4226 sizeResults = main.FALSE
4227 main.log.error( "ONOS" + str( i + 1 ) +
4228 " expected a size of " + str( size ) +
4229 " for set " + onosSetName +
4230 " but got " + str( sizeResponses[ i ] ) )
4231 addAllResults = addAllResults and getResults and sizeResults
4232 utilities.assert_equals( expect=main.TRUE,
4233 actual=addAllResults,
4234 onpass="Set addAll correct",
4235 onfail="Set addAll was incorrect" )
4236
4237 main.step( "Distributed Set retain()" )
4238 onosSet.intersection_update( retainValue.split() )
4239 retainResponses = []
4240 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004241 for i in range( main.numCtrls ):
4242 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004243 name="setTestRetain-" + str( i ),
4244 args=[ onosSetName, retainValue ],
4245 kwargs={ "retain": True } )
4246 threads.append( t )
4247 t.start()
4248 for t in threads:
4249 t.join()
4250 retainResponses.append( t.result )
4251
4252 # main.TRUE = successfully changed the set
4253 # main.FALSE = action resulted in no change in set
4254 # main.ERROR - Some error in executing the function
4255 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004256 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004257 if retainResponses[ i ] == main.TRUE:
4258 # All is well
4259 pass
4260 elif retainResponses[ i ] == main.FALSE:
4261 # Already in set, probably fine
4262 pass
4263 elif retainResponses[ i ] == main.ERROR:
4264 # Error in execution
4265 retainResults = main.FALSE
4266 else:
4267 # unexpected result
4268 retainResults = main.FALSE
4269 if retainResults != main.TRUE:
4270 main.log.error( "Error executing set retain" )
4271
4272 # Check if set is still correct
4273 size = len( onosSet )
4274 getResponses = []
4275 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004276 for i in range( main.numCtrls ):
4277 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004278 name="setTestGet-" + str( i ),
4279 args=[ onosSetName ] )
4280 threads.append( t )
4281 t.start()
4282 for t in threads:
4283 t.join()
4284 getResponses.append( t.result )
4285 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004286 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004287 if isinstance( getResponses[ i ], list):
4288 current = set( getResponses[ i ] )
4289 if len( current ) == len( getResponses[ i ] ):
4290 # no repeats
4291 if onosSet != current:
4292 main.log.error( "ONOS" + str( i + 1 ) +
4293 " has incorrect view" +
4294 " of set " + onosSetName + ":\n" +
4295 str( getResponses[ i ] ) )
4296 main.log.debug( "Expected: " + str( onosSet ) )
4297 main.log.debug( "Actual: " + str( current ) )
4298 getResults = main.FALSE
4299 else:
4300 # error, set is not a set
4301 main.log.error( "ONOS" + str( i + 1 ) +
4302 " has repeat elements in" +
4303 " set " + onosSetName + ":\n" +
4304 str( getResponses[ i ] ) )
4305 getResults = main.FALSE
4306 elif getResponses[ i ] == main.ERROR:
4307 getResults = main.FALSE
4308 sizeResponses = []
4309 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004310 for i in range( main.numCtrls ):
4311 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004312 name="setTestSize-" + str( i ),
4313 args=[ onosSetName ] )
4314 threads.append( t )
4315 t.start()
4316 for t in threads:
4317 t.join()
4318 sizeResponses.append( t.result )
4319 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004320 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004321 if size != sizeResponses[ i ]:
4322 sizeResults = main.FALSE
4323 main.log.error( "ONOS" + str( i + 1 ) +
4324 " expected a size of " +
4325 str( size ) + " for set " + onosSetName +
4326 " but got " + str( sizeResponses[ i ] ) )
4327 retainResults = retainResults and getResults and sizeResults
4328 utilities.assert_equals( expect=main.TRUE,
4329 actual=retainResults,
4330 onpass="Set retain correct",
4331 onfail="Set retain was incorrect" )
4332
Jon Hall2a5002c2015-08-21 16:49:11 -07004333 # Transactional maps
4334 main.step( "Partitioned Transactional maps put" )
4335 tMapValue = "Testing"
4336 numKeys = 100
4337 putResult = True
4338 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4339 if len( putResponses ) == 100:
4340 for i in putResponses:
4341 if putResponses[ i ][ 'value' ] != tMapValue:
4342 putResult = False
4343 else:
4344 putResult = False
4345 if not putResult:
4346 main.log.debug( "Put response values: " + str( putResponses ) )
4347 utilities.assert_equals( expect=True,
4348 actual=putResult,
4349 onpass="Partitioned Transactional Map put successful",
4350 onfail="Partitioned Transactional Map put values are incorrect" )
4351
4352 main.step( "Partitioned Transactional maps get" )
4353 getCheck = True
4354 for n in range( 1, numKeys + 1 ):
4355 getResponses = []
4356 threads = []
4357 valueCheck = True
4358 for i in range( main.numCtrls ):
4359 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4360 name="TMap-get-" + str( i ),
4361 args=[ "Key" + str ( n ) ] )
4362 threads.append( t )
4363 t.start()
4364 for t in threads:
4365 t.join()
4366 getResponses.append( t.result )
4367 for node in getResponses:
4368 if node != tMapValue:
4369 valueCheck = False
4370 if not valueCheck:
4371 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4372 main.log.warn( getResponses )
4373 getCheck = getCheck and valueCheck
4374 utilities.assert_equals( expect=True,
4375 actual=getCheck,
4376 onpass="Partitioned Transactional Map get values were correct",
4377 onfail="Partitioned Transactional Map values incorrect" )
4378
4379 main.step( "In-memory Transactional maps put" )
4380 tMapValue = "Testing"
4381 numKeys = 100
4382 putResult = True
4383 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4384 if len( putResponses ) == 100:
4385 for i in putResponses:
4386 if putResponses[ i ][ 'value' ] != tMapValue:
4387 putResult = False
4388 else:
4389 putResult = False
4390 if not putResult:
4391 main.log.debug( "Put response values: " + str( putResponses ) )
4392 utilities.assert_equals( expect=True,
4393 actual=putResult,
4394 onpass="In-Memory Transactional Map put successful",
4395 onfail="In-Memory Transactional Map put values are incorrect" )
4396
4397 main.step( "In-Memory Transactional maps get" )
4398 getCheck = True
4399 for n in range( 1, numKeys + 1 ):
4400 getResponses = []
4401 threads = []
4402 valueCheck = True
4403 for i in range( main.numCtrls ):
4404 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4405 name="TMap-get-" + str( i ),
4406 args=[ "Key" + str ( n ) ],
4407 kwargs={ "inMemory": True } )
4408 threads.append( t )
4409 t.start()
4410 for t in threads:
4411 t.join()
4412 getResponses.append( t.result )
4413 for node in getResponses:
4414 if node != tMapValue:
4415 valueCheck = False
4416 if not valueCheck:
4417 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4418 main.log.warn( getResponses )
4419 getCheck = getCheck and valueCheck
4420 utilities.assert_equals( expect=True,
4421 actual=getCheck,
4422 onpass="In-Memory Transactional Map get values were correct",
4423 onfail="In-Memory Transactional Map values incorrect" )