blob: 352009f5f7d863d719c49361935260f74b142301 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAclusterRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hall5cf14d52015-07-16 12:15:19 -070051 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
52 "initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
70 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070071 global ONOS1Port
72 global ONOS2Port
73 global ONOS3Port
74 global ONOS4Port
75 global ONOS5Port
76 global ONOS6Port
77 global ONOS7Port
78 # These are for csv plotting in jenkins
79 global labels
80 global data
81 labels = []
82 data = []
83
84 # FIXME: just get controller port from params?
85 # TODO: do we really need all these?
86 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
87 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
88 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
89 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
90 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
91 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
92 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
93
Jon Halle1a3b752015-07-22 13:02:46 -070094 try:
95 fileName = "Counters"
96 path = main.params[ 'imports' ][ 'path' ]
97 main.Counters = imp.load_source( fileName,
98 path + fileName + ".py" )
99 except Exception as e:
100 main.log.exception( e )
101 main.cleanup()
102 main.exit()
103
104 main.CLIs = []
105 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700106 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700107 for i in range( 1, main.numCtrls + 1 ):
108 try:
109 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
110 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
111 ipList.append( main.nodes[ -1 ].ip_address )
112 except AttributeError:
113 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700114
115 main.step( "Create cell file" )
116 cellAppString = main.params[ 'ENV' ][ 'appString' ]
117 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
118 main.Mininet1.ip_address,
119 cellAppString, ipList )
120 main.step( "Applying cell variable to environment" )
121 cellResult = main.ONOSbench.setCell( cellName )
122 verifyResult = main.ONOSbench.verifyCell()
123
124 # FIXME:this is short term fix
125 main.log.info( "Removing raft logs" )
126 main.ONOSbench.onosRemoveRaftLogs()
127
128 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700129 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700130 main.ONOSbench.onosUninstall( node.ip_address )
131
132 # Make sure ONOS is DEAD
133 main.log.info( "Killing any ONOS processes" )
134 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700135 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700136 killed = main.ONOSbench.onosKill( node.ip_address )
137 killResults = killResults and killed
138
139 cleanInstallResult = main.TRUE
140 gitPullResult = main.TRUE
141
142 main.step( "Starting Mininet" )
143 # scp topo file to mininet
144 # TODO: move to params?
145 topoName = "obelisk.py"
146 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700147 main.ONOSbench.scp( main.Mininet1,
148 filePath + topoName,
149 main.Mininet1.home,
150 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700151 mnResult = main.Mininet1.startNet( )
152 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
153 onpass="Mininet Started",
154 onfail="Error starting Mininet" )
155
156 main.step( "Git checkout and pull " + gitBranch )
157 if PULLCODE:
158 main.ONOSbench.gitCheckout( gitBranch )
159 gitPullResult = main.ONOSbench.gitPull()
160 # values of 1 or 3 are good
161 utilities.assert_lesser( expect=0, actual=gitPullResult,
162 onpass="Git pull successful",
163 onfail="Git pull failed" )
164 main.ONOSbench.getVersion( report=True )
165
166 main.step( "Using mvn clean install" )
167 cleanInstallResult = main.TRUE
168 if PULLCODE and gitPullResult == main.TRUE:
169 cleanInstallResult = main.ONOSbench.cleanInstall()
170 else:
171 main.log.warn( "Did not pull new code so skipping mvn " +
172 "clean install" )
173 utilities.assert_equals( expect=main.TRUE,
174 actual=cleanInstallResult,
175 onpass="MCI successful",
176 onfail="MCI failed" )
177 # GRAPHS
178 # NOTE: important params here:
179 # job = name of Jenkins job
180 # Plot Name = Plot-HA, only can be used if multiple plots
181 # index = The number of the graph under plot name
182 job = "HAclusterRestart"
183 plotName = "Plot-HA"
184 graphs = '<ac:structured-macro ac:name="html">\n'
185 graphs += '<ac:plain-text-body><![CDATA[\n'
186 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
187 '/plot/' + plotName + '/getPlot?index=0' +\
188 '&width=500&height=300"' +\
189 'noborder="0" width="500" height="300" scrolling="yes" ' +\
190 'seamless="seamless"></iframe>\n'
191 graphs += ']]></ac:plain-text-body>\n'
192 graphs += '</ac:structured-macro>\n'
193 main.log.wiki(graphs)
194
195 main.step( "Creating ONOS package" )
196 packageResult = main.ONOSbench.onosPackage()
197 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
198 onpass="ONOS package successful",
199 onfail="ONOS package failed" )
200
201 main.step( "Installing ONOS package" )
202 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700203 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700204 tmpResult = main.ONOSbench.onosInstall( options="-f",
205 node=node.ip_address )
206 onosInstallResult = onosInstallResult and tmpResult
207 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
208 onpass="ONOS install successful",
209 onfail="ONOS install failed" )
210
211 main.step( "Checking if ONOS is up yet" )
212 for i in range( 2 ):
213 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700214 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700215 started = main.ONOSbench.isup( node.ip_address )
216 if not started:
217 main.log.error( node.name + " didn't start!" )
218 main.ONOSbench.onosStop( node.ip_address )
219 main.ONOSbench.onosStart( node.ip_address )
220 onosIsupResult = onosIsupResult and started
221 if onosIsupResult == main.TRUE:
222 break
223 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
224 onpass="ONOS startup successful",
225 onfail="ONOS startup failed" )
226
227 main.log.step( "Starting ONOS CLI sessions" )
228 cliResults = main.TRUE
229 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700230 for i in range( main.numCtrls ):
231 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700232 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700233 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700234 threads.append( t )
235 t.start()
236
237 for t in threads:
238 t.join()
239 cliResults = cliResults and t.result
240 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
241 onpass="ONOS cli startup successful",
242 onfail="ONOS cli startup failed" )
243
244 if main.params[ 'tcpdump' ].lower() == "true":
245 main.step( "Start Packet Capture MN" )
246 main.Mininet2.startTcpdump(
247 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
248 + "-MN.pcap",
249 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
250 port=main.params[ 'MNtcpdump' ][ 'port' ] )
251
252 main.step( "App Ids check" )
253 appCheck = main.TRUE
254 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700255 for i in range( main.numCtrls ):
256 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700257 name="appToIDCheck-" + str( i ),
258 args=[] )
259 threads.append( t )
260 t.start()
261
262 for t in threads:
263 t.join()
264 appCheck = appCheck and t.result
265 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700266 main.log.warn( main.CLIs[0].apps() )
267 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700268 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
269 onpass="App Ids seem to be correct",
270 onfail="Something is wrong with app Ids" )
271
272 if cliResults == main.FALSE:
273 main.log.error( "Failed to start ONOS, stopping test" )
274 main.cleanup()
275 main.exit()
276
277 def CASE2( self, main ):
278 """
279 Assign devices to controllers
280 """
281 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700282 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700283 assert main, "main not defined"
284 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700285 assert main.CLIs, "main.CLIs not defined"
286 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700287 assert ONOS1Port, "ONOS1Port not defined"
288 assert ONOS2Port, "ONOS2Port not defined"
289 assert ONOS3Port, "ONOS3Port not defined"
290 assert ONOS4Port, "ONOS4Port not defined"
291 assert ONOS5Port, "ONOS5Port not defined"
292 assert ONOS6Port, "ONOS6Port not defined"
293 assert ONOS7Port, "ONOS7Port not defined"
294
295 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700296 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700297 "and check that an ONOS node becomes the " +\
298 "master of the device."
299 main.step( "Assign switches to controllers" )
300
301 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700302 for i in range( main.numCtrls ):
303 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700304 swList = []
305 for i in range( 1, 29 ):
306 swList.append( "s" + str( i ) )
307 main.Mininet1.assignSwController( sw=swList, ip=ipList )
308
309 mastershipCheck = main.TRUE
310 for i in range( 1, 29 ):
311 response = main.Mininet1.getSwController( "s" + str( i ) )
312 try:
313 main.log.info( str( response ) )
314 except Exception:
315 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700316 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700317 if re.search( "tcp:" + node.ip_address, response ):
318 mastershipCheck = mastershipCheck and main.TRUE
319 else:
320 main.log.error( "Error, node " + node.ip_address + " is " +
321 "not in the list of controllers s" +
322 str( i ) + " is connecting to." )
323 mastershipCheck = main.FALSE
324 utilities.assert_equals(
325 expect=main.TRUE,
326 actual=mastershipCheck,
327 onpass="Switch mastership assigned correctly",
328 onfail="Switches not assigned correctly to controllers" )
329
330 def CASE21( self, main ):
331 """
332 Assign mastership to controllers
333 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700334 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700335 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700336 assert main, "main not defined"
337 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700338 assert main.CLIs, "main.CLIs not defined"
339 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700340 assert ONOS1Port, "ONOS1Port not defined"
341 assert ONOS2Port, "ONOS2Port not defined"
342 assert ONOS3Port, "ONOS3Port not defined"
343 assert ONOS4Port, "ONOS4Port not defined"
344 assert ONOS5Port, "ONOS5Port not defined"
345 assert ONOS6Port, "ONOS6Port not defined"
346 assert ONOS7Port, "ONOS7Port not defined"
347
348 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700349 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700350 "device. Then manually assign" +\
351 " mastership to specific ONOS nodes using" +\
352 " 'device-role'"
353 main.step( "Assign mastership of switches to specific controllers" )
354 # Manually assign mastership to the controller we want
355 roleCall = main.TRUE
356
357 ipList = [ ]
358 deviceList = []
359 try:
360 # Assign mastership to specific controllers. This assignment was
361 # determined for a 7 node cluser, but will work with any sized
362 # cluster
363 for i in range( 1, 29 ): # switches 1 through 28
364 # set up correct variables:
365 if i == 1:
366 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700367 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
369 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700370 c = 1 % main.numCtrls
371 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
373 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700374 c = 1 % main.numCtrls
375 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700376 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
377 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700378 c = 3 % main.numCtrls
379 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700380 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
381 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700382 c = 2 % main.numCtrls
383 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700384 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
385 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700386 c = 2 % main.numCtrls
387 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700388 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
389 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700390 c = 5 % main.numCtrls
391 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700392 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
393 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700394 c = 4 % main.numCtrls
395 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700396 dpid = '3' + str( i ).zfill( 3 )
397 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
398 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700399 c = 6 % main.numCtrls
400 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700401 dpid = '6' + str( i ).zfill( 3 )
402 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
403 elif i == 28:
404 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700405 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700406 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
407 else:
408 main.log.error( "You didn't write an else statement for " +
409 "switch s" + str( i ) )
410 roleCall = main.FALSE
411 # Assign switch
412 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
413 # TODO: make this controller dynamic
414 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
415 ip )
416 ipList.append( ip )
417 deviceList.append( deviceId )
418 except ( AttributeError, AssertionError ):
419 main.log.exception( "Something is wrong with ONOS device view" )
420 main.log.info( main.ONOScli1.devices() )
421 utilities.assert_equals(
422 expect=main.TRUE,
423 actual=roleCall,
424 onpass="Re-assigned switch mastership to designated controller",
425 onfail="Something wrong with deviceRole calls" )
426
427 main.step( "Check mastership was correctly assigned" )
428 roleCheck = main.TRUE
429 # NOTE: This is due to the fact that device mastership change is not
430 # atomic and is actually a multi step process
431 time.sleep( 5 )
432 for i in range( len( ipList ) ):
433 ip = ipList[i]
434 deviceId = deviceList[i]
435 # Check assignment
436 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
437 if ip in master:
438 roleCheck = roleCheck and main.TRUE
439 else:
440 roleCheck = roleCheck and main.FALSE
441 main.log.error( "Error, controller " + ip + " is not" +
442 " master " + "of device " +
443 str( deviceId ) + ". Master is " +
444 repr( master ) + "." )
445 utilities.assert_equals(
446 expect=main.TRUE,
447 actual=roleCheck,
448 onpass="Switches were successfully reassigned to designated " +
449 "controller",
450 onfail="Switches were not successfully reassigned" )
451
452 def CASE3( self, main ):
453 """
454 Assign intents
455 """
456 import time
457 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700458 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 assert main, "main not defined"
460 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700461 assert main.CLIs, "main.CLIs not defined"
462 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 try:
464 labels
465 except NameError:
466 main.log.error( "labels not defined, setting to []" )
467 labels = []
468 try:
469 data
470 except NameError:
471 main.log.error( "data not defined, setting to []" )
472 data = []
473 # NOTE: we must reinstall intents until we have a persistant intent
474 # datastore!
475 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700476 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700477 "assign predetermined host-to-host intents." +\
478 " After installation, check that the intent" +\
479 " is distributed to all nodes and the state" +\
480 " is INSTALLED"
481
482 # install onos-app-fwd
483 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700484 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700485 utilities.assert_equals( expect=main.TRUE, actual=installResults,
486 onpass="Install fwd successful",
487 onfail="Install fwd failed" )
488
489 main.step( "Check app ids" )
490 appCheck = main.TRUE
491 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700492 for i in range( main.numCtrls ):
493 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700494 name="appToIDCheck-" + str( i ),
495 args=[] )
496 threads.append( t )
497 t.start()
498
499 for t in threads:
500 t.join()
501 appCheck = appCheck and t.result
502 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700503 main.log.warn( main.CLIs[0].apps() )
504 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700505 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
506 onpass="App Ids seem to be correct",
507 onfail="Something is wrong with app Ids" )
508
509 main.step( "Discovering Hosts( Via pingall for now )" )
510 # FIXME: Once we have a host discovery mechanism, use that instead
511 # REACTIVE FWD test
512 pingResult = main.FALSE
513 for i in range(2): # Retry if pingall fails first time
514 time1 = time.time()
515 pingResult = main.Mininet1.pingall()
516 if i == 0:
517 utilities.assert_equals(
518 expect=main.TRUE,
519 actual=pingResult,
520 onpass="Reactive Pingall test passed",
521 onfail="Reactive Pingall failed, " +
522 "one or more ping pairs failed" )
523 time2 = time.time()
524 main.log.info( "Time for pingall: %2f seconds" %
525 ( time2 - time1 ) )
526 # timeout for fwd flows
527 time.sleep( 11 )
528 # uninstall onos-app-fwd
529 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700530 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700531 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
532 onpass="Uninstall fwd successful",
533 onfail="Uninstall fwd failed" )
534
535 main.step( "Check app ids" )
536 threads = []
537 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700538 for i in range( main.numCtrls ):
539 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700540 name="appToIDCheck-" + str( i ),
541 args=[] )
542 threads.append( t )
543 t.start()
544
545 for t in threads:
546 t.join()
547 appCheck2 = appCheck2 and t.result
548 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700549 main.log.warn( main.CLIs[0].apps() )
550 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700551 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
552 onpass="App Ids seem to be correct",
553 onfail="Something is wrong with app Ids" )
554
555 main.step( "Add host intents via cli" )
556 intentIds = []
557 # TODO: move the host numbers to params
558 # Maybe look at all the paths we ping?
559 intentAddResult = True
560 hostResult = main.TRUE
561 for i in range( 8, 18 ):
562 main.log.info( "Adding host intent between h" + str( i ) +
563 " and h" + str( i + 10 ) )
564 host1 = "00:00:00:00:00:" + \
565 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
566 host2 = "00:00:00:00:00:" + \
567 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
568 # NOTE: getHost can return None
569 host1Dict = main.ONOScli1.getHost( host1 )
570 host2Dict = main.ONOScli1.getHost( host2 )
571 host1Id = None
572 host2Id = None
573 if host1Dict and host2Dict:
574 host1Id = host1Dict.get( 'id', None )
575 host2Id = host2Dict.get( 'id', None )
576 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700577 nodeNum = ( i % main.numCtrls )
578 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700579 if tmpId:
580 main.log.info( "Added intent with id: " + tmpId )
581 intentIds.append( tmpId )
582 else:
583 main.log.error( "addHostIntent returned: " +
584 repr( tmpId ) )
585 else:
586 main.log.error( "Error, getHost() failed for h" + str( i ) +
587 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700588 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700589 main.log.warn( "Hosts output: " )
590 try:
591 main.log.warn( json.dumps( json.loads( hosts ),
592 sort_keys=True,
593 indent=4,
594 separators=( ',', ': ' ) ) )
595 except ( ValueError, TypeError ):
596 main.log.warn( repr( hosts ) )
597 hostResult = main.FALSE
598 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
599 onpass="Found a host id for each host",
600 onfail="Error looking up host ids" )
601
602 intentStart = time.time()
603 onosIds = main.ONOScli1.getAllIntentsId()
604 main.log.info( "Submitted intents: " + str( intentIds ) )
605 main.log.info( "Intents in ONOS: " + str( onosIds ) )
606 for intent in intentIds:
607 if intent in onosIds:
608 pass # intent submitted is in onos
609 else:
610 intentAddResult = False
611 if intentAddResult:
612 intentStop = time.time()
613 else:
614 intentStop = None
615 # Print the intent states
616 intents = main.ONOScli1.intents()
617 intentStates = []
618 installedCheck = True
619 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
620 count = 0
621 try:
622 for intent in json.loads( intents ):
623 state = intent.get( 'state', None )
624 if "INSTALLED" not in state:
625 installedCheck = False
626 intentId = intent.get( 'id', None )
627 intentStates.append( ( intentId, state ) )
628 except ( ValueError, TypeError ):
629 main.log.exception( "Error parsing intents" )
630 # add submitted intents not in the store
631 tmplist = [ i for i, s in intentStates ]
632 missingIntents = False
633 for i in intentIds:
634 if i not in tmplist:
635 intentStates.append( ( i, " - " ) )
636 missingIntents = True
637 intentStates.sort()
638 for i, s in intentStates:
639 count += 1
640 main.log.info( "%-6s%-15s%-15s" %
641 ( str( count ), str( i ), str( s ) ) )
642 leaders = main.ONOScli1.leaders()
643 try:
644 missing = False
645 if leaders:
646 parsedLeaders = json.loads( leaders )
647 main.log.warn( json.dumps( parsedLeaders,
648 sort_keys=True,
649 indent=4,
650 separators=( ',', ': ' ) ) )
651 # check for all intent partitions
652 topics = []
653 for i in range( 14 ):
654 topics.append( "intent-partition-" + str( i ) )
655 main.log.debug( topics )
656 ONOStopics = [ j['topic'] for j in parsedLeaders ]
657 for topic in topics:
658 if topic not in ONOStopics:
659 main.log.error( "Error: " + topic +
660 " not in leaders" )
661 missing = True
662 else:
663 main.log.error( "leaders() returned None" )
664 except ( ValueError, TypeError ):
665 main.log.exception( "Error parsing leaders" )
666 main.log.error( repr( leaders ) )
667 # Check all nodes
668 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700669 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700670 response = node.leaders( jsonFormat=False)
671 main.log.warn( str( node.name ) + " leaders output: \n" +
672 str( response ) )
673
674 partitions = main.ONOScli1.partitions()
675 try:
676 if partitions :
677 parsedPartitions = json.loads( partitions )
678 main.log.warn( json.dumps( parsedPartitions,
679 sort_keys=True,
680 indent=4,
681 separators=( ',', ': ' ) ) )
682 # TODO check for a leader in all paritions
683 # TODO check for consistency among nodes
684 else:
685 main.log.error( "partitions() returned None" )
686 except ( ValueError, TypeError ):
687 main.log.exception( "Error parsing partitions" )
688 main.log.error( repr( partitions ) )
689 pendingMap = main.ONOScli1.pendingMap()
690 try:
691 if pendingMap :
692 parsedPending = json.loads( pendingMap )
693 main.log.warn( json.dumps( parsedPending,
694 sort_keys=True,
695 indent=4,
696 separators=( ',', ': ' ) ) )
697 # TODO check something here?
698 else:
699 main.log.error( "pendingMap() returned None" )
700 except ( ValueError, TypeError ):
701 main.log.exception( "Error parsing pending map" )
702 main.log.error( repr( pendingMap ) )
703
704 intentAddResult = bool( intentAddResult and not missingIntents and
705 installedCheck )
706 if not intentAddResult:
707 main.log.error( "Error in pushing host intents to ONOS" )
708
709 main.step( "Intent Anti-Entropy dispersion" )
710 for i in range(100):
711 correct = True
712 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700713 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700714 onosIds = []
715 ids = cli.getAllIntentsId()
716 onosIds.append( ids )
717 main.log.debug( "Intents in " + cli.name + ": " +
718 str( sorted( onosIds ) ) )
719 if sorted( ids ) != sorted( intentIds ):
720 main.log.warn( "Set of intent IDs doesn't match" )
721 correct = False
722 break
723 else:
724 intents = json.loads( cli.intents() )
725 for intent in intents:
726 if intent[ 'state' ] != "INSTALLED":
727 main.log.warn( "Intent " + intent[ 'id' ] +
728 " is " + intent[ 'state' ] )
729 correct = False
730 break
731 if correct:
732 break
733 else:
734 time.sleep(1)
735 if not intentStop:
736 intentStop = time.time()
737 global gossipTime
738 gossipTime = intentStop - intentStart
739 main.log.info( "It took about " + str( gossipTime ) +
740 " seconds for all intents to appear in each node" )
741 append = False
742 title = "Gossip Intents"
743 count = 1
744 while append is False:
745 curTitle = title + str( count )
746 if curTitle not in labels:
747 labels.append( curTitle )
748 data.append( str( gossipTime ) )
749 append = True
750 else:
751 count += 1
752 # FIXME: make this time configurable/calculate based off of number of
753 # nodes and gossip rounds
754 utilities.assert_greater_equals(
755 expect=40, actual=gossipTime,
756 onpass="ECM anti-entropy for intents worked within " +
757 "expected time",
758 onfail="Intent ECM anti-entropy took too long" )
759 if gossipTime <= 40:
760 intentAddResult = True
761
762 if not intentAddResult or "key" in pendingMap:
763 import time
764 installedCheck = True
765 main.log.info( "Sleeping 60 seconds to see if intents are found" )
766 time.sleep( 60 )
767 onosIds = main.ONOScli1.getAllIntentsId()
768 main.log.info( "Submitted intents: " + str( intentIds ) )
769 main.log.info( "Intents in ONOS: " + str( onosIds ) )
770 # Print the intent states
771 intents = main.ONOScli1.intents()
772 intentStates = []
773 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
774 count = 0
775 try:
776 for intent in json.loads( intents ):
777 # Iter through intents of a node
778 state = intent.get( 'state', None )
779 if "INSTALLED" not in state:
780 installedCheck = False
781 intentId = intent.get( 'id', None )
782 intentStates.append( ( intentId, state ) )
783 except ( ValueError, TypeError ):
784 main.log.exception( "Error parsing intents" )
785 # add submitted intents not in the store
786 tmplist = [ i for i, s in intentStates ]
787 for i in intentIds:
788 if i not in tmplist:
789 intentStates.append( ( i, " - " ) )
790 intentStates.sort()
791 for i, s in intentStates:
792 count += 1
793 main.log.info( "%-6s%-15s%-15s" %
794 ( str( count ), str( i ), str( s ) ) )
795 leaders = main.ONOScli1.leaders()
796 try:
797 missing = False
798 if leaders:
799 parsedLeaders = json.loads( leaders )
800 main.log.warn( json.dumps( parsedLeaders,
801 sort_keys=True,
802 indent=4,
803 separators=( ',', ': ' ) ) )
804 # check for all intent partitions
805 # check for election
806 topics = []
807 for i in range( 14 ):
808 topics.append( "intent-partition-" + str( i ) )
809 # FIXME: this should only be after we start the app
810 topics.append( "org.onosproject.election" )
811 main.log.debug( topics )
812 ONOStopics = [ j['topic'] for j in parsedLeaders ]
813 for topic in topics:
814 if topic not in ONOStopics:
815 main.log.error( "Error: " + topic +
816 " not in leaders" )
817 missing = True
818 else:
819 main.log.error( "leaders() returned None" )
820 except ( ValueError, TypeError ):
821 main.log.exception( "Error parsing leaders" )
822 main.log.error( repr( leaders ) )
823 # Check all nodes
824 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700825 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700826 response = node.leaders( jsonFormat=False)
827 main.log.warn( str( node.name ) + " leaders output: \n" +
828 str( response ) )
829
830 partitions = main.ONOScli1.partitions()
831 try:
832 if partitions :
833 parsedPartitions = json.loads( partitions )
834 main.log.warn( json.dumps( parsedPartitions,
835 sort_keys=True,
836 indent=4,
837 separators=( ',', ': ' ) ) )
838 # TODO check for a leader in all paritions
839 # TODO check for consistency among nodes
840 else:
841 main.log.error( "partitions() returned None" )
842 except ( ValueError, TypeError ):
843 main.log.exception( "Error parsing partitions" )
844 main.log.error( repr( partitions ) )
845 pendingMap = main.ONOScli1.pendingMap()
846 try:
847 if pendingMap :
848 parsedPending = json.loads( pendingMap )
849 main.log.warn( json.dumps( parsedPending,
850 sort_keys=True,
851 indent=4,
852 separators=( ',', ': ' ) ) )
853 # TODO check something here?
854 else:
855 main.log.error( "pendingMap() returned None" )
856 except ( ValueError, TypeError ):
857 main.log.exception( "Error parsing pending map" )
858 main.log.error( repr( pendingMap ) )
859
860 def CASE4( self, main ):
861 """
862 Ping across added host intents
863 """
864 import json
865 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700866 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700867 assert main, "main not defined"
868 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700869 assert main.CLIs, "main.CLIs not defined"
870 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700871 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700872 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700873 "functionality and check the state of " +\
874 "the intent"
875 main.step( "Ping across added host intents" )
876 PingResult = main.TRUE
877 for i in range( 8, 18 ):
878 ping = main.Mininet1.pingHost( src="h" + str( i ),
879 target="h" + str( i + 10 ) )
880 PingResult = PingResult and ping
881 if ping == main.FALSE:
882 main.log.warn( "Ping failed between h" + str( i ) +
883 " and h" + str( i + 10 ) )
884 elif ping == main.TRUE:
885 main.log.info( "Ping test passed!" )
886 # Don't set PingResult or you'd override failures
887 if PingResult == main.FALSE:
888 main.log.error(
889 "Intents have not been installed correctly, pings failed." )
890 # TODO: pretty print
891 main.log.warn( "ONOS1 intents: " )
892 try:
893 tmpIntents = main.ONOScli1.intents()
894 main.log.warn( json.dumps( json.loads( tmpIntents ),
895 sort_keys=True,
896 indent=4,
897 separators=( ',', ': ' ) ) )
898 except ( ValueError, TypeError ):
899 main.log.warn( repr( tmpIntents ) )
900 utilities.assert_equals(
901 expect=main.TRUE,
902 actual=PingResult,
903 onpass="Intents have been installed correctly and pings work",
904 onfail="Intents have not been installed correctly, pings failed." )
905
906 main.step( "Check Intent state" )
907 installedCheck = False
908 loopCount = 0
909 while not installedCheck and loopCount < 40:
910 installedCheck = True
911 # Print the intent states
912 intents = main.ONOScli1.intents()
913 intentStates = []
914 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700915 count = 0
Jon Hall5cf14d52015-07-16 12:15:19 -0700916 # Iter through intents of a node
917 try:
918 for intent in json.loads( intents ):
919 state = intent.get( 'state', None )
920 if "INSTALLED" not in state:
921 installedCheck = False
922 intentId = intent.get( 'id', None )
923 intentStates.append( ( intentId, state ) )
924 except ( ValueError, TypeError ):
925 main.log.exception( "Error parsing intents." )
926 # Print states
927 intentStates.sort()
928 for i, s in intentStates:
929 count += 1
930 main.log.info( "%-6s%-15s%-15s" %
931 ( str( count ), str( i ), str( s ) ) )
932 if not installedCheck:
933 time.sleep( 1 )
934 loopCount += 1
935 utilities.assert_equals( expect=True, actual=installedCheck,
936 onpass="Intents are all INSTALLED",
937 onfail="Intents are not all in " +
938 "INSTALLED state" )
939
940 main.step( "Check leadership of topics" )
941 leaders = main.ONOScli1.leaders()
942 topicCheck = main.TRUE
943 try:
944 if leaders:
945 parsedLeaders = json.loads( leaders )
946 main.log.warn( json.dumps( parsedLeaders,
947 sort_keys=True,
948 indent=4,
949 separators=( ',', ': ' ) ) )
950 # check for all intent partitions
951 # check for election
952 # TODO: Look at Devices as topics now that it uses this system
953 topics = []
954 for i in range( 14 ):
955 topics.append( "intent-partition-" + str( i ) )
956 # FIXME: this should only be after we start the app
957 # FIXME: topics.append( "org.onosproject.election" )
958 # Print leaders output
959 main.log.debug( topics )
960 ONOStopics = [ j['topic'] for j in parsedLeaders ]
961 for topic in topics:
962 if topic not in ONOStopics:
963 main.log.error( "Error: " + topic +
964 " not in leaders" )
965 topicCheck = main.FALSE
966 else:
967 main.log.error( "leaders() returned None" )
968 topicCheck = main.FALSE
969 except ( ValueError, TypeError ):
970 topicCheck = main.FALSE
971 main.log.exception( "Error parsing leaders" )
972 main.log.error( repr( leaders ) )
973 # TODO: Check for a leader of these topics
974 # Check all nodes
975 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700976 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700977 response = node.leaders( jsonFormat=False)
978 main.log.warn( str( node.name ) + " leaders output: \n" +
979 str( response ) )
980
981 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
982 onpass="intent Partitions is in leaders",
983 onfail="Some topics were lost " )
984 # Print partitions
985 partitions = main.ONOScli1.partitions()
986 try:
987 if partitions :
988 parsedPartitions = json.loads( partitions )
989 main.log.warn( json.dumps( parsedPartitions,
990 sort_keys=True,
991 indent=4,
992 separators=( ',', ': ' ) ) )
993 # TODO check for a leader in all paritions
994 # TODO check for consistency among nodes
995 else:
996 main.log.error( "partitions() returned None" )
997 except ( ValueError, TypeError ):
998 main.log.exception( "Error parsing partitions" )
999 main.log.error( repr( partitions ) )
1000 # Print Pending Map
1001 pendingMap = main.ONOScli1.pendingMap()
1002 try:
1003 if pendingMap :
1004 parsedPending = json.loads( pendingMap )
1005 main.log.warn( json.dumps( parsedPending,
1006 sort_keys=True,
1007 indent=4,
1008 separators=( ',', ': ' ) ) )
1009 # TODO check something here?
1010 else:
1011 main.log.error( "pendingMap() returned None" )
1012 except ( ValueError, TypeError ):
1013 main.log.exception( "Error parsing pending map" )
1014 main.log.error( repr( pendingMap ) )
1015
1016 if not installedCheck:
1017 main.log.info( "Waiting 60 seconds to see if the state of " +
1018 "intents change" )
1019 time.sleep( 60 )
1020 # Print the intent states
1021 intents = main.ONOScli1.intents()
1022 intentStates = []
1023 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1024 count = 0
1025 # Iter through intents of a node
1026 try:
1027 for intent in json.loads( intents ):
1028 state = intent.get( 'state', None )
1029 if "INSTALLED" not in state:
1030 installedCheck = False
1031 intentId = intent.get( 'id', None )
1032 intentStates.append( ( intentId, state ) )
1033 except ( ValueError, TypeError ):
1034 main.log.exception( "Error parsing intents." )
1035 intentStates.sort()
1036 for i, s in intentStates:
1037 count += 1
1038 main.log.info( "%-6s%-15s%-15s" %
1039 ( str( count ), str( i ), str( s ) ) )
1040 leaders = main.ONOScli1.leaders()
1041 try:
1042 missing = False
1043 if leaders:
1044 parsedLeaders = json.loads( leaders )
1045 main.log.warn( json.dumps( parsedLeaders,
1046 sort_keys=True,
1047 indent=4,
1048 separators=( ',', ': ' ) ) )
1049 # check for all intent partitions
1050 # check for election
1051 topics = []
1052 for i in range( 14 ):
1053 topics.append( "intent-partition-" + str( i ) )
1054 # FIXME: this should only be after we start the app
1055 topics.append( "org.onosproject.election" )
1056 main.log.debug( topics )
1057 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1058 for topic in topics:
1059 if topic not in ONOStopics:
1060 main.log.error( "Error: " + topic +
1061 " not in leaders" )
1062 missing = True
1063 else:
1064 main.log.error( "leaders() returned None" )
1065 except ( ValueError, TypeError ):
1066 main.log.exception( "Error parsing leaders" )
1067 main.log.error( repr( leaders ) )
1068 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001069 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001070 response = node.leaders( jsonFormat=False)
1071 main.log.warn( str( node.name ) + " leaders output: \n" +
1072 str( response ) )
1073
1074 partitions = main.ONOScli1.partitions()
1075 try:
1076 if partitions :
1077 parsedPartitions = json.loads( partitions )
1078 main.log.warn( json.dumps( parsedPartitions,
1079 sort_keys=True,
1080 indent=4,
1081 separators=( ',', ': ' ) ) )
1082 # TODO check for a leader in all paritions
1083 # TODO check for consistency among nodes
1084 else:
1085 main.log.error( "partitions() returned None" )
1086 except ( ValueError, TypeError ):
1087 main.log.exception( "Error parsing partitions" )
1088 main.log.error( repr( partitions ) )
1089 pendingMap = main.ONOScli1.pendingMap()
1090 try:
1091 if pendingMap :
1092 parsedPending = json.loads( pendingMap )
1093 main.log.warn( json.dumps( parsedPending,
1094 sort_keys=True,
1095 indent=4,
1096 separators=( ',', ': ' ) ) )
1097 # TODO check something here?
1098 else:
1099 main.log.error( "pendingMap() returned None" )
1100 except ( ValueError, TypeError ):
1101 main.log.exception( "Error parsing pending map" )
1102 main.log.error( repr( pendingMap ) )
1103 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001104 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001105 main.step( "Wait a minute then ping again" )
1106 # the wait is above
1107 PingResult = main.TRUE
1108 for i in range( 8, 18 ):
1109 ping = main.Mininet1.pingHost( src="h" + str( i ),
1110 target="h" + str( i + 10 ) )
1111 PingResult = PingResult and ping
1112 if ping == main.FALSE:
1113 main.log.warn( "Ping failed between h" + str( i ) +
1114 " and h" + str( i + 10 ) )
1115 elif ping == main.TRUE:
1116 main.log.info( "Ping test passed!" )
1117 # Don't set PingResult or you'd override failures
1118 if PingResult == main.FALSE:
1119 main.log.error(
1120 "Intents have not been installed correctly, pings failed." )
1121 # TODO: pretty print
1122 main.log.warn( "ONOS1 intents: " )
1123 try:
1124 tmpIntents = main.ONOScli1.intents()
1125 main.log.warn( json.dumps( json.loads( tmpIntents ),
1126 sort_keys=True,
1127 indent=4,
1128 separators=( ',', ': ' ) ) )
1129 except ( ValueError, TypeError ):
1130 main.log.warn( repr( tmpIntents ) )
1131 utilities.assert_equals(
1132 expect=main.TRUE,
1133 actual=PingResult,
1134 onpass="Intents have been installed correctly and pings work",
1135 onfail="Intents have not been installed correctly, pings failed." )
1136
1137 def CASE5( self, main ):
1138 """
1139 Reading state of ONOS
1140 """
1141 import json
1142 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001143 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001144 assert main, "main not defined"
1145 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001146 assert main.CLIs, "main.CLIs not defined"
1147 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001148
1149 main.case( "Setting up and gathering data for current state" )
1150 # The general idea for this test case is to pull the state of
1151 # ( intents,flows, topology,... ) from each ONOS node
1152 # We can then compare them with each other and also with past states
1153
1154 main.step( "Check that each switch has a master" )
1155 global mastershipState
1156 mastershipState = '[]'
1157
1158 # Assert that each device has a master
1159 rolesNotNull = main.TRUE
1160 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001161 for i in range( main.numCtrls ):
1162 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001163 name="rolesNotNull-" + str( i ),
1164 args=[] )
1165 threads.append( t )
1166 t.start()
1167
1168 for t in threads:
1169 t.join()
1170 rolesNotNull = rolesNotNull and t.result
1171 utilities.assert_equals(
1172 expect=main.TRUE,
1173 actual=rolesNotNull,
1174 onpass="Each device has a master",
1175 onfail="Some devices don't have a master assigned" )
1176
1177 main.step( "Get the Mastership of each switch from each controller" )
1178 ONOSMastership = []
1179 mastershipCheck = main.FALSE
1180 consistentMastership = True
1181 rolesResults = True
1182 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001183 for i in range( main.numCtrls ):
1184 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001185 name="roles-" + str( i ),
1186 args=[] )
1187 threads.append( t )
1188 t.start()
1189
1190 for t in threads:
1191 t.join()
1192 ONOSMastership.append( t.result )
1193
Jon Halle1a3b752015-07-22 13:02:46 -07001194 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001195 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1196 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1197 " roles" )
1198 main.log.warn(
1199 "ONOS" + str( i + 1 ) + " mastership response: " +
1200 repr( ONOSMastership[i] ) )
1201 rolesResults = False
1202 utilities.assert_equals(
1203 expect=True,
1204 actual=rolesResults,
1205 onpass="No error in reading roles output",
1206 onfail="Error in reading roles from ONOS" )
1207
1208 main.step( "Check for consistency in roles from each controller" )
1209 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1210 main.log.info(
1211 "Switch roles are consistent across all ONOS nodes" )
1212 else:
1213 consistentMastership = False
1214 utilities.assert_equals(
1215 expect=True,
1216 actual=consistentMastership,
1217 onpass="Switch roles are consistent across all ONOS nodes",
1218 onfail="ONOS nodes have different views of switch roles" )
1219
1220 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001221 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001222 try:
1223 main.log.warn(
1224 "ONOS" + str( i + 1 ) + " roles: ",
1225 json.dumps(
1226 json.loads( ONOSMastership[ i ] ),
1227 sort_keys=True,
1228 indent=4,
1229 separators=( ',', ': ' ) ) )
1230 except ( ValueError, TypeError ):
1231 main.log.warn( repr( ONOSMastership[ i ] ) )
1232 elif rolesResults and consistentMastership:
1233 mastershipCheck = main.TRUE
1234 mastershipState = ONOSMastership[ 0 ]
1235
1236 main.step( "Get the intents from each controller" )
1237 global intentState
1238 intentState = []
1239 ONOSIntents = []
1240 intentCheck = main.FALSE
1241 consistentIntents = True
1242 intentsResults = True
1243 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001244 for i in range( main.numCtrls ):
1245 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001246 name="intents-" + str( i ),
1247 args=[],
1248 kwargs={ 'jsonFormat': True } )
1249 threads.append( t )
1250 t.start()
1251
1252 for t in threads:
1253 t.join()
1254 ONOSIntents.append( t.result )
1255
Jon Halle1a3b752015-07-22 13:02:46 -07001256 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001257 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1258 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1259 " intents" )
1260 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1261 repr( ONOSIntents[ i ] ) )
1262 intentsResults = False
1263 utilities.assert_equals(
1264 expect=True,
1265 actual=intentsResults,
1266 onpass="No error in reading intents output",
1267 onfail="Error in reading intents from ONOS" )
1268
1269 main.step( "Check for consistency in Intents from each controller" )
1270 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1271 main.log.info( "Intents are consistent across all ONOS " +
1272 "nodes" )
1273 else:
1274 consistentIntents = False
1275 main.log.error( "Intents not consistent" )
1276 utilities.assert_equals(
1277 expect=True,
1278 actual=consistentIntents,
1279 onpass="Intents are consistent across all ONOS nodes",
1280 onfail="ONOS nodes have different views of intents" )
1281
1282 if intentsResults:
1283 # Try to make it easy to figure out what is happening
1284 #
1285 # Intent ONOS1 ONOS2 ...
1286 # 0x01 INSTALLED INSTALLING
1287 # ... ... ...
1288 # ... ... ...
1289 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001290 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001291 title += " " * 10 + "ONOS" + str( n + 1 )
1292 main.log.warn( title )
1293 # get all intent keys in the cluster
1294 keys = []
1295 for nodeStr in ONOSIntents:
1296 node = json.loads( nodeStr )
1297 for intent in node:
1298 keys.append( intent.get( 'id' ) )
1299 keys = set( keys )
1300 for key in keys:
1301 row = "%-13s" % key
1302 for nodeStr in ONOSIntents:
1303 node = json.loads( nodeStr )
1304 for intent in node:
1305 if intent.get( 'id', "Error" ) == key:
1306 row += "%-15s" % intent.get( 'state' )
1307 main.log.warn( row )
1308 # End table view
1309
1310 if intentsResults and not consistentIntents:
1311 # print the json objects
1312 n = len(ONOSIntents)
1313 main.log.debug( "ONOS" + str( n ) + " intents: " )
1314 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1315 sort_keys=True,
1316 indent=4,
1317 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001318 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001319 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1320 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1321 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1322 sort_keys=True,
1323 indent=4,
1324 separators=( ',', ': ' ) ) )
1325 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001326 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001327 str( n ) + " intents" )
1328 elif intentsResults and consistentIntents:
1329 intentCheck = main.TRUE
1330 intentState = ONOSIntents[ 0 ]
1331
1332 main.step( "Get the flows from each controller" )
1333 global flowState
1334 flowState = []
1335 ONOSFlows = []
1336 ONOSFlowsJson = []
1337 flowCheck = main.FALSE
1338 consistentFlows = True
1339 flowsResults = True
1340 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001341 for i in range( main.numCtrls ):
1342 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001343 name="flows-" + str( i ),
1344 args=[],
1345 kwargs={ 'jsonFormat': True } )
1346 threads.append( t )
1347 t.start()
1348
1349 # NOTE: Flows command can take some time to run
1350 time.sleep(30)
1351 for t in threads:
1352 t.join()
1353 result = t.result
1354 ONOSFlows.append( result )
1355
Jon Halle1a3b752015-07-22 13:02:46 -07001356 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001357 num = str( i + 1 )
1358 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1359 main.log.error( "Error in getting ONOS" + num + " flows" )
1360 main.log.warn( "ONOS" + num + " flows response: " +
1361 repr( ONOSFlows[ i ] ) )
1362 flowsResults = False
1363 ONOSFlowsJson.append( None )
1364 else:
1365 try:
1366 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1367 except ( ValueError, TypeError ):
1368 # FIXME: change this to log.error?
1369 main.log.exception( "Error in parsing ONOS" + num +
1370 " response as json." )
1371 main.log.error( repr( ONOSFlows[ i ] ) )
1372 ONOSFlowsJson.append( None )
1373 flowsResults = False
1374 utilities.assert_equals(
1375 expect=True,
1376 actual=flowsResults,
1377 onpass="No error in reading flows output",
1378 onfail="Error in reading flows from ONOS" )
1379
1380 main.step( "Check for consistency in Flows from each controller" )
1381 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1382 if all( tmp ):
1383 main.log.info( "Flow count is consistent across all ONOS nodes" )
1384 else:
1385 consistentFlows = False
1386 utilities.assert_equals(
1387 expect=True,
1388 actual=consistentFlows,
1389 onpass="The flow count is consistent across all ONOS nodes",
1390 onfail="ONOS nodes have different flow counts" )
1391
1392 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001393 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001394 try:
1395 main.log.warn(
1396 "ONOS" + str( i + 1 ) + " flows: " +
1397 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1398 indent=4, separators=( ',', ': ' ) ) )
1399 except ( ValueError, TypeError ):
1400 main.log.warn(
1401 "ONOS" + str( i + 1 ) + " flows: " +
1402 repr( ONOSFlows[ i ] ) )
1403 elif flowsResults and consistentFlows:
1404 flowCheck = main.TRUE
1405 flowState = ONOSFlows[ 0 ]
1406
1407 main.step( "Get the OF Table entries" )
1408 global flows
1409 flows = []
1410 for i in range( 1, 29 ):
Jon Hall9043c902015-07-30 14:23:44 -07001411 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001412 if flowCheck == main.FALSE:
1413 for table in flows:
1414 main.log.warn( table )
1415 # TODO: Compare switch flow tables with ONOS flow tables
1416
1417 main.step( "Start continuous pings" )
1418 main.Mininet2.pingLong(
1419 src=main.params[ 'PING' ][ 'source1' ],
1420 target=main.params[ 'PING' ][ 'target1' ],
1421 pingTime=500 )
1422 main.Mininet2.pingLong(
1423 src=main.params[ 'PING' ][ 'source2' ],
1424 target=main.params[ 'PING' ][ 'target2' ],
1425 pingTime=500 )
1426 main.Mininet2.pingLong(
1427 src=main.params[ 'PING' ][ 'source3' ],
1428 target=main.params[ 'PING' ][ 'target3' ],
1429 pingTime=500 )
1430 main.Mininet2.pingLong(
1431 src=main.params[ 'PING' ][ 'source4' ],
1432 target=main.params[ 'PING' ][ 'target4' ],
1433 pingTime=500 )
1434 main.Mininet2.pingLong(
1435 src=main.params[ 'PING' ][ 'source5' ],
1436 target=main.params[ 'PING' ][ 'target5' ],
1437 pingTime=500 )
1438 main.Mininet2.pingLong(
1439 src=main.params[ 'PING' ][ 'source6' ],
1440 target=main.params[ 'PING' ][ 'target6' ],
1441 pingTime=500 )
1442 main.Mininet2.pingLong(
1443 src=main.params[ 'PING' ][ 'source7' ],
1444 target=main.params[ 'PING' ][ 'target7' ],
1445 pingTime=500 )
1446 main.Mininet2.pingLong(
1447 src=main.params[ 'PING' ][ 'source8' ],
1448 target=main.params[ 'PING' ][ 'target8' ],
1449 pingTime=500 )
1450 main.Mininet2.pingLong(
1451 src=main.params[ 'PING' ][ 'source9' ],
1452 target=main.params[ 'PING' ][ 'target9' ],
1453 pingTime=500 )
1454 main.Mininet2.pingLong(
1455 src=main.params[ 'PING' ][ 'source10' ],
1456 target=main.params[ 'PING' ][ 'target10' ],
1457 pingTime=500 )
1458
1459 main.step( "Collecting topology information from ONOS" )
1460 devices = []
1461 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001462 for i in range( main.numCtrls ):
1463 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001464 name="devices-" + str( i ),
1465 args=[ ] )
1466 threads.append( t )
1467 t.start()
1468
1469 for t in threads:
1470 t.join()
1471 devices.append( t.result )
1472 hosts = []
1473 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001474 for i in range( main.numCtrls ):
1475 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001476 name="hosts-" + str( i ),
1477 args=[ ] )
1478 threads.append( t )
1479 t.start()
1480
1481 for t in threads:
1482 t.join()
1483 try:
1484 hosts.append( json.loads( t.result ) )
1485 except ( ValueError, TypeError ):
1486 # FIXME: better handling of this, print which node
1487 # Maybe use thread name?
1488 main.log.exception( "Error parsing json output of hosts" )
1489 # FIXME: should this be an empty json object instead?
1490 hosts.append( None )
1491
1492 ports = []
1493 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001494 for i in range( main.numCtrls ):
1495 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001496 name="ports-" + str( i ),
1497 args=[ ] )
1498 threads.append( t )
1499 t.start()
1500
1501 for t in threads:
1502 t.join()
1503 ports.append( t.result )
1504 links = []
1505 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001506 for i in range( main.numCtrls ):
1507 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001508 name="links-" + str( i ),
1509 args=[ ] )
1510 threads.append( t )
1511 t.start()
1512
1513 for t in threads:
1514 t.join()
1515 links.append( t.result )
1516 clusters = []
1517 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001518 for i in range( main.numCtrls ):
1519 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001520 name="clusters-" + str( i ),
1521 args=[ ] )
1522 threads.append( t )
1523 t.start()
1524
1525 for t in threads:
1526 t.join()
1527 clusters.append( t.result )
1528 # Compare json objects for hosts and dataplane clusters
1529
1530 # hosts
1531 main.step( "Host view is consistent across ONOS nodes" )
1532 consistentHostsResult = main.TRUE
1533 for controller in range( len( hosts ) ):
1534 controllerStr = str( controller + 1 )
1535 if "Error" not in hosts[ controller ]:
1536 if hosts[ controller ] == hosts[ 0 ]:
1537 continue
1538 else: # hosts not consistent
1539 main.log.error( "hosts from ONOS" +
1540 controllerStr +
1541 " is inconsistent with ONOS1" )
1542 main.log.warn( repr( hosts[ controller ] ) )
1543 consistentHostsResult = main.FALSE
1544
1545 else:
1546 main.log.error( "Error in getting ONOS hosts from ONOS" +
1547 controllerStr )
1548 consistentHostsResult = main.FALSE
1549 main.log.warn( "ONOS" + controllerStr +
1550 " hosts response: " +
1551 repr( hosts[ controller ] ) )
1552 utilities.assert_equals(
1553 expect=main.TRUE,
1554 actual=consistentHostsResult,
1555 onpass="Hosts view is consistent across all ONOS nodes",
1556 onfail="ONOS nodes have different views of hosts" )
1557
1558 main.step( "Each host has an IP address" )
1559 ipResult = main.TRUE
1560 for controller in range( 0, len( hosts ) ):
1561 controllerStr = str( controller + 1 )
1562 for host in hosts[ controller ]:
1563 if not host.get( 'ipAddresses', [ ] ):
1564 main.log.error( "DEBUG:Error with host ips on controller" +
1565 controllerStr + ": " + str( host ) )
1566 ipResult = main.FALSE
1567 utilities.assert_equals(
1568 expect=main.TRUE,
1569 actual=ipResult,
1570 onpass="The ips of the hosts aren't empty",
1571 onfail="The ip of at least one host is missing" )
1572
1573 # Strongly connected clusters of devices
1574 main.step( "Cluster view is consistent across ONOS nodes" )
1575 consistentClustersResult = main.TRUE
1576 for controller in range( len( clusters ) ):
1577 controllerStr = str( controller + 1 )
1578 if "Error" not in clusters[ controller ]:
1579 if clusters[ controller ] == clusters[ 0 ]:
1580 continue
1581 else: # clusters not consistent
1582 main.log.error( "clusters from ONOS" + controllerStr +
1583 " is inconsistent with ONOS1" )
1584 consistentClustersResult = main.FALSE
1585
1586 else:
1587 main.log.error( "Error in getting dataplane clusters " +
1588 "from ONOS" + controllerStr )
1589 consistentClustersResult = main.FALSE
1590 main.log.warn( "ONOS" + controllerStr +
1591 " clusters response: " +
1592 repr( clusters[ controller ] ) )
1593 utilities.assert_equals(
1594 expect=main.TRUE,
1595 actual=consistentClustersResult,
1596 onpass="Clusters view is consistent across all ONOS nodes",
1597 onfail="ONOS nodes have different views of clusters" )
1598 # there should always only be one cluster
1599 main.step( "Cluster view correct across ONOS nodes" )
1600 try:
1601 numClusters = len( json.loads( clusters[ 0 ] ) )
1602 except ( ValueError, TypeError ):
1603 main.log.exception( "Error parsing clusters[0]: " +
1604 repr( clusters[ 0 ] ) )
1605 clusterResults = main.FALSE
1606 if numClusters == 1:
1607 clusterResults = main.TRUE
1608 utilities.assert_equals(
1609 expect=1,
1610 actual=numClusters,
1611 onpass="ONOS shows 1 SCC",
1612 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1613
1614 main.step( "Comparing ONOS topology to MN" )
1615 devicesResults = main.TRUE
1616 linksResults = main.TRUE
1617 hostsResults = main.TRUE
1618 mnSwitches = main.Mininet1.getSwitches()
1619 mnLinks = main.Mininet1.getLinks()
1620 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001621 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001622 controllerStr = str( controller + 1 )
1623 if devices[ controller ] and ports[ controller ] and\
1624 "Error" not in devices[ controller ] and\
1625 "Error" not in ports[ controller ]:
Jon Halle1a3b752015-07-22 13:02:46 -07001626 currentDevicesResult = main.Mininet1.compareSwitches(
1627 mnSwitches,
1628 json.loads( devices[ controller ] ),
1629 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001630 else:
1631 currentDevicesResult = main.FALSE
1632 utilities.assert_equals( expect=main.TRUE,
1633 actual=currentDevicesResult,
1634 onpass="ONOS" + controllerStr +
1635 " Switches view is correct",
1636 onfail="ONOS" + controllerStr +
1637 " Switches view is incorrect" )
1638 if links[ controller ] and "Error" not in links[ controller ]:
1639 currentLinksResult = main.Mininet1.compareLinks(
1640 mnSwitches, mnLinks,
1641 json.loads( links[ controller ] ) )
1642 else:
1643 currentLinksResult = main.FALSE
1644 utilities.assert_equals( expect=main.TRUE,
1645 actual=currentLinksResult,
1646 onpass="ONOS" + controllerStr +
1647 " links view is correct",
1648 onfail="ONOS" + controllerStr +
1649 " links view is incorrect" )
1650
1651 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1652 currentHostsResult = main.Mininet1.compareHosts(
1653 mnHosts,
1654 hosts[ controller ] )
1655 else:
1656 currentHostsResult = main.FALSE
1657 utilities.assert_equals( expect=main.TRUE,
1658 actual=currentHostsResult,
1659 onpass="ONOS" + controllerStr +
1660 " hosts exist in Mininet",
1661 onfail="ONOS" + controllerStr +
1662 " hosts don't match Mininet" )
1663
1664 devicesResults = devicesResults and currentDevicesResult
1665 linksResults = linksResults and currentLinksResult
1666 hostsResults = hostsResults and currentHostsResult
1667
1668 main.step( "Device information is correct" )
1669 utilities.assert_equals(
1670 expect=main.TRUE,
1671 actual=devicesResults,
1672 onpass="Device information is correct",
1673 onfail="Device information is incorrect" )
1674
1675 main.step( "Links are correct" )
1676 utilities.assert_equals(
1677 expect=main.TRUE,
1678 actual=linksResults,
1679 onpass="Link are correct",
1680 onfail="Links are incorrect" )
1681
1682 main.step( "Hosts are correct" )
1683 utilities.assert_equals(
1684 expect=main.TRUE,
1685 actual=hostsResults,
1686 onpass="Hosts are correct",
1687 onfail="Hosts are incorrect" )
1688
1689 def CASE6( self, main ):
1690 """
1691 The Failure case.
1692 """
1693 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001694 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001695 assert main, "main not defined"
1696 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001697 assert main.CLIs, "main.CLIs not defined"
1698 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001699 try:
1700 labels
1701 except NameError:
1702 main.log.error( "labels not defined, setting to []" )
1703 global labels
1704 labels = []
1705 try:
1706 data
1707 except NameError:
1708 main.log.error( "data not defined, setting to []" )
1709 global data
1710 data = []
1711 # Reset non-persistent variables
1712 try:
1713 iCounterValue = 0
1714 except NameError:
1715 main.log.error( "iCounterValue not defined, setting to 0" )
1716 iCounterValue = 0
1717
1718 main.case( "Restart entire ONOS cluster" )
1719
1720 main.step( "Killing ONOS nodes" )
1721 killResults = main.TRUE
1722 killTime = time.time()
Jon Halle1a3b752015-07-22 13:02:46 -07001723 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001724 killed = main.ONOSbench.onosKill( node.ip_address )
1725 killResults = killResults and killed
1726 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1727 onpass="ONOS nodes killed",
1728 onfail="ONOS kill unsuccessful" )
1729
1730 main.step( "Checking if ONOS is up yet" )
1731 for i in range( 2 ):
1732 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001733 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001734 started = main.ONOSbench.isup( node.ip_address )
1735 if not started:
1736 main.log.error( node.name + " didn't start!" )
1737 onosIsupResult = onosIsupResult and started
1738 if onosIsupResult == main.TRUE:
1739 break
1740 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1741 onpass="ONOS restarted",
1742 onfail="ONOS restart NOT successful" )
1743
1744 main.log.step( "Starting ONOS CLI sessions" )
1745 cliResults = main.TRUE
1746 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001747 for i in range( main.numCtrls ):
1748 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -07001749 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -07001750 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001751 threads.append( t )
1752 t.start()
1753
1754 for t in threads:
1755 t.join()
1756 cliResults = cliResults and t.result
1757 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1758 onpass="ONOS cli started",
1759 onfail="ONOS clis did not restart" )
1760
1761 # Grab the time of restart so we chan check how long the gossip
1762 # protocol has had time to work
1763 main.restartTime = time.time() - killTime
1764 main.log.debug( "Restart time: " + str( main.restartTime ) )
1765 labels.append( "Restart" )
1766 data.append( str( main.restartTime ) )
1767
1768 # FIXME: revisit test plan for election with madan
1769 # Rerun for election on restarted nodes
1770 runResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001771 for cli in main.CLIs:
1772 run = main.CLIs[0].electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001773 if run != main.TRUE:
1774 main.log.error( "Error running for election on " + cli.name )
1775 runResults = runResults and run
1776 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1777 onpass="Reran for election",
1778 onfail="Failed to rerun for election" )
1779
1780 # TODO: Make this configurable
1781 time.sleep( 60 )
Jon Halle1a3b752015-07-22 13:02:46 -07001782 main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) )
1783 main.log.debug( main.CLIs[0].leaders( jsonFormat=False ) )
1784 main.log.debug( main.CLIs[0].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001785
1786 def CASE7( self, main ):
1787 """
1788 Check state after ONOS failure
1789 """
1790 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001791 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001792 assert main, "main not defined"
1793 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001794 assert main.CLIs, "main.CLIs not defined"
1795 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001796 main.case( "Running ONOS Constant State Tests" )
1797
1798 main.step( "Check that each switch has a master" )
1799 # Assert that each device has a master
1800 rolesNotNull = main.TRUE
1801 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001802 for i in range( main.numCtrls ):
1803 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001804 name="rolesNotNull-" + str( i ),
1805 args=[ ] )
1806 threads.append( t )
1807 t.start()
1808
1809 for t in threads:
1810 t.join()
1811 rolesNotNull = rolesNotNull and t.result
1812 utilities.assert_equals(
1813 expect=main.TRUE,
1814 actual=rolesNotNull,
1815 onpass="Each device has a master",
1816 onfail="Some devices don't have a master assigned" )
1817
1818 main.step( "Read device roles from ONOS" )
1819 ONOSMastership = []
1820 mastershipCheck = main.FALSE
1821 consistentMastership = True
1822 rolesResults = True
1823 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001824 for i in range( main.numCtrls ):
1825 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001826 name="roles-" + str( i ),
1827 args=[] )
1828 threads.append( t )
1829 t.start()
1830
1831 for t in threads:
1832 t.join()
1833 ONOSMastership.append( t.result )
1834
Jon Halle1a3b752015-07-22 13:02:46 -07001835 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001836 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1837 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1838 " roles" )
1839 main.log.warn(
1840 "ONOS" + str( i + 1 ) + " mastership response: " +
1841 repr( ONOSMastership[i] ) )
1842 rolesResults = False
1843 utilities.assert_equals(
1844 expect=True,
1845 actual=rolesResults,
1846 onpass="No error in reading roles output",
1847 onfail="Error in reading roles from ONOS" )
1848
1849 main.step( "Check for consistency in roles from each controller" )
1850 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1851 main.log.info(
1852 "Switch roles are consistent across all ONOS nodes" )
1853 else:
1854 consistentMastership = False
1855 utilities.assert_equals(
1856 expect=True,
1857 actual=consistentMastership,
1858 onpass="Switch roles are consistent across all ONOS nodes",
1859 onfail="ONOS nodes have different views of switch roles" )
1860
1861 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001862 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001863 main.log.warn(
1864 "ONOS" + str( i + 1 ) + " roles: ",
1865 json.dumps(
1866 json.loads( ONOSMastership[ i ] ),
1867 sort_keys=True,
1868 indent=4,
1869 separators=( ',', ': ' ) ) )
1870 elif rolesResults and not consistentMastership:
1871 mastershipCheck = main.TRUE
1872
1873 '''
1874 description2 = "Compare switch roles from before failure"
1875 main.step( description2 )
1876 try:
1877 currentJson = json.loads( ONOSMastership[0] )
1878 oldJson = json.loads( mastershipState )
1879 except ( ValueError, TypeError ):
1880 main.log.exception( "Something is wrong with parsing " +
1881 "ONOSMastership[0] or mastershipState" )
1882 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1883 main.log.error( "mastershipState" + repr( mastershipState ) )
1884 main.cleanup()
1885 main.exit()
1886 mastershipCheck = main.TRUE
1887 for i in range( 1, 29 ):
1888 switchDPID = str(
1889 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1890 current = [ switch[ 'master' ] for switch in currentJson
1891 if switchDPID in switch[ 'id' ] ]
1892 old = [ switch[ 'master' ] for switch in oldJson
1893 if switchDPID in switch[ 'id' ] ]
1894 if current == old:
1895 mastershipCheck = mastershipCheck and main.TRUE
1896 else:
1897 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1898 mastershipCheck = main.FALSE
1899 utilities.assert_equals(
1900 expect=main.TRUE,
1901 actual=mastershipCheck,
1902 onpass="Mastership of Switches was not changed",
1903 onfail="Mastership of some switches changed" )
1904 '''
1905 # NOTE: we expect mastership to change on controller failure
1906
1907 main.step( "Get the intents and compare across all nodes" )
1908 ONOSIntents = []
1909 intentCheck = main.FALSE
1910 consistentIntents = True
1911 intentsResults = True
1912 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001913 for i in range( main.numCtrls ):
1914 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001915 name="intents-" + str( i ),
1916 args=[],
1917 kwargs={ 'jsonFormat': True } )
1918 threads.append( t )
1919 t.start()
1920
1921 for t in threads:
1922 t.join()
1923 ONOSIntents.append( t.result )
1924
Jon Halle1a3b752015-07-22 13:02:46 -07001925 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001926 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1927 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1928 " intents" )
1929 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1930 repr( ONOSIntents[ i ] ) )
1931 intentsResults = False
1932 utilities.assert_equals(
1933 expect=True,
1934 actual=intentsResults,
1935 onpass="No error in reading intents output",
1936 onfail="Error in reading intents from ONOS" )
1937
1938 main.step( "Check for consistency in Intents from each controller" )
1939 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1940 main.log.info( "Intents are consistent across all ONOS " +
1941 "nodes" )
1942 else:
1943 consistentIntents = False
1944
1945 # Try to make it easy to figure out what is happening
1946 #
1947 # Intent ONOS1 ONOS2 ...
1948 # 0x01 INSTALLED INSTALLING
1949 # ... ... ...
1950 # ... ... ...
1951 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001952 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001953 title += " " * 10 + "ONOS" + str( n + 1 )
1954 main.log.warn( title )
1955 # get all intent keys in the cluster
1956 keys = []
1957 for nodeStr in ONOSIntents:
1958 node = json.loads( nodeStr )
1959 for intent in node:
1960 keys.append( intent.get( 'id' ) )
1961 keys = set( keys )
1962 for key in keys:
1963 row = "%-13s" % key
1964 for nodeStr in ONOSIntents:
1965 node = json.loads( nodeStr )
1966 for intent in node:
1967 if intent.get( 'id' ) == key:
1968 row += "%-15s" % intent.get( 'state' )
1969 main.log.warn( row )
1970 # End table view
1971
1972 utilities.assert_equals(
1973 expect=True,
1974 actual=consistentIntents,
1975 onpass="Intents are consistent across all ONOS nodes",
1976 onfail="ONOS nodes have different views of intents" )
1977 intentStates = []
1978 for node in ONOSIntents: # Iter through ONOS nodes
1979 nodeStates = []
1980 # Iter through intents of a node
1981 try:
1982 for intent in json.loads( node ):
1983 nodeStates.append( intent[ 'state' ] )
1984 except ( ValueError, TypeError ):
1985 main.log.exception( "Error in parsing intents" )
1986 main.log.error( repr( node ) )
1987 intentStates.append( nodeStates )
1988 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1989 main.log.info( dict( out ) )
1990
1991 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001992 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001993 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1994 main.log.warn( json.dumps(
1995 json.loads( ONOSIntents[ i ] ),
1996 sort_keys=True,
1997 indent=4,
1998 separators=( ',', ': ' ) ) )
1999 elif intentsResults and consistentIntents:
2000 intentCheck = main.TRUE
2001
2002 # NOTE: Store has no durability, so intents are lost across system
2003 # restarts
2004 """
2005 main.step( "Compare current intents with intents before the failure" )
2006 # NOTE: this requires case 5 to pass for intentState to be set.
2007 # maybe we should stop the test if that fails?
2008 sameIntents = main.FALSE
2009 if intentState and intentState == ONOSIntents[ 0 ]:
2010 sameIntents = main.TRUE
2011 main.log.info( "Intents are consistent with before failure" )
2012 # TODO: possibly the states have changed? we may need to figure out
2013 # what the acceptable states are
2014 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2015 sameIntents = main.TRUE
2016 try:
2017 before = json.loads( intentState )
2018 after = json.loads( ONOSIntents[ 0 ] )
2019 for intent in before:
2020 if intent not in after:
2021 sameIntents = main.FALSE
2022 main.log.debug( "Intent is not currently in ONOS " +
2023 "(at least in the same form):" )
2024 main.log.debug( json.dumps( intent ) )
2025 except ( ValueError, TypeError ):
2026 main.log.exception( "Exception printing intents" )
2027 main.log.debug( repr( ONOSIntents[0] ) )
2028 main.log.debug( repr( intentState ) )
2029 if sameIntents == main.FALSE:
2030 try:
2031 main.log.debug( "ONOS intents before: " )
2032 main.log.debug( json.dumps( json.loads( intentState ),
2033 sort_keys=True, indent=4,
2034 separators=( ',', ': ' ) ) )
2035 main.log.debug( "Current ONOS intents: " )
2036 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2037 sort_keys=True, indent=4,
2038 separators=( ',', ': ' ) ) )
2039 except ( ValueError, TypeError ):
2040 main.log.exception( "Exception printing intents" )
2041 main.log.debug( repr( ONOSIntents[0] ) )
2042 main.log.debug( repr( intentState ) )
2043 utilities.assert_equals(
2044 expect=main.TRUE,
2045 actual=sameIntents,
2046 onpass="Intents are consistent with before failure",
2047 onfail="The Intents changed during failure" )
2048 intentCheck = intentCheck and sameIntents
2049 """
2050 main.step( "Get the OF Table entries and compare to before " +
2051 "component failure" )
2052 FlowTables = main.TRUE
2053 flows2 = []
2054 for i in range( 28 ):
2055 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall9043c902015-07-30 14:23:44 -07002056 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002057 flows2.append( tmpFlows )
Jon Hall9043c902015-07-30 14:23:44 -07002058 tempResult = main.Mininet1.flowComp(
Jon Hall5cf14d52015-07-16 12:15:19 -07002059 flow1=flows[ i ],
2060 flow2=tmpFlows )
2061 FlowTables = FlowTables and tempResult
2062 if FlowTables == main.FALSE:
2063 main.log.info( "Differences in flow table for switch: s" +
2064 str( i + 1 ) )
2065 utilities.assert_equals(
2066 expect=main.TRUE,
2067 actual=FlowTables,
2068 onpass="No changes were found in the flow tables",
2069 onfail="Changes were found in the flow tables" )
2070
2071 main.Mininet2.pingLongKill()
2072 '''
2073 # main.step( "Check the continuous pings to ensure that no packets " +
2074 # "were dropped during component failure" )
2075 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2076 main.params[ 'TESTONIP' ] )
2077 LossInPings = main.FALSE
2078 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2079 for i in range( 8, 18 ):
2080 main.log.info(
2081 "Checking for a loss in pings along flow from s" +
2082 str( i ) )
2083 LossInPings = main.Mininet2.checkForLoss(
2084 "/tmp/ping.h" +
2085 str( i ) ) or LossInPings
2086 if LossInPings == main.TRUE:
2087 main.log.info( "Loss in ping detected" )
2088 elif LossInPings == main.ERROR:
2089 main.log.info( "There are multiple mininet process running" )
2090 elif LossInPings == main.FALSE:
2091 main.log.info( "No Loss in the pings" )
2092 main.log.info( "No loss of dataplane connectivity" )
2093 # utilities.assert_equals(
2094 # expect=main.FALSE,
2095 # actual=LossInPings,
2096 # onpass="No Loss of connectivity",
2097 # onfail="Loss of dataplane connectivity detected" )
2098
2099 # NOTE: Since intents are not persisted with IntnentStore,
2100 # we expect loss in dataplane connectivity
2101 LossInPings = main.FALSE
2102 '''
2103
2104 main.step( "Leadership Election is still functional" )
2105 # Test of LeadershipElection
2106 leaderList = []
2107 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002108 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002109 leaderN = cli.electionTestLeader()
2110 leaderList.append( leaderN )
2111 if leaderN == main.FALSE:
2112 # error in response
2113 main.log.error( "Something is wrong with " +
2114 "electionTestLeader function, check the" +
2115 " error logs" )
2116 leaderResult = main.FALSE
2117 elif leaderN is None:
2118 main.log.error( cli.name +
2119 " shows no leader for the election-app." )
2120 leaderResult = main.FALSE
2121 if len( set( leaderList ) ) != 1:
2122 leaderResult = main.FALSE
2123 main.log.error(
2124 "Inconsistent view of leader for the election test app" )
2125 # TODO: print the list
2126 utilities.assert_equals(
2127 expect=main.TRUE,
2128 actual=leaderResult,
2129 onpass="Leadership election passed",
2130 onfail="Something went wrong with Leadership election" )
2131
2132 def CASE8( self, main ):
2133 """
2134 Compare topo
2135 """
2136 import json
2137 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002138 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002139 assert main, "main not defined"
2140 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002141 assert main.CLIs, "main.CLIs not defined"
2142 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002143
2144 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002145 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002146 " and ONOS"
2147
2148 main.step( "Comparing ONOS topology to MN" )
2149 devicesResults = main.TRUE
2150 linksResults = main.TRUE
2151 hostsResults = main.TRUE
2152 hostAttachmentResults = True
2153 topoResult = main.FALSE
2154 elapsed = 0
2155 count = 0
2156 main.step( "Collecting topology information from ONOS" )
2157 startTime = time.time()
2158 # Give time for Gossip to work
2159 while topoResult == main.FALSE and elapsed < 60:
2160 count += 1
2161 cliStart = time.time()
2162 devices = []
2163 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002164 for i in range( main.numCtrls ):
2165 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002166 name="devices-" + str( i ),
2167 args=[ ] )
2168 threads.append( t )
2169 t.start()
2170
2171 for t in threads:
2172 t.join()
2173 devices.append( t.result )
2174 hosts = []
2175 ipResult = main.TRUE
2176 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002177 for i in range( main.numCtrls ):
2178 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07002179 name="hosts-" + str( i ),
2180 args=[ ] )
2181 threads.append( t )
2182 t.start()
2183
2184 for t in threads:
2185 t.join()
2186 try:
2187 hosts.append( json.loads( t.result ) )
2188 except ( ValueError, TypeError ):
2189 main.log.exception( "Error parsing hosts results" )
2190 main.log.error( repr( t.result ) )
2191 for controller in range( 0, len( hosts ) ):
2192 controllerStr = str( controller + 1 )
2193 for host in hosts[ controller ]:
2194 if host is None or host.get( 'ipAddresses', [] ) == []:
2195 main.log.error(
2196 "DEBUG:Error with host ipAddresses on controller" +
2197 controllerStr + ": " + str( host ) )
2198 ipResult = main.FALSE
2199 ports = []
2200 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002201 for i in range( main.numCtrls ):
2202 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002203 name="ports-" + str( i ),
2204 args=[ ] )
2205 threads.append( t )
2206 t.start()
2207
2208 for t in threads:
2209 t.join()
2210 ports.append( t.result )
2211 links = []
2212 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002213 for i in range( main.numCtrls ):
2214 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002215 name="links-" + str( i ),
2216 args=[ ] )
2217 threads.append( t )
2218 t.start()
2219
2220 for t in threads:
2221 t.join()
2222 links.append( t.result )
2223 clusters = []
2224 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002225 for i in range( main.numCtrls ):
2226 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002227 name="clusters-" + str( i ),
2228 args=[ ] )
2229 threads.append( t )
2230 t.start()
2231
2232 for t in threads:
2233 t.join()
2234 clusters.append( t.result )
2235
2236 elapsed = time.time() - startTime
2237 cliTime = time.time() - cliStart
2238 print "Elapsed time: " + str( elapsed )
2239 print "CLI time: " + str( cliTime )
2240
2241 mnSwitches = main.Mininet1.getSwitches()
2242 mnLinks = main.Mininet1.getLinks()
2243 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002244 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002245 controllerStr = str( controller + 1 )
2246 if devices[ controller ] and ports[ controller ] and\
2247 "Error" not in devices[ controller ] and\
2248 "Error" not in ports[ controller ]:
2249
2250 currentDevicesResult = main.Mininet1.compareSwitches(
2251 mnSwitches,
2252 json.loads( devices[ controller ] ),
2253 json.loads( ports[ controller ] ) )
2254 else:
2255 currentDevicesResult = main.FALSE
2256 utilities.assert_equals( expect=main.TRUE,
2257 actual=currentDevicesResult,
2258 onpass="ONOS" + controllerStr +
2259 " Switches view is correct",
2260 onfail="ONOS" + controllerStr +
2261 " Switches view is incorrect" )
2262
2263 if links[ controller ] and "Error" not in links[ controller ]:
2264 currentLinksResult = main.Mininet1.compareLinks(
2265 mnSwitches, mnLinks,
2266 json.loads( links[ controller ] ) )
2267 else:
2268 currentLinksResult = main.FALSE
2269 utilities.assert_equals( expect=main.TRUE,
2270 actual=currentLinksResult,
2271 onpass="ONOS" + controllerStr +
2272 " links view is correct",
2273 onfail="ONOS" + controllerStr +
2274 " links view is incorrect" )
2275
2276 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2277 currentHostsResult = main.Mininet1.compareHosts(
2278 mnHosts,
2279 hosts[ controller ] )
2280 else:
2281 currentHostsResult = main.FALSE
2282 utilities.assert_equals( expect=main.TRUE,
2283 actual=currentHostsResult,
2284 onpass="ONOS" + controllerStr +
2285 " hosts exist in Mininet",
2286 onfail="ONOS" + controllerStr +
2287 " hosts don't match Mininet" )
2288 # CHECKING HOST ATTACHMENT POINTS
2289 hostAttachment = True
2290 noHosts = False
2291 # FIXME: topo-HA/obelisk specific mappings:
2292 # key is mac and value is dpid
2293 mappings = {}
2294 for i in range( 1, 29 ): # hosts 1 through 28
2295 # set up correct variables:
2296 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2297 if i == 1:
2298 deviceId = "1000".zfill(16)
2299 elif i == 2:
2300 deviceId = "2000".zfill(16)
2301 elif i == 3:
2302 deviceId = "3000".zfill(16)
2303 elif i == 4:
2304 deviceId = "3004".zfill(16)
2305 elif i == 5:
2306 deviceId = "5000".zfill(16)
2307 elif i == 6:
2308 deviceId = "6000".zfill(16)
2309 elif i == 7:
2310 deviceId = "6007".zfill(16)
2311 elif i >= 8 and i <= 17:
2312 dpid = '3' + str( i ).zfill( 3 )
2313 deviceId = dpid.zfill(16)
2314 elif i >= 18 and i <= 27:
2315 dpid = '6' + str( i ).zfill( 3 )
2316 deviceId = dpid.zfill(16)
2317 elif i == 28:
2318 deviceId = "2800".zfill(16)
2319 mappings[ macId ] = deviceId
2320 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2321 if hosts[ controller ] == []:
2322 main.log.warn( "There are no hosts discovered" )
2323 noHosts = True
2324 else:
2325 for host in hosts[ controller ]:
2326 mac = None
2327 location = None
2328 device = None
2329 port = None
2330 try:
2331 mac = host.get( 'mac' )
2332 assert mac, "mac field could not be found for this host object"
2333
2334 location = host.get( 'location' )
2335 assert location, "location field could not be found for this host object"
2336
2337 # Trim the protocol identifier off deviceId
2338 device = str( location.get( 'elementId' ) ).split(':')[1]
2339 assert device, "elementId field could not be found for this host location object"
2340
2341 port = location.get( 'port' )
2342 assert port, "port field could not be found for this host location object"
2343
2344 # Now check if this matches where they should be
2345 if mac and device and port:
2346 if str( port ) != "1":
2347 main.log.error( "The attachment port is incorrect for " +
2348 "host " + str( mac ) +
2349 ". Expected: 1 Actual: " + str( port) )
2350 hostAttachment = False
2351 if device != mappings[ str( mac ) ]:
2352 main.log.error( "The attachment device is incorrect for " +
2353 "host " + str( mac ) +
2354 ". Expected: " + mappings[ str( mac ) ] +
2355 " Actual: " + device )
2356 hostAttachment = False
2357 else:
2358 hostAttachment = False
2359 except AssertionError:
2360 main.log.exception( "Json object not as expected" )
2361 main.log.error( repr( host ) )
2362 hostAttachment = False
2363 else:
2364 main.log.error( "No hosts json output or \"Error\"" +
2365 " in output. hosts = " +
2366 repr( hosts[ controller ] ) )
2367 if noHosts is False:
2368 # TODO: Find a way to know if there should be hosts in a
2369 # given point of the test
2370 hostAttachment = True
2371
2372 # END CHECKING HOST ATTACHMENT POINTS
2373 devicesResults = devicesResults and currentDevicesResult
2374 linksResults = linksResults and currentLinksResult
2375 hostsResults = hostsResults and currentHostsResult
2376 hostAttachmentResults = hostAttachmentResults and\
2377 hostAttachment
2378 topoResult = ( devicesResults and linksResults
2379 and hostsResults and ipResult and
2380 hostAttachmentResults )
2381
2382 # Compare json objects for hosts and dataplane clusters
2383
2384 # hosts
2385 main.step( "Hosts view is consistent across all ONOS nodes" )
2386 consistentHostsResult = main.TRUE
2387 for controller in range( len( hosts ) ):
2388 controllerStr = str( controller + 1 )
2389 if "Error" not in hosts[ controller ]:
2390 if hosts[ controller ] == hosts[ 0 ]:
2391 continue
2392 else: # hosts not consistent
2393 main.log.error( "hosts from ONOS" + controllerStr +
2394 " is inconsistent with ONOS1" )
2395 main.log.warn( repr( hosts[ controller ] ) )
2396 consistentHostsResult = main.FALSE
2397
2398 else:
2399 main.log.error( "Error in getting ONOS hosts from ONOS" +
2400 controllerStr )
2401 consistentHostsResult = main.FALSE
2402 main.log.warn( "ONOS" + controllerStr +
2403 " hosts response: " +
2404 repr( hosts[ controller ] ) )
2405 utilities.assert_equals(
2406 expect=main.TRUE,
2407 actual=consistentHostsResult,
2408 onpass="Hosts view is consistent across all ONOS nodes",
2409 onfail="ONOS nodes have different views of hosts" )
2410
2411 main.step( "Hosts information is correct" )
2412 hostsResults = hostsResults and ipResult
2413 utilities.assert_equals(
2414 expect=main.TRUE,
2415 actual=hostsResults,
2416 onpass="Host information is correct",
2417 onfail="Host information is incorrect" )
2418
2419 main.step( "Host attachment points to the network" )
2420 utilities.assert_equals(
2421 expect=True,
2422 actual=hostAttachmentResults,
2423 onpass="Hosts are correctly attached to the network",
2424 onfail="ONOS did not correctly attach hosts to the network" )
2425
2426 # Strongly connected clusters of devices
2427 main.step( "Clusters view is consistent across all ONOS nodes" )
2428 consistentClustersResult = main.TRUE
2429 for controller in range( len( clusters ) ):
2430 controllerStr = str( controller + 1 )
2431 if "Error" not in clusters[ controller ]:
2432 if clusters[ controller ] == clusters[ 0 ]:
2433 continue
2434 else: # clusters not consistent
2435 main.log.error( "clusters from ONOS" +
2436 controllerStr +
2437 " is inconsistent with ONOS1" )
2438 consistentClustersResult = main.FALSE
2439
2440 else:
2441 main.log.error( "Error in getting dataplane clusters " +
2442 "from ONOS" + controllerStr )
2443 consistentClustersResult = main.FALSE
2444 main.log.warn( "ONOS" + controllerStr +
2445 " clusters response: " +
2446 repr( clusters[ controller ] ) )
2447 utilities.assert_equals(
2448 expect=main.TRUE,
2449 actual=consistentClustersResult,
2450 onpass="Clusters view is consistent across all ONOS nodes",
2451 onfail="ONOS nodes have different views of clusters" )
2452
2453 main.step( "There is only one SCC" )
2454 # there should always only be one cluster
2455 try:
2456 numClusters = len( json.loads( clusters[ 0 ] ) )
2457 except ( ValueError, TypeError ):
2458 main.log.exception( "Error parsing clusters[0]: " +
2459 repr( clusters[0] ) )
2460 clusterResults = main.FALSE
2461 if numClusters == 1:
2462 clusterResults = main.TRUE
2463 utilities.assert_equals(
2464 expect=1,
2465 actual=numClusters,
2466 onpass="ONOS shows 1 SCC",
2467 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2468
2469 topoResult = ( devicesResults and linksResults
2470 and hostsResults and consistentHostsResult
2471 and consistentClustersResult and clusterResults
2472 and ipResult and hostAttachmentResults )
2473
2474 topoResult = topoResult and int( count <= 2 )
2475 note = "note it takes about " + str( int( cliTime ) ) + \
2476 " seconds for the test to make all the cli calls to fetch " +\
2477 "the topology from each ONOS instance"
2478 main.log.info(
2479 "Very crass estimate for topology discovery/convergence( " +
2480 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2481 str( count ) + " tries" )
2482
2483 main.step( "Device information is correct" )
2484 utilities.assert_equals(
2485 expect=main.TRUE,
2486 actual=devicesResults,
2487 onpass="Device information is correct",
2488 onfail="Device information is incorrect" )
2489
2490 main.step( "Links are correct" )
2491 utilities.assert_equals(
2492 expect=main.TRUE,
2493 actual=linksResults,
2494 onpass="Link are correct",
2495 onfail="Links are incorrect" )
2496
2497 # FIXME: move this to an ONOS state case
2498 main.step( "Checking ONOS nodes" )
2499 nodesOutput = []
2500 nodeResults = main.TRUE
2501 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002502 for i in range( main.numCtrls ):
2503 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002504 name="nodes-" + str( i ),
2505 args=[ ] )
2506 threads.append( t )
2507 t.start()
2508
2509 for t in threads:
2510 t.join()
2511 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002512 ips = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002513 for i in nodesOutput:
2514 try:
2515 current = json.loads( i )
2516 for node in current:
2517 currentResult = main.FALSE
2518 if node['ip'] in ips: # node in nodes() output is in cell
2519 if node['state'] == 'ACTIVE':
2520 currentResult = main.TRUE
2521 else:
2522 main.log.error( "Error in ONOS node availability" )
2523 main.log.error(
2524 json.dumps( current,
2525 sort_keys=True,
2526 indent=4,
2527 separators=( ',', ': ' ) ) )
2528 break
2529 nodeResults = nodeResults and currentResult
2530 except ( ValueError, TypeError ):
2531 main.log.error( "Error parsing nodes output" )
2532 main.log.warn( repr( i ) )
2533 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2534 onpass="Nodes check successful",
2535 onfail="Nodes check NOT successful" )
2536
2537 def CASE9( self, main ):
2538 """
2539 Link s3-s28 down
2540 """
2541 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002542 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002543 assert main, "main not defined"
2544 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002545 assert main.CLIs, "main.CLIs not defined"
2546 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002547 # NOTE: You should probably run a topology check after this
2548
2549 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2550
2551 description = "Turn off a link to ensure that Link Discovery " +\
2552 "is working properly"
2553 main.case( description )
2554
2555 main.step( "Kill Link between s3 and s28" )
2556 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2557 main.log.info( "Waiting " + str( linkSleep ) +
2558 " seconds for link down to be discovered" )
2559 time.sleep( linkSleep )
2560 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2561 onpass="Link down successful",
2562 onfail="Failed to bring link down" )
2563 # TODO do some sort of check here
2564
2565 def CASE10( self, main ):
2566 """
2567 Link s3-s28 up
2568 """
2569 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002570 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002571 assert main, "main not defined"
2572 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002573 assert main.CLIs, "main.CLIs not defined"
2574 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002575 # NOTE: You should probably run a topology check after this
2576
2577 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2578
2579 description = "Restore a link to ensure that Link Discovery is " + \
2580 "working properly"
2581 main.case( description )
2582
2583 main.step( "Bring link between s3 and s28 back up" )
2584 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2585 main.log.info( "Waiting " + str( linkSleep ) +
2586 " seconds for link up to be discovered" )
2587 time.sleep( linkSleep )
2588 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2589 onpass="Link up successful",
2590 onfail="Failed to bring link up" )
2591 # TODO do some sort of check here
2592
2593 def CASE11( self, main ):
2594 """
2595 Switch Down
2596 """
2597 # NOTE: You should probably run a topology check after this
2598 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002599 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002600 assert main, "main not defined"
2601 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002602 assert main.CLIs, "main.CLIs not defined"
2603 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002604
2605 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2606
2607 description = "Killing a switch to ensure it is discovered correctly"
2608 main.case( description )
2609 switch = main.params[ 'kill' ][ 'switch' ]
2610 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2611
2612 # TODO: Make this switch parameterizable
2613 main.step( "Kill " + switch )
2614 main.log.info( "Deleting " + switch )
2615 main.Mininet1.delSwitch( switch )
2616 main.log.info( "Waiting " + str( switchSleep ) +
2617 " seconds for switch down to be discovered" )
2618 time.sleep( switchSleep )
2619 device = main.ONOScli1.getDevice( dpid=switchDPID )
2620 # Peek at the deleted switch
2621 main.log.warn( str( device ) )
2622 result = main.FALSE
2623 if device and device[ 'available' ] is False:
2624 result = main.TRUE
2625 utilities.assert_equals( expect=main.TRUE, actual=result,
2626 onpass="Kill switch successful",
2627 onfail="Failed to kill switch?" )
2628
2629 def CASE12( self, main ):
2630 """
2631 Switch Up
2632 """
2633 # NOTE: You should probably run a topology check after this
2634 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002635 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002636 assert main, "main not defined"
2637 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002638 assert main.CLIs, "main.CLIs not defined"
2639 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002640 assert ONOS1Port, "ONOS1Port not defined"
2641 assert ONOS2Port, "ONOS2Port not defined"
2642 assert ONOS3Port, "ONOS3Port not defined"
2643 assert ONOS4Port, "ONOS4Port not defined"
2644 assert ONOS5Port, "ONOS5Port not defined"
2645 assert ONOS6Port, "ONOS6Port not defined"
2646 assert ONOS7Port, "ONOS7Port not defined"
2647
2648 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2649 switch = main.params[ 'kill' ][ 'switch' ]
2650 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2651 links = main.params[ 'kill' ][ 'links' ].split()
2652 description = "Adding a switch to ensure it is discovered correctly"
2653 main.case( description )
2654
2655 main.step( "Add back " + switch )
2656 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2657 for peer in links:
2658 main.Mininet1.addLink( switch, peer )
2659 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002660 for i in range( main.numCtrls ):
2661 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002662 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2663 main.log.info( "Waiting " + str( switchSleep ) +
2664 " seconds for switch up to be discovered" )
2665 time.sleep( switchSleep )
2666 device = main.ONOScli1.getDevice( dpid=switchDPID )
2667 # Peek at the deleted switch
2668 main.log.warn( str( device ) )
2669 result = main.FALSE
2670 if device and device[ 'available' ]:
2671 result = main.TRUE
2672 utilities.assert_equals( expect=main.TRUE, actual=result,
2673 onpass="add switch successful",
2674 onfail="Failed to add switch?" )
2675
2676 def CASE13( self, main ):
2677 """
2678 Clean up
2679 """
2680 import os
2681 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002682 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002683 assert main, "main not defined"
2684 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002685 assert main.CLIs, "main.CLIs not defined"
2686 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002687
2688 # printing colors to terminal
2689 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2690 'blue': '\033[94m', 'green': '\033[92m',
2691 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2692 main.case( "Test Cleanup" )
2693 main.step( "Killing tcpdumps" )
2694 main.Mininet2.stopTcpdump()
2695
2696 testname = main.TEST
2697 if main.params[ 'BACKUP' ] == "True":
2698 main.step( "Copying MN pcap and ONOS log files to test station" )
2699 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2700 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
2701 # NOTE: MN Pcap file is being saved to ~/packet_captures
2702 # scp this file as MN and TestON aren't necessarily the same vm
2703 # FIXME: scp
2704 # mn files
2705 # TODO: Load these from params
2706 # NOTE: must end in /
2707 logFolder = "/opt/onos/log/"
2708 logFiles = [ "karaf.log", "karaf.log.1" ]
2709 # NOTE: must end in /
2710 dstDir = "~/packet_captures/"
2711 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002712 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002713 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2714 ":" + logFolder + f + " " +
2715 teststationUser + "@" +
2716 teststationIP + ":" +
2717 dstDir + str( testname ) +
2718 "-" + node.name + "-" + f )
2719 main.ONOSbench.handle.expect( "\$" )
2720
2721 # std*.log's
2722 # NOTE: must end in /
2723 logFolder = "/opt/onos/var/"
2724 logFiles = [ "stderr.log", "stdout.log" ]
2725 # NOTE: must end in /
2726 dstDir = "~/packet_captures/"
2727 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002728 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002729 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2730 ":" + logFolder + f + " " +
2731 teststationUser + "@" +
2732 teststationIP + ":" +
2733 dstDir + str( testname ) +
2734 "-" + node.name + "-" + f )
2735 main.ONOSbench.handle.expect( "\$" )
2736 # sleep so scp can finish
2737 time.sleep( 10 )
2738 main.step( "Packing and rotating pcap archives" )
2739 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
2740
2741 main.step( "Stopping Mininet" )
2742 mnResult = main.Mininet1.stopNet()
2743 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2744 onpass="Mininet stopped",
2745 onfail="MN cleanup NOT successful" )
2746
2747 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002748 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002749 print colors[ 'purple' ] + "Checking logs for errors on " + \
2750 node.name + ":" + colors[ 'end' ]
2751 print main.ONOSbench.checkLogs( node.ip_address, restart=True )
2752
2753 try:
2754 timerLog = open( main.logdir + "/Timers.csv", 'w')
2755 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2756 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2757 timerLog.close()
2758 except NameError, e:
2759 main.log.exception(e)
2760
2761 def CASE14( self, main ):
2762 """
2763 start election app on all onos nodes
2764 """
Jon Halle1a3b752015-07-22 13:02:46 -07002765 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002766 assert main, "main not defined"
2767 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002768 assert main.CLIs, "main.CLIs not defined"
2769 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002770
2771 main.case("Start Leadership Election app")
2772 main.step( "Install leadership election app" )
2773 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2774 utilities.assert_equals(
2775 expect=main.TRUE,
2776 actual=appResult,
2777 onpass="Election app installed",
2778 onfail="Something went wrong with installing Leadership election" )
2779
2780 main.step( "Run for election on each node" )
2781 leaderResult = main.TRUE
2782 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002783 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002784 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002785 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002786 leader = cli.electionTestLeader()
2787 if leader is None or leader == main.FALSE:
2788 main.log.error( cli.name + ": Leader for the election app " +
2789 "should be an ONOS node, instead got '" +
2790 str( leader ) + "'" )
2791 leaderResult = main.FALSE
2792 leaders.append( leader )
2793 utilities.assert_equals(
2794 expect=main.TRUE,
2795 actual=leaderResult,
2796 onpass="Successfully ran for leadership",
2797 onfail="Failed to run for leadership" )
2798
2799 main.step( "Check that each node shows the same leader" )
2800 sameLeader = main.TRUE
2801 if len( set( leaders ) ) != 1:
2802 sameLeader = main.FALSE
2803 main.log.error( "Results of electionTestLeader is order of CLIs:" +
2804 str( leaders ) )
2805 utilities.assert_equals(
2806 expect=main.TRUE,
2807 actual=sameLeader,
2808 onpass="Leadership is consistent for the election topic",
2809 onfail="Nodes have different leaders" )
2810
2811 def CASE15( self, main ):
2812 """
2813 Check that Leadership Election is still functional
acsmars9475b1c2015-08-28 18:02:08 -07002814 15.1 Run election on each node
2815 15.2 Check that each node has the same leaders and candidates
2816 15.3 Find current leader and withdraw
2817 15.4 Check that a new node was elected leader
2818 15.5 Check that that new leader was the candidate of old leader
2819 15.6 Run for election on old leader
2820 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2821 15.8 Make sure that the old leader was added to the candidate list
2822
2823 old and new variable prefixes refer to data from before vs after
2824 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002825 """
2826 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002827 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002828 assert main, "main not defined"
2829 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002830 assert main.CLIs, "main.CLIs not defined"
2831 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002832
Jon Hall5cf14d52015-07-16 12:15:19 -07002833 description = "Check that Leadership Election is still functional"
2834 main.case( description )
2835 # NOTE: Need to re-run since being a canidate is not persistant
acsmars9475b1c2015-08-28 18:02:08 -07002836 # TODO: add check for "Command not found:" in the driver, this
2837 # means the election test app isn't loaded
2838
2839 oldLeaders = [] # leaders by node before withdrawl from candidates
2840 newLeaders = [] # leaders by node after withdrawl from candidates
2841 oldAllCandidates = [] # list of lists of each nodes' candidates before
2842 newAllCandidates = [] # list of lists of each nodes' candidates after
2843 oldCandidates = [] # list of candidates from node 0 before withdrawl
2844 newCandidates = [] # list of candidates from node 0 after withdrawl
2845 oldLeader = '' # the old leader from oldLeaders, None if not same
2846 newLeader = '' # the new leaders fron newLoeaders, None if not same
2847 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2848
Jon Hall5cf14d52015-07-16 12:15:19 -07002849 main.step( "Run for election on each node" )
acsmars9475b1c2015-08-28 18:02:08 -07002850 electionResult = main.TRUE
2851
2852 for cli in main.CLIs: # run test election on each node
2853 if cli.electionTestRun() == main.FALSE:
2854 electionResult = main.FALSE
2855
Jon Hall5cf14d52015-07-16 12:15:19 -07002856 utilities.assert_equals(
2857 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002858 actual=electionResult,
2859 onpass="All nodes successfully ran for leadership",
2860 onfail="At least one node failed to run for leadership" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002861
acsmars9475b1c2015-08-28 18:02:08 -07002862 main.step( "Check that each node shows the same leader and candidates" )
2863 sameResult = main.TRUE
2864 failMessage = "Nodes have different leaders"
2865 for cli in main.CLIs:
2866 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2867 oldAllCandidates.append( node )
2868 oldLeaders.append( node[ 0 ] )
2869 oldCandidates = oldAllCandidates[ 0 ]
2870
2871 # Check that each node has the same leader. Defines oldLeader
2872 if len( set( oldLeaders ) ) != 1:
2873 sameResult = main.FALSE
2874 main.log.error( "Results of electionTestLeader is order of " +
2875 "main.CLIs:" + str( oldLeaders ) )
2876 oldLeader = None
2877 else:
2878 oldLeader = oldLeaders[ 0 ]
2879
2880 # Check that each node's candidate list is the same
2881 for candidates in oldAllCandidates:
2882 if set( candidates ) != set( oldCandidates ):
2883 sameResult = main.FALSE
2884 failMessage += "and candidates"
2885
Jon Hall5cf14d52015-07-16 12:15:19 -07002886 utilities.assert_equals(
2887 expect=main.TRUE,
2888 actual=sameLeader,
2889 onpass="Leadership is consistent for the election topic",
acsmars9475b1c2015-08-28 18:02:08 -07002890 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002891
2892 main.step( "Find current leader and withdraw" )
acsmars9475b1c2015-08-28 18:02:08 -07002893 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002894 # do some sanity checking on leader before using it
acsmars9475b1c2015-08-28 18:02:08 -07002895 if oldLeader is None:
2896 main.log.error( "Leadership isn't consistent." )
2897 withdrawResult = main.FALSE
2898 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002899 for i in range( len( main.CLIs ) ):
acsmars9475b1c2015-08-28 18:02:08 -07002900 if oldLeader == main.nodes[ i ].ip_address:
2901 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002902 break
2903 else: # FOR/ELSE statement
2904 main.log.error( "Leader election, could not find current leader" )
2905 if oldLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002906 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002907 utilities.assert_equals(
2908 expect=main.TRUE,
2909 actual=withdrawResult,
2910 onpass="Node was withdrawn from election",
2911 onfail="Node was not withdrawn from election" )
2912
acsmars9475b1c2015-08-28 18:02:08 -07002913 main.step( "Check that a new node was elected leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002914 # FIXME: use threads
acsmars9475b1c2015-08-28 18:02:08 -07002915 newLeaderResult = main.TRUE
2916 failMessage = "Nodes have different leaders"
2917
2918 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002919 for cli in main.CLIs:
acsmars9475b1c2015-08-28 18:02:08 -07002920 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2921 if node[ 0 ] == 'none': # election might no have finished yet
2922 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2923 "sure elections are complete." )
2924 time.sleep(5)
2925 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2926 if node[ 0 ] == 'none': # election still isn't done, errors
2927 main.log.error( "No leader was elected on at least one node" )
2928 newLeaderResult = main.FALSE
2929 newAllCandidates.append( node )
2930 newLeaders.append( node[ 0 ] )
2931 newCandidates = newAllCandidates[ 0 ]
2932
2933 # Check that each node has the same leader. Defines newLeader
2934 if len( set( newLeaders ) ) != 1:
2935 newLeaderResult = main.FALSE
2936 main.log.error( "Nodes have different leaders: " +
2937 str( newLeaders ) )
2938 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002939 else:
acsmars9475b1c2015-08-28 18:02:08 -07002940 newLeader = newLeaders[ 0 ]
2941
2942 # Check that the new leader is not the older leader, which was withdrawn
2943 if newLeader == oldLeader:
2944 newLeaderResult = main.FALSE
2945 main.log.error( "All nodes still see old leader: " + oldLeader +
2946 " as the current leader" )
2947
Jon Hall5cf14d52015-07-16 12:15:19 -07002948 utilities.assert_equals(
2949 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002950 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002951 onpass="Leadership election passed",
2952 onfail="Something went wrong with Leadership election" )
2953
acsmars9475b1c2015-08-28 18:02:08 -07002954 main.step( "Check that that new leader was the candidate of old leader")
2955 # candidates[ 2 ] should be come the top candidate after withdrawl
2956 correctCandidateResult = main.TRUE
2957
2958 if newLeader != oldCandidates[ 2 ]:
2959 correctCandidateResult = main.FALSE
2960 main.log.error( "Candidate " + newLeader + " was elected. " +
2961 oldCandidates[ 2 ] + " should have had priority." )
2962
2963 utilities.assert_equals(
2964 expect=main.TRUE,
2965 actual=correctCandidateResult,
2966 onpass="Correct Candidate Elected",
2967 onfail="Incorrect Candidate Elected" )
2968
Jon Hall5cf14d52015-07-16 12:15:19 -07002969 main.step( "Run for election on old leader( just so everyone " +
2970 "is in the hat )" )
acsmars9475b1c2015-08-28 18:02:08 -07002971 if oldLeaderCLI is not None:
2972 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002973 else:
acsmars9475b1c2015-08-28 18:02:08 -07002974 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002975 runResult = main.FALSE
2976 utilities.assert_equals(
2977 expect=main.TRUE,
2978 actual=runResult,
2979 onpass="App re-ran for election",
2980 onfail="App failed to run for election" )
2981
acsmars9475b1c2015-08-28 18:02:08 -07002982
2983
2984 main.step(
2985 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002986 # verify leader didn't just change
acsmars9475b1c2015-08-28 18:02:08 -07002987 positionResult = main.TRUE
2988 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
2989
2990 # Reset and reuse the new candidate and leaders lists
2991 newAllCandidates = []
2992 newCandidates = []
2993 newLeaders = []
2994 for cli in main.CLIs:
2995 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2996 if oldLeader not in node: # election might no have finished yet
2997 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2998 "be sure elections are complete" )
2999 time.sleep(5)
3000 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
3001 if oldLeader not in node: # election still isn't done, errors
3002 main.log.error(
3003 "Old leader was not elected on at least one node" )
3004 positionResult = main.FALSE
3005 newAllCandidates.append( node )
3006 newLeaders.append( node[ 0 ] )
3007 newCandidates = newAllCandidates[ 0 ]
3008
3009 # Check that each node has the same leader. Defines newLeader
3010 if len( set( newLeaders ) ) != 1:
3011 positionResult = main.FALSE
3012 main.log.error( "Nodes have different leaders: " +
3013 str( newLeaders ) )
3014 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07003015 else:
acsmars9475b1c2015-08-28 18:02:08 -07003016 newLeader = newLeaders[ 0 ]
3017
3018 # Check that the re-elected node is last on the candidate List
3019 if oldLeader != newCandidates[ -1 ]:
3020 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
3021 str( newCandidates ) )
3022 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003023
3024 utilities.assert_equals(
3025 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07003026 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003027 onpass="Old leader successfully re-ran for election",
3028 onfail="Something went wrong with Leadership election after " +
3029 "the old leader re-ran for election" )
3030
3031 def CASE16( self, main ):
3032 """
3033 Install Distributed Primitives app
3034 """
3035 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003036 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003037 assert main, "main not defined"
3038 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003039 assert main.CLIs, "main.CLIs not defined"
3040 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003041
3042 # Variables for the distributed primitives tests
3043 global pCounterName
3044 global iCounterName
3045 global pCounterValue
3046 global iCounterValue
3047 global onosSet
3048 global onosSetName
3049 pCounterName = "TestON-Partitions"
3050 iCounterName = "TestON-inMemory"
3051 pCounterValue = 0
3052 iCounterValue = 0
3053 onosSet = set([])
3054 onosSetName = "TestON-set"
3055
3056 description = "Install Primitives app"
3057 main.case( description )
3058 main.step( "Install Primitives app" )
3059 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07003060 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003061 utilities.assert_equals( expect=main.TRUE,
3062 actual=appResults,
3063 onpass="Primitives app activated",
3064 onfail="Primitives app not activated" )
3065 time.sleep( 5 ) # To allow all nodes to activate
3066
3067 def CASE17( self, main ):
3068 """
3069 Check for basic functionality with distributed primitives
3070 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003071 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003072 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003073 assert main, "main not defined"
3074 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003075 assert main.CLIs, "main.CLIs not defined"
3076 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003077 assert pCounterName, "pCounterName not defined"
3078 assert iCounterName, "iCounterName not defined"
3079 assert onosSetName, "onosSetName not defined"
3080 # NOTE: assert fails if value is 0/None/Empty/False
3081 try:
3082 pCounterValue
3083 except NameError:
3084 main.log.error( "pCounterValue not defined, setting to 0" )
3085 pCounterValue = 0
3086 try:
3087 iCounterValue
3088 except NameError:
3089 main.log.error( "iCounterValue not defined, setting to 0" )
3090 iCounterValue = 0
3091 try:
3092 onosSet
3093 except NameError:
3094 main.log.error( "onosSet not defined, setting to empty Set" )
3095 onosSet = set([])
3096 # Variables for the distributed primitives tests. These are local only
3097 addValue = "a"
3098 addAllValue = "a b c d e f"
3099 retainValue = "c d e f"
3100
3101 description = "Check for basic functionality with distributed " +\
3102 "primitives"
3103 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003104 main.caseExplanation = "Test the methods of the distributed " +\
3105 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003106 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003107 # Partitioned counters
3108 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003109 pCounters = []
3110 threads = []
3111 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003112 for i in range( main.numCtrls ):
3113 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3114 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003115 args=[ pCounterName ] )
3116 pCounterValue += 1
3117 addedPValues.append( pCounterValue )
3118 threads.append( t )
3119 t.start()
3120
3121 for t in threads:
3122 t.join()
3123 pCounters.append( t.result )
3124 # Check that counter incremented numController times
3125 pCounterResults = True
3126 for i in addedPValues:
3127 tmpResult = i in pCounters
3128 pCounterResults = pCounterResults and tmpResult
3129 if not tmpResult:
3130 main.log.error( str( i ) + " is not in partitioned "
3131 "counter incremented results" )
3132 utilities.assert_equals( expect=True,
3133 actual=pCounterResults,
3134 onpass="Default counter incremented",
3135 onfail="Error incrementing default" +
3136 " counter" )
3137
Jon Halle1a3b752015-07-22 13:02:46 -07003138 main.step( "Get then Increment a default counter on each node" )
3139 pCounters = []
3140 threads = []
3141 addedPValues = []
3142 for i in range( main.numCtrls ):
3143 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3144 name="counterGetAndAdd-" + str( i ),
3145 args=[ pCounterName ] )
3146 addedPValues.append( pCounterValue )
3147 pCounterValue += 1
3148 threads.append( t )
3149 t.start()
3150
3151 for t in threads:
3152 t.join()
3153 pCounters.append( t.result )
3154 # Check that counter incremented numController times
3155 pCounterResults = True
3156 for i in addedPValues:
3157 tmpResult = i in pCounters
3158 pCounterResults = pCounterResults and tmpResult
3159 if not tmpResult:
3160 main.log.error( str( i ) + " is not in partitioned "
3161 "counter incremented results" )
3162 utilities.assert_equals( expect=True,
3163 actual=pCounterResults,
3164 onpass="Default counter incremented",
3165 onfail="Error incrementing default" +
3166 " counter" )
3167
3168 main.step( "Counters we added have the correct values" )
3169 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3170 utilities.assert_equals( expect=main.TRUE,
3171 actual=incrementCheck,
3172 onpass="Added counters are correct",
3173 onfail="Added counters are incorrect" )
3174
3175 main.step( "Add -8 to then get a default counter on each node" )
3176 pCounters = []
3177 threads = []
3178 addedPValues = []
3179 for i in range( main.numCtrls ):
3180 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3181 name="counterIncrement-" + str( i ),
3182 args=[ pCounterName ],
3183 kwargs={ "delta": -8 } )
3184 pCounterValue += -8
3185 addedPValues.append( pCounterValue )
3186 threads.append( t )
3187 t.start()
3188
3189 for t in threads:
3190 t.join()
3191 pCounters.append( t.result )
3192 # Check that counter incremented numController times
3193 pCounterResults = True
3194 for i in addedPValues:
3195 tmpResult = i in pCounters
3196 pCounterResults = pCounterResults and tmpResult
3197 if not tmpResult:
3198 main.log.error( str( i ) + " is not in partitioned "
3199 "counter incremented results" )
3200 utilities.assert_equals( expect=True,
3201 actual=pCounterResults,
3202 onpass="Default counter incremented",
3203 onfail="Error incrementing default" +
3204 " counter" )
3205
3206 main.step( "Add 5 to then get a default counter on each node" )
3207 pCounters = []
3208 threads = []
3209 addedPValues = []
3210 for i in range( main.numCtrls ):
3211 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3212 name="counterIncrement-" + str( i ),
3213 args=[ pCounterName ],
3214 kwargs={ "delta": 5 } )
3215 pCounterValue += 5
3216 addedPValues.append( pCounterValue )
3217 threads.append( t )
3218 t.start()
3219
3220 for t in threads:
3221 t.join()
3222 pCounters.append( t.result )
3223 # Check that counter incremented numController times
3224 pCounterResults = True
3225 for i in addedPValues:
3226 tmpResult = i in pCounters
3227 pCounterResults = pCounterResults and tmpResult
3228 if not tmpResult:
3229 main.log.error( str( i ) + " is not in partitioned "
3230 "counter incremented results" )
3231 utilities.assert_equals( expect=True,
3232 actual=pCounterResults,
3233 onpass="Default counter incremented",
3234 onfail="Error incrementing default" +
3235 " counter" )
3236
3237 main.step( "Get then add 5 to a default counter on each node" )
3238 pCounters = []
3239 threads = []
3240 addedPValues = []
3241 for i in range( main.numCtrls ):
3242 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3243 name="counterIncrement-" + str( i ),
3244 args=[ pCounterName ],
3245 kwargs={ "delta": 5 } )
3246 addedPValues.append( pCounterValue )
3247 pCounterValue += 5
3248 threads.append( t )
3249 t.start()
3250
3251 for t in threads:
3252 t.join()
3253 pCounters.append( t.result )
3254 # Check that counter incremented numController times
3255 pCounterResults = True
3256 for i in addedPValues:
3257 tmpResult = i in pCounters
3258 pCounterResults = pCounterResults and tmpResult
3259 if not tmpResult:
3260 main.log.error( str( i ) + " is not in partitioned "
3261 "counter incremented results" )
3262 utilities.assert_equals( expect=True,
3263 actual=pCounterResults,
3264 onpass="Default counter incremented",
3265 onfail="Error incrementing default" +
3266 " counter" )
3267
3268 main.step( "Counters we added have the correct values" )
3269 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3270 utilities.assert_equals( expect=main.TRUE,
3271 actual=incrementCheck,
3272 onpass="Added counters are correct",
3273 onfail="Added counters are incorrect" )
3274
3275 # In-Memory counters
3276 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003277 iCounters = []
3278 addedIValues = []
3279 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003280 for i in range( main.numCtrls ):
3281 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003282 name="icounterIncrement-" + str( i ),
3283 args=[ iCounterName ],
3284 kwargs={ "inMemory": True } )
3285 iCounterValue += 1
3286 addedIValues.append( iCounterValue )
3287 threads.append( t )
3288 t.start()
3289
3290 for t in threads:
3291 t.join()
3292 iCounters.append( t.result )
3293 # Check that counter incremented numController times
3294 iCounterResults = True
3295 for i in addedIValues:
3296 tmpResult = i in iCounters
3297 iCounterResults = iCounterResults and tmpResult
3298 if not tmpResult:
3299 main.log.error( str( i ) + " is not in the in-memory "
3300 "counter incremented results" )
3301 utilities.assert_equals( expect=True,
3302 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003303 onpass="In-memory counter incremented",
3304 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003305 " counter" )
3306
Jon Halle1a3b752015-07-22 13:02:46 -07003307 main.step( "Get then Increment a in-memory counter on each node" )
3308 iCounters = []
3309 threads = []
3310 addedIValues = []
3311 for i in range( main.numCtrls ):
3312 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3313 name="counterGetAndAdd-" + str( i ),
3314 args=[ iCounterName ],
3315 kwargs={ "inMemory": True } )
3316 addedIValues.append( iCounterValue )
3317 iCounterValue += 1
3318 threads.append( t )
3319 t.start()
3320
3321 for t in threads:
3322 t.join()
3323 iCounters.append( t.result )
3324 # Check that counter incremented numController times
3325 iCounterResults = True
3326 for i in addedIValues:
3327 tmpResult = i in iCounters
3328 iCounterResults = iCounterResults and tmpResult
3329 if not tmpResult:
3330 main.log.error( str( i ) + " is not in in-memory "
3331 "counter incremented results" )
3332 utilities.assert_equals( expect=True,
3333 actual=iCounterResults,
3334 onpass="In-memory counter incremented",
3335 onfail="Error incrementing in-memory" +
3336 " counter" )
3337
3338 main.step( "Counters we added have the correct values" )
3339 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3340 utilities.assert_equals( expect=main.TRUE,
3341 actual=incrementCheck,
3342 onpass="Added counters are correct",
3343 onfail="Added counters are incorrect" )
3344
3345 main.step( "Add -8 to then get a in-memory counter on each node" )
3346 iCounters = []
3347 threads = []
3348 addedIValues = []
3349 for i in range( main.numCtrls ):
3350 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3351 name="counterIncrement-" + str( i ),
3352 args=[ iCounterName ],
3353 kwargs={ "delta": -8, "inMemory": True } )
3354 iCounterValue += -8
3355 addedIValues.append( iCounterValue )
3356 threads.append( t )
3357 t.start()
3358
3359 for t in threads:
3360 t.join()
3361 iCounters.append( t.result )
3362 # Check that counter incremented numController times
3363 iCounterResults = True
3364 for i in addedIValues:
3365 tmpResult = i in iCounters
3366 iCounterResults = iCounterResults and tmpResult
3367 if not tmpResult:
3368 main.log.error( str( i ) + " is not in in-memory "
3369 "counter incremented results" )
3370 utilities.assert_equals( expect=True,
3371 actual=pCounterResults,
3372 onpass="In-memory counter incremented",
3373 onfail="Error incrementing in-memory" +
3374 " counter" )
3375
3376 main.step( "Add 5 to then get a in-memory counter on each node" )
3377 iCounters = []
3378 threads = []
3379 addedIValues = []
3380 for i in range( main.numCtrls ):
3381 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3382 name="counterIncrement-" + str( i ),
3383 args=[ iCounterName ],
3384 kwargs={ "delta": 5, "inMemory": True } )
3385 iCounterValue += 5
3386 addedIValues.append( iCounterValue )
3387 threads.append( t )
3388 t.start()
3389
3390 for t in threads:
3391 t.join()
3392 iCounters.append( t.result )
3393 # Check that counter incremented numController times
3394 iCounterResults = True
3395 for i in addedIValues:
3396 tmpResult = i in iCounters
3397 iCounterResults = iCounterResults and tmpResult
3398 if not tmpResult:
3399 main.log.error( str( i ) + " is not in in-memory "
3400 "counter incremented results" )
3401 utilities.assert_equals( expect=True,
3402 actual=pCounterResults,
3403 onpass="In-memory counter incremented",
3404 onfail="Error incrementing in-memory" +
3405 " counter" )
3406
3407 main.step( "Get then add 5 to a in-memory counter on each node" )
3408 iCounters = []
3409 threads = []
3410 addedIValues = []
3411 for i in range( main.numCtrls ):
3412 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3413 name="counterIncrement-" + str( i ),
3414 args=[ iCounterName ],
3415 kwargs={ "delta": 5, "inMemory": True } )
3416 addedIValues.append( iCounterValue )
3417 iCounterValue += 5
3418 threads.append( t )
3419 t.start()
3420
3421 for t in threads:
3422 t.join()
3423 iCounters.append( t.result )
3424 # Check that counter incremented numController times
3425 iCounterResults = True
3426 for i in addedIValues:
3427 tmpResult = i in iCounters
3428 iCounterResults = iCounterResults and tmpResult
3429 if not tmpResult:
3430 main.log.error( str( i ) + " is not in in-memory "
3431 "counter incremented results" )
3432 utilities.assert_equals( expect=True,
3433 actual=iCounterResults,
3434 onpass="In-memory counter incremented",
3435 onfail="Error incrementing in-memory" +
3436 " counter" )
3437
3438 main.step( "Counters we added have the correct values" )
3439 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3440 utilities.assert_equals( expect=main.TRUE,
3441 actual=incrementCheck,
3442 onpass="Added counters are correct",
3443 onfail="Added counters are incorrect" )
3444
Jon Hall5cf14d52015-07-16 12:15:19 -07003445 main.step( "Check counters are consistant across nodes" )
3446 onosCounters = []
3447 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003448 for i in range( main.numCtrls ):
3449 t = main.Thread( target=main.CLIs[i].counters,
Jon Hall5cf14d52015-07-16 12:15:19 -07003450 name="counters-" + str( i ) )
3451 threads.append( t )
3452 t.start()
3453 for t in threads:
3454 t.join()
3455 onosCounters.append( t.result )
3456 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3457 if all( tmp ):
3458 main.log.info( "Counters are consistent across all nodes" )
3459 consistentCounterResults = main.TRUE
3460 else:
3461 main.log.error( "Counters are not consistent across all nodes" )
3462 consistentCounterResults = main.FALSE
3463 utilities.assert_equals( expect=main.TRUE,
3464 actual=consistentCounterResults,
3465 onpass="ONOS counters are consistent " +
3466 "across nodes",
3467 onfail="ONOS Counters are inconsistent " +
3468 "across nodes" )
3469
3470 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003471 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3472 incrementCheck = incrementCheck and \
3473 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003474 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003475 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003476 onpass="Added counters are correct",
3477 onfail="Added counters are incorrect" )
Jon Halle1a3b752015-07-22 13:02:46 -07003478
Jon Hall5cf14d52015-07-16 12:15:19 -07003479 # DISTRIBUTED SETS
3480 main.step( "Distributed Set get" )
3481 size = len( onosSet )
3482 getResponses = []
3483 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003484 for i in range( main.numCtrls ):
3485 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003486 name="setTestGet-" + str( i ),
3487 args=[ onosSetName ] )
3488 threads.append( t )
3489 t.start()
3490 for t in threads:
3491 t.join()
3492 getResponses.append( t.result )
3493
3494 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003495 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003496 if isinstance( getResponses[ i ], list):
3497 current = set( getResponses[ i ] )
3498 if len( current ) == len( getResponses[ i ] ):
3499 # no repeats
3500 if onosSet != current:
3501 main.log.error( "ONOS" + str( i + 1 ) +
3502 " has incorrect view" +
3503 " of set " + onosSetName + ":\n" +
3504 str( getResponses[ i ] ) )
3505 main.log.debug( "Expected: " + str( onosSet ) )
3506 main.log.debug( "Actual: " + str( current ) )
3507 getResults = main.FALSE
3508 else:
3509 # error, set is not a set
3510 main.log.error( "ONOS" + str( i + 1 ) +
3511 " has repeat elements in" +
3512 " set " + onosSetName + ":\n" +
3513 str( getResponses[ i ] ) )
3514 getResults = main.FALSE
3515 elif getResponses[ i ] == main.ERROR:
3516 getResults = main.FALSE
3517 utilities.assert_equals( expect=main.TRUE,
3518 actual=getResults,
3519 onpass="Set elements are correct",
3520 onfail="Set elements are incorrect" )
3521
3522 main.step( "Distributed Set size" )
3523 sizeResponses = []
3524 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003525 for i in range( main.numCtrls ):
3526 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003527 name="setTestSize-" + str( i ),
3528 args=[ onosSetName ] )
3529 threads.append( t )
3530 t.start()
3531 for t in threads:
3532 t.join()
3533 sizeResponses.append( t.result )
3534
3535 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003536 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003537 if size != sizeResponses[ i ]:
3538 sizeResults = main.FALSE
3539 main.log.error( "ONOS" + str( i + 1 ) +
3540 " expected a size of " + str( size ) +
3541 " for set " + onosSetName +
3542 " but got " + str( sizeResponses[ i ] ) )
3543 utilities.assert_equals( expect=main.TRUE,
3544 actual=sizeResults,
3545 onpass="Set sizes are correct",
3546 onfail="Set sizes are incorrect" )
3547
3548 main.step( "Distributed Set add()" )
3549 onosSet.add( addValue )
3550 addResponses = []
3551 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003552 for i in range( main.numCtrls ):
3553 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003554 name="setTestAdd-" + str( i ),
3555 args=[ onosSetName, addValue ] )
3556 threads.append( t )
3557 t.start()
3558 for t in threads:
3559 t.join()
3560 addResponses.append( t.result )
3561
3562 # main.TRUE = successfully changed the set
3563 # main.FALSE = action resulted in no change in set
3564 # main.ERROR - Some error in executing the function
3565 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003566 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003567 if addResponses[ i ] == main.TRUE:
3568 # All is well
3569 pass
3570 elif addResponses[ i ] == main.FALSE:
3571 # Already in set, probably fine
3572 pass
3573 elif addResponses[ i ] == main.ERROR:
3574 # Error in execution
3575 addResults = main.FALSE
3576 else:
3577 # unexpected result
3578 addResults = main.FALSE
3579 if addResults != main.TRUE:
3580 main.log.error( "Error executing set add" )
3581
3582 # Check if set is still correct
3583 size = len( onosSet )
3584 getResponses = []
3585 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003586 for i in range( main.numCtrls ):
3587 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003588 name="setTestGet-" + str( i ),
3589 args=[ onosSetName ] )
3590 threads.append( t )
3591 t.start()
3592 for t in threads:
3593 t.join()
3594 getResponses.append( t.result )
3595 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003596 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003597 if isinstance( getResponses[ i ], list):
3598 current = set( getResponses[ i ] )
3599 if len( current ) == len( getResponses[ i ] ):
3600 # no repeats
3601 if onosSet != current:
3602 main.log.error( "ONOS" + str( i + 1 ) +
3603 " has incorrect view" +
3604 " of set " + onosSetName + ":\n" +
3605 str( getResponses[ i ] ) )
3606 main.log.debug( "Expected: " + str( onosSet ) )
3607 main.log.debug( "Actual: " + str( current ) )
3608 getResults = main.FALSE
3609 else:
3610 # error, set is not a set
3611 main.log.error( "ONOS" + str( i + 1 ) +
3612 " has repeat elements in" +
3613 " set " + onosSetName + ":\n" +
3614 str( getResponses[ i ] ) )
3615 getResults = main.FALSE
3616 elif getResponses[ i ] == main.ERROR:
3617 getResults = main.FALSE
3618 sizeResponses = []
3619 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003620 for i in range( main.numCtrls ):
3621 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003622 name="setTestSize-" + str( i ),
3623 args=[ onosSetName ] )
3624 threads.append( t )
3625 t.start()
3626 for t in threads:
3627 t.join()
3628 sizeResponses.append( t.result )
3629 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003630 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003631 if size != sizeResponses[ i ]:
3632 sizeResults = main.FALSE
3633 main.log.error( "ONOS" + str( i + 1 ) +
3634 " expected a size of " + str( size ) +
3635 " for set " + onosSetName +
3636 " but got " + str( sizeResponses[ i ] ) )
3637 addResults = addResults and getResults and sizeResults
3638 utilities.assert_equals( expect=main.TRUE,
3639 actual=addResults,
3640 onpass="Set add correct",
3641 onfail="Set add was incorrect" )
3642
3643 main.step( "Distributed Set addAll()" )
3644 onosSet.update( addAllValue.split() )
3645 addResponses = []
3646 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003647 for i in range( main.numCtrls ):
3648 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003649 name="setTestAddAll-" + str( i ),
3650 args=[ onosSetName, addAllValue ] )
3651 threads.append( t )
3652 t.start()
3653 for t in threads:
3654 t.join()
3655 addResponses.append( t.result )
3656
3657 # main.TRUE = successfully changed the set
3658 # main.FALSE = action resulted in no change in set
3659 # main.ERROR - Some error in executing the function
3660 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003661 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003662 if addResponses[ i ] == main.TRUE:
3663 # All is well
3664 pass
3665 elif addResponses[ i ] == main.FALSE:
3666 # Already in set, probably fine
3667 pass
3668 elif addResponses[ i ] == main.ERROR:
3669 # Error in execution
3670 addAllResults = main.FALSE
3671 else:
3672 # unexpected result
3673 addAllResults = main.FALSE
3674 if addAllResults != main.TRUE:
3675 main.log.error( "Error executing set addAll" )
3676
3677 # Check if set is still correct
3678 size = len( onosSet )
3679 getResponses = []
3680 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003681 for i in range( main.numCtrls ):
3682 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003683 name="setTestGet-" + str( i ),
3684 args=[ onosSetName ] )
3685 threads.append( t )
3686 t.start()
3687 for t in threads:
3688 t.join()
3689 getResponses.append( t.result )
3690 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003691 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003692 if isinstance( getResponses[ i ], list):
3693 current = set( getResponses[ i ] )
3694 if len( current ) == len( getResponses[ i ] ):
3695 # no repeats
3696 if onosSet != current:
3697 main.log.error( "ONOS" + str( i + 1 ) +
3698 " has incorrect view" +
3699 " of set " + onosSetName + ":\n" +
3700 str( getResponses[ i ] ) )
3701 main.log.debug( "Expected: " + str( onosSet ) )
3702 main.log.debug( "Actual: " + str( current ) )
3703 getResults = main.FALSE
3704 else:
3705 # error, set is not a set
3706 main.log.error( "ONOS" + str( i + 1 ) +
3707 " has repeat elements in" +
3708 " set " + onosSetName + ":\n" +
3709 str( getResponses[ i ] ) )
3710 getResults = main.FALSE
3711 elif getResponses[ i ] == main.ERROR:
3712 getResults = main.FALSE
3713 sizeResponses = []
3714 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003715 for i in range( main.numCtrls ):
3716 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003717 name="setTestSize-" + str( i ),
3718 args=[ onosSetName ] )
3719 threads.append( t )
3720 t.start()
3721 for t in threads:
3722 t.join()
3723 sizeResponses.append( t.result )
3724 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003725 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003726 if size != sizeResponses[ i ]:
3727 sizeResults = main.FALSE
3728 main.log.error( "ONOS" + str( i + 1 ) +
3729 " expected a size of " + str( size ) +
3730 " for set " + onosSetName +
3731 " but got " + str( sizeResponses[ i ] ) )
3732 addAllResults = addAllResults and getResults and sizeResults
3733 utilities.assert_equals( expect=main.TRUE,
3734 actual=addAllResults,
3735 onpass="Set addAll correct",
3736 onfail="Set addAll was incorrect" )
3737
3738 main.step( "Distributed Set contains()" )
3739 containsResponses = []
3740 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003741 for i in range( main.numCtrls ):
3742 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003743 name="setContains-" + str( i ),
3744 args=[ onosSetName ],
3745 kwargs={ "values": addValue } )
3746 threads.append( t )
3747 t.start()
3748 for t in threads:
3749 t.join()
3750 # NOTE: This is the tuple
3751 containsResponses.append( t.result )
3752
3753 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003754 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003755 if containsResponses[ i ] == main.ERROR:
3756 containsResults = main.FALSE
3757 else:
3758 containsResults = containsResults and\
3759 containsResponses[ i ][ 1 ]
3760 utilities.assert_equals( expect=main.TRUE,
3761 actual=containsResults,
3762 onpass="Set contains is functional",
3763 onfail="Set contains failed" )
3764
3765 main.step( "Distributed Set containsAll()" )
3766 containsAllResponses = []
3767 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003768 for i in range( main.numCtrls ):
3769 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003770 name="setContainsAll-" + str( i ),
3771 args=[ onosSetName ],
3772 kwargs={ "values": addAllValue } )
3773 threads.append( t )
3774 t.start()
3775 for t in threads:
3776 t.join()
3777 # NOTE: This is the tuple
3778 containsAllResponses.append( t.result )
3779
3780 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003781 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003782 if containsResponses[ i ] == main.ERROR:
3783 containsResults = main.FALSE
3784 else:
3785 containsResults = containsResults and\
3786 containsResponses[ i ][ 1 ]
3787 utilities.assert_equals( expect=main.TRUE,
3788 actual=containsAllResults,
3789 onpass="Set containsAll is functional",
3790 onfail="Set containsAll failed" )
3791
3792 main.step( "Distributed Set remove()" )
3793 onosSet.remove( addValue )
3794 removeResponses = []
3795 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003796 for i in range( main.numCtrls ):
3797 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003798 name="setTestRemove-" + str( i ),
3799 args=[ onosSetName, addValue ] )
3800 threads.append( t )
3801 t.start()
3802 for t in threads:
3803 t.join()
3804 removeResponses.append( t.result )
3805
3806 # main.TRUE = successfully changed the set
3807 # main.FALSE = action resulted in no change in set
3808 # main.ERROR - Some error in executing the function
3809 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003810 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003811 if removeResponses[ i ] == main.TRUE:
3812 # All is well
3813 pass
3814 elif removeResponses[ i ] == main.FALSE:
3815 # not in set, probably fine
3816 pass
3817 elif removeResponses[ i ] == main.ERROR:
3818 # Error in execution
3819 removeResults = main.FALSE
3820 else:
3821 # unexpected result
3822 removeResults = main.FALSE
3823 if removeResults != main.TRUE:
3824 main.log.error( "Error executing set remove" )
3825
3826 # Check if set is still correct
3827 size = len( onosSet )
3828 getResponses = []
3829 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003830 for i in range( main.numCtrls ):
3831 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003832 name="setTestGet-" + str( i ),
3833 args=[ onosSetName ] )
3834 threads.append( t )
3835 t.start()
3836 for t in threads:
3837 t.join()
3838 getResponses.append( t.result )
3839 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003840 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003841 if isinstance( getResponses[ i ], list):
3842 current = set( getResponses[ i ] )
3843 if len( current ) == len( getResponses[ i ] ):
3844 # no repeats
3845 if onosSet != current:
3846 main.log.error( "ONOS" + str( i + 1 ) +
3847 " has incorrect view" +
3848 " of set " + onosSetName + ":\n" +
3849 str( getResponses[ i ] ) )
3850 main.log.debug( "Expected: " + str( onosSet ) )
3851 main.log.debug( "Actual: " + str( current ) )
3852 getResults = main.FALSE
3853 else:
3854 # error, set is not a set
3855 main.log.error( "ONOS" + str( i + 1 ) +
3856 " has repeat elements in" +
3857 " set " + onosSetName + ":\n" +
3858 str( getResponses[ i ] ) )
3859 getResults = main.FALSE
3860 elif getResponses[ i ] == main.ERROR:
3861 getResults = main.FALSE
3862 sizeResponses = []
3863 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003864 for i in range( main.numCtrls ):
3865 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003866 name="setTestSize-" + str( i ),
3867 args=[ onosSetName ] )
3868 threads.append( t )
3869 t.start()
3870 for t in threads:
3871 t.join()
3872 sizeResponses.append( t.result )
3873 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003874 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003875 if size != sizeResponses[ i ]:
3876 sizeResults = main.FALSE
3877 main.log.error( "ONOS" + str( i + 1 ) +
3878 " expected a size of " + str( size ) +
3879 " for set " + onosSetName +
3880 " but got " + str( sizeResponses[ i ] ) )
3881 removeResults = removeResults and getResults and sizeResults
3882 utilities.assert_equals( expect=main.TRUE,
3883 actual=removeResults,
3884 onpass="Set remove correct",
3885 onfail="Set remove was incorrect" )
3886
3887 main.step( "Distributed Set removeAll()" )
3888 onosSet.difference_update( addAllValue.split() )
3889 removeAllResponses = []
3890 threads = []
3891 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003892 for i in range( main.numCtrls ):
3893 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003894 name="setTestRemoveAll-" + str( i ),
3895 args=[ onosSetName, addAllValue ] )
3896 threads.append( t )
3897 t.start()
3898 for t in threads:
3899 t.join()
3900 removeAllResponses.append( t.result )
3901 except Exception, e:
3902 main.log.exception(e)
3903
3904 # main.TRUE = successfully changed the set
3905 # main.FALSE = action resulted in no change in set
3906 # main.ERROR - Some error in executing the function
3907 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003908 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003909 if removeAllResponses[ i ] == main.TRUE:
3910 # All is well
3911 pass
3912 elif removeAllResponses[ i ] == main.FALSE:
3913 # not in set, probably fine
3914 pass
3915 elif removeAllResponses[ i ] == main.ERROR:
3916 # Error in execution
3917 removeAllResults = main.FALSE
3918 else:
3919 # unexpected result
3920 removeAllResults = main.FALSE
3921 if removeAllResults != main.TRUE:
3922 main.log.error( "Error executing set removeAll" )
3923
3924 # Check if set is still correct
3925 size = len( onosSet )
3926 getResponses = []
3927 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003928 for i in range( main.numCtrls ):
3929 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003930 name="setTestGet-" + str( i ),
3931 args=[ onosSetName ] )
3932 threads.append( t )
3933 t.start()
3934 for t in threads:
3935 t.join()
3936 getResponses.append( t.result )
3937 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003938 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003939 if isinstance( getResponses[ i ], list):
3940 current = set( getResponses[ i ] )
3941 if len( current ) == len( getResponses[ i ] ):
3942 # no repeats
3943 if onosSet != current:
3944 main.log.error( "ONOS" + str( i + 1 ) +
3945 " has incorrect view" +
3946 " of set " + onosSetName + ":\n" +
3947 str( getResponses[ i ] ) )
3948 main.log.debug( "Expected: " + str( onosSet ) )
3949 main.log.debug( "Actual: " + str( current ) )
3950 getResults = main.FALSE
3951 else:
3952 # error, set is not a set
3953 main.log.error( "ONOS" + str( i + 1 ) +
3954 " has repeat elements in" +
3955 " set " + onosSetName + ":\n" +
3956 str( getResponses[ i ] ) )
3957 getResults = main.FALSE
3958 elif getResponses[ i ] == main.ERROR:
3959 getResults = main.FALSE
3960 sizeResponses = []
3961 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003962 for i in range( main.numCtrls ):
3963 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003964 name="setTestSize-" + str( i ),
3965 args=[ onosSetName ] )
3966 threads.append( t )
3967 t.start()
3968 for t in threads:
3969 t.join()
3970 sizeResponses.append( t.result )
3971 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003972 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003973 if size != sizeResponses[ i ]:
3974 sizeResults = main.FALSE
3975 main.log.error( "ONOS" + str( i + 1 ) +
3976 " expected a size of " + str( size ) +
3977 " for set " + onosSetName +
3978 " but got " + str( sizeResponses[ i ] ) )
3979 removeAllResults = removeAllResults and getResults and sizeResults
3980 utilities.assert_equals( expect=main.TRUE,
3981 actual=removeAllResults,
3982 onpass="Set removeAll correct",
3983 onfail="Set removeAll was incorrect" )
3984
3985 main.step( "Distributed Set addAll()" )
3986 onosSet.update( addAllValue.split() )
3987 addResponses = []
3988 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003989 for i in range( main.numCtrls ):
3990 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003991 name="setTestAddAll-" + str( i ),
3992 args=[ onosSetName, addAllValue ] )
3993 threads.append( t )
3994 t.start()
3995 for t in threads:
3996 t.join()
3997 addResponses.append( t.result )
3998
3999 # main.TRUE = successfully changed the set
4000 # main.FALSE = action resulted in no change in set
4001 # main.ERROR - Some error in executing the function
4002 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004003 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004004 if addResponses[ i ] == main.TRUE:
4005 # All is well
4006 pass
4007 elif addResponses[ i ] == main.FALSE:
4008 # Already in set, probably fine
4009 pass
4010 elif addResponses[ i ] == main.ERROR:
4011 # Error in execution
4012 addAllResults = main.FALSE
4013 else:
4014 # unexpected result
4015 addAllResults = main.FALSE
4016 if addAllResults != main.TRUE:
4017 main.log.error( "Error executing set addAll" )
4018
4019 # Check if set is still correct
4020 size = len( onosSet )
4021 getResponses = []
4022 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004023 for i in range( main.numCtrls ):
4024 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004025 name="setTestGet-" + str( i ),
4026 args=[ onosSetName ] )
4027 threads.append( t )
4028 t.start()
4029 for t in threads:
4030 t.join()
4031 getResponses.append( t.result )
4032 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004033 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004034 if isinstance( getResponses[ i ], list):
4035 current = set( getResponses[ i ] )
4036 if len( current ) == len( getResponses[ i ] ):
4037 # no repeats
4038 if onosSet != current:
4039 main.log.error( "ONOS" + str( i + 1 ) +
4040 " has incorrect view" +
4041 " of set " + onosSetName + ":\n" +
4042 str( getResponses[ i ] ) )
4043 main.log.debug( "Expected: " + str( onosSet ) )
4044 main.log.debug( "Actual: " + str( current ) )
4045 getResults = main.FALSE
4046 else:
4047 # error, set is not a set
4048 main.log.error( "ONOS" + str( i + 1 ) +
4049 " has repeat elements in" +
4050 " set " + onosSetName + ":\n" +
4051 str( getResponses[ i ] ) )
4052 getResults = main.FALSE
4053 elif getResponses[ i ] == main.ERROR:
4054 getResults = main.FALSE
4055 sizeResponses = []
4056 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004057 for i in range( main.numCtrls ):
4058 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004059 name="setTestSize-" + str( i ),
4060 args=[ onosSetName ] )
4061 threads.append( t )
4062 t.start()
4063 for t in threads:
4064 t.join()
4065 sizeResponses.append( t.result )
4066 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004067 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004068 if size != sizeResponses[ i ]:
4069 sizeResults = main.FALSE
4070 main.log.error( "ONOS" + str( i + 1 ) +
4071 " expected a size of " + str( size ) +
4072 " for set " + onosSetName +
4073 " but got " + str( sizeResponses[ i ] ) )
4074 addAllResults = addAllResults and getResults and sizeResults
4075 utilities.assert_equals( expect=main.TRUE,
4076 actual=addAllResults,
4077 onpass="Set addAll correct",
4078 onfail="Set addAll was incorrect" )
4079
4080 main.step( "Distributed Set clear()" )
4081 onosSet.clear()
4082 clearResponses = []
4083 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004084 for i in range( main.numCtrls ):
4085 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004086 name="setTestClear-" + str( i ),
4087 args=[ onosSetName, " "], # Values doesn't matter
4088 kwargs={ "clear": True } )
4089 threads.append( t )
4090 t.start()
4091 for t in threads:
4092 t.join()
4093 clearResponses.append( t.result )
4094
4095 # main.TRUE = successfully changed the set
4096 # main.FALSE = action resulted in no change in set
4097 # main.ERROR - Some error in executing the function
4098 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004099 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004100 if clearResponses[ i ] == main.TRUE:
4101 # All is well
4102 pass
4103 elif clearResponses[ i ] == main.FALSE:
4104 # Nothing set, probably fine
4105 pass
4106 elif clearResponses[ i ] == main.ERROR:
4107 # Error in execution
4108 clearResults = main.FALSE
4109 else:
4110 # unexpected result
4111 clearResults = main.FALSE
4112 if clearResults != main.TRUE:
4113 main.log.error( "Error executing set clear" )
4114
4115 # Check if set is still correct
4116 size = len( onosSet )
4117 getResponses = []
4118 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004119 for i in range( main.numCtrls ):
4120 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004121 name="setTestGet-" + str( i ),
4122 args=[ onosSetName ] )
4123 threads.append( t )
4124 t.start()
4125 for t in threads:
4126 t.join()
4127 getResponses.append( t.result )
4128 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004129 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004130 if isinstance( getResponses[ i ], list):
4131 current = set( getResponses[ i ] )
4132 if len( current ) == len( getResponses[ i ] ):
4133 # no repeats
4134 if onosSet != current:
4135 main.log.error( "ONOS" + str( i + 1 ) +
4136 " has incorrect view" +
4137 " of set " + onosSetName + ":\n" +
4138 str( getResponses[ i ] ) )
4139 main.log.debug( "Expected: " + str( onosSet ) )
4140 main.log.debug( "Actual: " + str( current ) )
4141 getResults = main.FALSE
4142 else:
4143 # error, set is not a set
4144 main.log.error( "ONOS" + str( i + 1 ) +
4145 " has repeat elements in" +
4146 " set " + onosSetName + ":\n" +
4147 str( getResponses[ i ] ) )
4148 getResults = main.FALSE
4149 elif getResponses[ i ] == main.ERROR:
4150 getResults = main.FALSE
4151 sizeResponses = []
4152 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004153 for i in range( main.numCtrls ):
4154 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004155 name="setTestSize-" + str( i ),
4156 args=[ onosSetName ] )
4157 threads.append( t )
4158 t.start()
4159 for t in threads:
4160 t.join()
4161 sizeResponses.append( t.result )
4162 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004163 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004164 if size != sizeResponses[ i ]:
4165 sizeResults = main.FALSE
4166 main.log.error( "ONOS" + str( i + 1 ) +
4167 " expected a size of " + str( size ) +
4168 " for set " + onosSetName +
4169 " but got " + str( sizeResponses[ i ] ) )
4170 clearResults = clearResults and getResults and sizeResults
4171 utilities.assert_equals( expect=main.TRUE,
4172 actual=clearResults,
4173 onpass="Set clear correct",
4174 onfail="Set clear was incorrect" )
4175
4176 main.step( "Distributed Set addAll()" )
4177 onosSet.update( addAllValue.split() )
4178 addResponses = []
4179 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004180 for i in range( main.numCtrls ):
4181 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004182 name="setTestAddAll-" + str( i ),
4183 args=[ onosSetName, addAllValue ] )
4184 threads.append( t )
4185 t.start()
4186 for t in threads:
4187 t.join()
4188 addResponses.append( t.result )
4189
4190 # main.TRUE = successfully changed the set
4191 # main.FALSE = action resulted in no change in set
4192 # main.ERROR - Some error in executing the function
4193 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004194 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004195 if addResponses[ i ] == main.TRUE:
4196 # All is well
4197 pass
4198 elif addResponses[ i ] == main.FALSE:
4199 # Already in set, probably fine
4200 pass
4201 elif addResponses[ i ] == main.ERROR:
4202 # Error in execution
4203 addAllResults = main.FALSE
4204 else:
4205 # unexpected result
4206 addAllResults = main.FALSE
4207 if addAllResults != main.TRUE:
4208 main.log.error( "Error executing set addAll" )
4209
4210 # Check if set is still correct
4211 size = len( onosSet )
4212 getResponses = []
4213 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004214 for i in range( main.numCtrls ):
4215 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004216 name="setTestGet-" + str( i ),
4217 args=[ onosSetName ] )
4218 threads.append( t )
4219 t.start()
4220 for t in threads:
4221 t.join()
4222 getResponses.append( t.result )
4223 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004224 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004225 if isinstance( getResponses[ i ], list):
4226 current = set( getResponses[ i ] )
4227 if len( current ) == len( getResponses[ i ] ):
4228 # no repeats
4229 if onosSet != current:
4230 main.log.error( "ONOS" + str( i + 1 ) +
4231 " has incorrect view" +
4232 " of set " + onosSetName + ":\n" +
4233 str( getResponses[ i ] ) )
4234 main.log.debug( "Expected: " + str( onosSet ) )
4235 main.log.debug( "Actual: " + str( current ) )
4236 getResults = main.FALSE
4237 else:
4238 # error, set is not a set
4239 main.log.error( "ONOS" + str( i + 1 ) +
4240 " has repeat elements in" +
4241 " set " + onosSetName + ":\n" +
4242 str( getResponses[ i ] ) )
4243 getResults = main.FALSE
4244 elif getResponses[ i ] == main.ERROR:
4245 getResults = main.FALSE
4246 sizeResponses = []
4247 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004248 for i in range( main.numCtrls ):
4249 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004250 name="setTestSize-" + str( i ),
4251 args=[ onosSetName ] )
4252 threads.append( t )
4253 t.start()
4254 for t in threads:
4255 t.join()
4256 sizeResponses.append( t.result )
4257 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004258 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004259 if size != sizeResponses[ i ]:
4260 sizeResults = main.FALSE
4261 main.log.error( "ONOS" + str( i + 1 ) +
4262 " expected a size of " + str( size ) +
4263 " for set " + onosSetName +
4264 " but got " + str( sizeResponses[ i ] ) )
4265 addAllResults = addAllResults and getResults and sizeResults
4266 utilities.assert_equals( expect=main.TRUE,
4267 actual=addAllResults,
4268 onpass="Set addAll correct",
4269 onfail="Set addAll was incorrect" )
4270
4271 main.step( "Distributed Set retain()" )
4272 onosSet.intersection_update( retainValue.split() )
4273 retainResponses = []
4274 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004275 for i in range( main.numCtrls ):
4276 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004277 name="setTestRetain-" + str( i ),
4278 args=[ onosSetName, retainValue ],
4279 kwargs={ "retain": True } )
4280 threads.append( t )
4281 t.start()
4282 for t in threads:
4283 t.join()
4284 retainResponses.append( t.result )
4285
4286 # main.TRUE = successfully changed the set
4287 # main.FALSE = action resulted in no change in set
4288 # main.ERROR - Some error in executing the function
4289 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004290 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004291 if retainResponses[ i ] == main.TRUE:
4292 # All is well
4293 pass
4294 elif retainResponses[ i ] == main.FALSE:
4295 # Already in set, probably fine
4296 pass
4297 elif retainResponses[ i ] == main.ERROR:
4298 # Error in execution
4299 retainResults = main.FALSE
4300 else:
4301 # unexpected result
4302 retainResults = main.FALSE
4303 if retainResults != main.TRUE:
4304 main.log.error( "Error executing set retain" )
4305
4306 # Check if set is still correct
4307 size = len( onosSet )
4308 getResponses = []
4309 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004310 for i in range( main.numCtrls ):
4311 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004312 name="setTestGet-" + str( i ),
4313 args=[ onosSetName ] )
4314 threads.append( t )
4315 t.start()
4316 for t in threads:
4317 t.join()
4318 getResponses.append( t.result )
4319 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004320 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004321 if isinstance( getResponses[ i ], list):
4322 current = set( getResponses[ i ] )
4323 if len( current ) == len( getResponses[ i ] ):
4324 # no repeats
4325 if onosSet != current:
4326 main.log.error( "ONOS" + str( i + 1 ) +
4327 " has incorrect view" +
4328 " of set " + onosSetName + ":\n" +
4329 str( getResponses[ i ] ) )
4330 main.log.debug( "Expected: " + str( onosSet ) )
4331 main.log.debug( "Actual: " + str( current ) )
4332 getResults = main.FALSE
4333 else:
4334 # error, set is not a set
4335 main.log.error( "ONOS" + str( i + 1 ) +
4336 " has repeat elements in" +
4337 " set " + onosSetName + ":\n" +
4338 str( getResponses[ i ] ) )
4339 getResults = main.FALSE
4340 elif getResponses[ i ] == main.ERROR:
4341 getResults = main.FALSE
4342 sizeResponses = []
4343 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004344 for i in range( main.numCtrls ):
4345 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004346 name="setTestSize-" + str( i ),
4347 args=[ onosSetName ] )
4348 threads.append( t )
4349 t.start()
4350 for t in threads:
4351 t.join()
4352 sizeResponses.append( t.result )
4353 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004354 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004355 if size != sizeResponses[ i ]:
4356 sizeResults = main.FALSE
4357 main.log.error( "ONOS" + str( i + 1 ) +
4358 " expected a size of " +
4359 str( size ) + " for set " + onosSetName +
4360 " but got " + str( sizeResponses[ i ] ) )
4361 retainResults = retainResults and getResults and sizeResults
4362 utilities.assert_equals( expect=main.TRUE,
4363 actual=retainResults,
4364 onpass="Set retain correct",
4365 onfail="Set retain was incorrect" )
4366
Jon Hall2a5002c2015-08-21 16:49:11 -07004367 # Transactional maps
4368 main.step( "Partitioned Transactional maps put" )
4369 tMapValue = "Testing"
4370 numKeys = 100
4371 putResult = True
4372 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4373 if len( putResponses ) == 100:
4374 for i in putResponses:
4375 if putResponses[ i ][ 'value' ] != tMapValue:
4376 putResult = False
4377 else:
4378 putResult = False
4379 if not putResult:
4380 main.log.debug( "Put response values: " + str( putResponses ) )
4381 utilities.assert_equals( expect=True,
4382 actual=putResult,
4383 onpass="Partitioned Transactional Map put successful",
4384 onfail="Partitioned Transactional Map put values are incorrect" )
4385
4386 main.step( "Partitioned Transactional maps get" )
4387 getCheck = True
4388 for n in range( 1, numKeys + 1 ):
4389 getResponses = []
4390 threads = []
4391 valueCheck = True
4392 for i in range( main.numCtrls ):
4393 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4394 name="TMap-get-" + str( i ),
4395 args=[ "Key" + str ( n ) ] )
4396 threads.append( t )
4397 t.start()
4398 for t in threads:
4399 t.join()
4400 getResponses.append( t.result )
4401 for node in getResponses:
4402 if node != tMapValue:
4403 valueCheck = False
4404 if not valueCheck:
4405 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4406 main.log.warn( getResponses )
4407 getCheck = getCheck and valueCheck
4408 utilities.assert_equals( expect=True,
4409 actual=getCheck,
4410 onpass="Partitioned Transactional Map get values were correct",
4411 onfail="Partitioned Transactional Map values incorrect" )
4412
4413 main.step( "In-memory Transactional maps put" )
4414 tMapValue = "Testing"
4415 numKeys = 100
4416 putResult = True
4417 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4418 if len( putResponses ) == 100:
4419 for i in putResponses:
4420 if putResponses[ i ][ 'value' ] != tMapValue:
4421 putResult = False
4422 else:
4423 putResult = False
4424 if not putResult:
4425 main.log.debug( "Put response values: " + str( putResponses ) )
4426 utilities.assert_equals( expect=True,
4427 actual=putResult,
4428 onpass="In-Memory Transactional Map put successful",
4429 onfail="In-Memory Transactional Map put values are incorrect" )
4430
4431 main.step( "In-Memory Transactional maps get" )
4432 getCheck = True
4433 for n in range( 1, numKeys + 1 ):
4434 getResponses = []
4435 threads = []
4436 valueCheck = True
4437 for i in range( main.numCtrls ):
4438 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4439 name="TMap-get-" + str( i ),
4440 args=[ "Key" + str ( n ) ],
4441 kwargs={ "inMemory": True } )
4442 threads.append( t )
4443 t.start()
4444 for t in threads:
4445 t.join()
4446 getResponses.append( t.result )
4447 for node in getResponses:
4448 if node != tMapValue:
4449 valueCheck = False
4450 if not valueCheck:
4451 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4452 main.log.warn( getResponses )
4453 getCheck = getCheck and valueCheck
4454 utilities.assert_equals( expect=True,
4455 actual=getCheck,
4456 onpass="In-Memory Transactional Map get values were correct",
4457 onfail="In-Memory Transactional Map values incorrect" )