blob: 4f8fadcdc36fadaa6e4ffc56de4cf1022d657045 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
28class HAsanity:
29
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall5cf14d52015-07-16 12:15:19 -070052 main.log.info( "ONOS HA Sanity test - initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070071 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79
80 # FIXME: just get controller port from params?
81 # TODO: do we really need all these?
82 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
83 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
84 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
85 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
86 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
87 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
88 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
89
Jon Halle1a3b752015-07-22 13:02:46 -070090 try:
91 fileName = "Counters"
92 path = main.params[ 'imports' ][ 'path' ]
93 main.Counters = imp.load_source( fileName,
94 path + fileName + ".py" )
95 except Exception as e:
96 main.log.exception( e )
97 main.cleanup()
98 main.exit()
99
100 main.CLIs = []
101 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700102 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700103 for i in range( 1, main.numCtrls + 1 ):
104 try:
105 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
106 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
107 ipList.append( main.nodes[ -1 ].ip_address )
108 except AttributeError:
109 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700110
111 main.step( "Create cell file" )
112 cellAppString = main.params[ 'ENV' ][ 'appString' ]
113 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
114 main.Mininet1.ip_address,
115 cellAppString, ipList )
116 main.step( "Applying cell variable to environment" )
117 cellResult = main.ONOSbench.setCell( cellName )
118 verifyResult = main.ONOSbench.verifyCell()
119
120 # FIXME:this is short term fix
121 main.log.info( "Removing raft logs" )
122 main.ONOSbench.onosRemoveRaftLogs()
123
124 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700125 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700126 main.ONOSbench.onosUninstall( node.ip_address )
127
128 # Make sure ONOS is DEAD
129 main.log.info( "Killing any ONOS processes" )
130 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700131 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700132 killed = main.ONOSbench.onosKill( node.ip_address )
133 killResults = killResults and killed
134
135 cleanInstallResult = main.TRUE
136 gitPullResult = main.TRUE
137
138 main.step( "Starting Mininet" )
139 # scp topo file to mininet
140 # TODO: move to params?
141 topoName = "obelisk.py"
142 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700143 main.ONOSbench.scp( main.Mininet1,
144 filePath + topoName,
145 main.Mininet1.home,
146 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700147 mnResult = main.Mininet1.startNet( )
148 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
149 onpass="Mininet Started",
150 onfail="Error starting Mininet" )
151
152 main.step( "Git checkout and pull " + gitBranch )
153 if PULLCODE:
154 main.ONOSbench.gitCheckout( gitBranch )
155 gitPullResult = main.ONOSbench.gitPull()
156 # values of 1 or 3 are good
157 utilities.assert_lesser( expect=0, actual=gitPullResult,
158 onpass="Git pull successful",
159 onfail="Git pull failed" )
160 main.ONOSbench.getVersion( report=True )
161
162 main.step( "Using mvn clean install" )
163 cleanInstallResult = main.TRUE
164 if PULLCODE and gitPullResult == main.TRUE:
165 cleanInstallResult = main.ONOSbench.cleanInstall()
166 else:
167 main.log.warn( "Did not pull new code so skipping mvn " +
168 "clean install" )
169 utilities.assert_equals( expect=main.TRUE,
170 actual=cleanInstallResult,
171 onpass="MCI successful",
172 onfail="MCI failed" )
173 # GRAPHS
174 # NOTE: important params here:
175 # job = name of Jenkins job
176 # Plot Name = Plot-HA, only can be used if multiple plots
177 # index = The number of the graph under plot name
178 job = "HAsanity"
179 plotName = "Plot-HA"
180 graphs = '<ac:structured-macro ac:name="html">\n'
181 graphs += '<ac:plain-text-body><![CDATA[\n'
182 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
183 '/plot/' + plotName + '/getPlot?index=0' +\
184 '&width=500&height=300"' +\
185 'noborder="0" width="500" height="300" scrolling="yes" ' +\
186 'seamless="seamless"></iframe>\n'
187 graphs += ']]></ac:plain-text-body>\n'
188 graphs += '</ac:structured-macro>\n'
189 main.log.wiki(graphs)
190
191 main.step( "Creating ONOS package" )
192 packageResult = main.ONOSbench.onosPackage()
193 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
194 onpass="ONOS package successful",
195 onfail="ONOS package failed" )
196
197 main.step( "Installing ONOS package" )
198 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700199 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700200 tmpResult = main.ONOSbench.onosInstall( options="-f",
201 node=node.ip_address )
202 onosInstallResult = onosInstallResult and tmpResult
203 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
204 onpass="ONOS install successful",
205 onfail="ONOS install failed" )
206
207 main.step( "Checking if ONOS is up yet" )
208 for i in range( 2 ):
209 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700210 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700211 started = main.ONOSbench.isup( node.ip_address )
212 if not started:
213 main.log.error( node.name + " didn't start!" )
214 main.ONOSbench.onosStop( node.ip_address )
215 main.ONOSbench.onosStart( node.ip_address )
216 onosIsupResult = onosIsupResult and started
217 if onosIsupResult == main.TRUE:
218 break
219 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
220 onpass="ONOS startup successful",
221 onfail="ONOS startup failed" )
222
223 main.log.step( "Starting ONOS CLI sessions" )
224 cliResults = main.TRUE
225 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700226 for i in range( main.numCtrls ):
227 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700228 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700229 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 threads.append( t )
231 t.start()
232
233 for t in threads:
234 t.join()
235 cliResults = cliResults and t.result
236 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
237 onpass="ONOS cli startup successful",
238 onfail="ONOS cli startup failed" )
239
240 if main.params[ 'tcpdump' ].lower() == "true":
241 main.step( "Start Packet Capture MN" )
242 main.Mininet2.startTcpdump(
243 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
244 + "-MN.pcap",
245 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
246 port=main.params[ 'MNtcpdump' ][ 'port' ] )
247
248 main.step( "App Ids check" )
249 appCheck = main.TRUE
250 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700251 for i in range( main.numCtrls ):
252 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700253 name="appToIDCheck-" + str( i ),
254 args=[] )
255 threads.append( t )
256 t.start()
257
258 for t in threads:
259 t.join()
260 appCheck = appCheck and t.result
261 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700262 main.log.warn( main.CLIs[0].apps() )
263 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700264 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
265 onpass="App Ids seem to be correct",
266 onfail="Something is wrong with app Ids" )
267
268 if cliResults == main.FALSE:
269 main.log.error( "Failed to start ONOS, stopping test" )
270 main.cleanup()
271 main.exit()
272
273 def CASE2( self, main ):
274 """
275 Assign devices to controllers
276 """
277 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700278 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700279 assert main, "main not defined"
280 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700281 assert main.CLIs, "main.CLIs not defined"
282 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700283 assert ONOS1Port, "ONOS1Port not defined"
284 assert ONOS2Port, "ONOS2Port not defined"
285 assert ONOS3Port, "ONOS3Port not defined"
286 assert ONOS4Port, "ONOS4Port not defined"
287 assert ONOS5Port, "ONOS5Port not defined"
288 assert ONOS6Port, "ONOS6Port not defined"
289 assert ONOS7Port, "ONOS7Port not defined"
290
291 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700292 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700293 "and check that an ONOS node becomes the " +\
294 "master of the device."
295 main.step( "Assign switches to controllers" )
296
297 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700298 for i in range( main.numCtrls ):
299 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700300 swList = []
301 for i in range( 1, 29 ):
302 swList.append( "s" + str( i ) )
303 main.Mininet1.assignSwController( sw=swList, ip=ipList )
304
305 mastershipCheck = main.TRUE
306 for i in range( 1, 29 ):
307 response = main.Mininet1.getSwController( "s" + str( i ) )
308 try:
309 main.log.info( str( response ) )
310 except Exception:
311 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700312 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700313 if re.search( "tcp:" + node.ip_address, response ):
314 mastershipCheck = mastershipCheck and main.TRUE
315 else:
316 main.log.error( "Error, node " + node.ip_address + " is " +
317 "not in the list of controllers s" +
318 str( i ) + " is connecting to." )
319 mastershipCheck = main.FALSE
320 utilities.assert_equals(
321 expect=main.TRUE,
322 actual=mastershipCheck,
323 onpass="Switch mastership assigned correctly",
324 onfail="Switches not assigned correctly to controllers" )
325
326 def CASE21( self, main ):
327 """
328 Assign mastership to controllers
329 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700330 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700331 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700332 assert main, "main not defined"
333 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700334 assert main.CLIs, "main.CLIs not defined"
335 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700336 assert ONOS1Port, "ONOS1Port not defined"
337 assert ONOS2Port, "ONOS2Port not defined"
338 assert ONOS3Port, "ONOS3Port not defined"
339 assert ONOS4Port, "ONOS4Port not defined"
340 assert ONOS5Port, "ONOS5Port not defined"
341 assert ONOS6Port, "ONOS6Port not defined"
342 assert ONOS7Port, "ONOS7Port not defined"
343
344 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700345 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700346 "device. Then manually assign" +\
347 " mastership to specific ONOS nodes using" +\
348 " 'device-role'"
349 main.step( "Assign mastership of switches to specific controllers" )
350 # Manually assign mastership to the controller we want
351 roleCall = main.TRUE
352
353 ipList = [ ]
354 deviceList = []
355 try:
356 # Assign mastership to specific controllers. This assignment was
357 # determined for a 7 node cluser, but will work with any sized
358 # cluster
359 for i in range( 1, 29 ): # switches 1 through 28
360 # set up correct variables:
361 if i == 1:
362 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700363 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700364 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
365 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700366 c = 1 % main.numCtrls
367 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
369 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700370 c = 1 % main.numCtrls
371 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
373 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700374 c = 3 % main.numCtrls
375 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700376 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
377 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700378 c = 2 % main.numCtrls
379 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700380 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
381 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700382 c = 2 % main.numCtrls
383 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700384 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
385 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700386 c = 5 % main.numCtrls
387 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700388 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
389 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700390 c = 4 % main.numCtrls
391 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700392 dpid = '3' + str( i ).zfill( 3 )
393 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
394 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700395 c = 6 % main.numCtrls
396 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700397 dpid = '6' + str( i ).zfill( 3 )
398 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
399 elif i == 28:
400 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700401 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700402 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
403 else:
404 main.log.error( "You didn't write an else statement for " +
405 "switch s" + str( i ) )
406 roleCall = main.FALSE
407 # Assign switch
408 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
409 # TODO: make this controller dynamic
410 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
411 ip )
412 ipList.append( ip )
413 deviceList.append( deviceId )
414 except ( AttributeError, AssertionError ):
415 main.log.exception( "Something is wrong with ONOS device view" )
416 main.log.info( main.ONOScli1.devices() )
417 utilities.assert_equals(
418 expect=main.TRUE,
419 actual=roleCall,
420 onpass="Re-assigned switch mastership to designated controller",
421 onfail="Something wrong with deviceRole calls" )
422
423 main.step( "Check mastership was correctly assigned" )
424 roleCheck = main.TRUE
425 # NOTE: This is due to the fact that device mastership change is not
426 # atomic and is actually a multi step process
427 time.sleep( 5 )
428 for i in range( len( ipList ) ):
429 ip = ipList[i]
430 deviceId = deviceList[i]
431 # Check assignment
432 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
433 if ip in master:
434 roleCheck = roleCheck and main.TRUE
435 else:
436 roleCheck = roleCheck and main.FALSE
437 main.log.error( "Error, controller " + ip + " is not" +
438 " master " + "of device " +
439 str( deviceId ) + ". Master is " +
440 repr( master ) + "." )
441 utilities.assert_equals(
442 expect=main.TRUE,
443 actual=roleCheck,
444 onpass="Switches were successfully reassigned to designated " +
445 "controller",
446 onfail="Switches were not successfully reassigned" )
447
448 def CASE3( self, main ):
449 """
450 Assign intents
451 """
452 import time
453 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700454 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700455 assert main, "main not defined"
456 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700457 assert main.CLIs, "main.CLIs not defined"
458 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700460 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700461 "assign predetermined host-to-host intents." +\
462 " After installation, check that the intent" +\
463 " is distributed to all nodes and the state" +\
464 " is INSTALLED"
465
466 # install onos-app-fwd
467 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700468 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700469 utilities.assert_equals( expect=main.TRUE, actual=installResults,
470 onpass="Install fwd successful",
471 onfail="Install fwd failed" )
472
473 main.step( "Check app ids" )
474 appCheck = main.TRUE
475 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700476 for i in range( main.numCtrls ):
477 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700478 name="appToIDCheck-" + str( i ),
479 args=[] )
480 threads.append( t )
481 t.start()
482
483 for t in threads:
484 t.join()
485 appCheck = appCheck and t.result
486 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700487 main.log.warn( main.CLIs[0].apps() )
488 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700489 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
490 onpass="App Ids seem to be correct",
491 onfail="Something is wrong with app Ids" )
492
493 main.step( "Discovering Hosts( Via pingall for now )" )
494 # FIXME: Once we have a host discovery mechanism, use that instead
495 # REACTIVE FWD test
496 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700497 passMsg = "Reactive Pingall test passed"
498 time1 = time.time()
499 pingResult = main.Mininet1.pingall()
500 time2 = time.time()
501 if not pingResult:
502 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700503 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700504 passMsg += " on the second try"
505 utilities.assert_equals(
506 expect=main.TRUE,
507 actual=pingResult,
508 onpass= passMsg,
509 onfail="Reactive Pingall failed, " +
510 "one or more ping pairs failed" )
511 main.log.info( "Time for pingall: %2f seconds" %
512 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700513 # timeout for fwd flows
514 time.sleep( 11 )
515 # uninstall onos-app-fwd
516 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700517 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700518 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
519 onpass="Uninstall fwd successful",
520 onfail="Uninstall fwd failed" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700521
522 main.step( "Check app ids" )
523 threads = []
524 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700525 for i in range( main.numCtrls ):
526 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700527 name="appToIDCheck-" + str( i ),
528 args=[] )
529 threads.append( t )
530 t.start()
531
532 for t in threads:
533 t.join()
534 appCheck2 = appCheck2 and t.result
535 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700536 main.log.warn( main.CLIs[0].apps() )
537 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700538 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
539 onpass="App Ids seem to be correct",
540 onfail="Something is wrong with app Ids" )
541
542 main.step( "Add host intents via cli" )
543 intentIds = []
544 # TODO: move the host numbers to params
545 # Maybe look at all the paths we ping?
546 intentAddResult = True
547 hostResult = main.TRUE
548 for i in range( 8, 18 ):
549 main.log.info( "Adding host intent between h" + str( i ) +
550 " and h" + str( i + 10 ) )
551 host1 = "00:00:00:00:00:" + \
552 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
553 host2 = "00:00:00:00:00:" + \
554 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
555 # NOTE: getHost can return None
556 host1Dict = main.ONOScli1.getHost( host1 )
557 host2Dict = main.ONOScli1.getHost( host2 )
558 host1Id = None
559 host2Id = None
560 if host1Dict and host2Dict:
561 host1Id = host1Dict.get( 'id', None )
562 host2Id = host2Dict.get( 'id', None )
563 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700564 nodeNum = ( i % main.numCtrls )
565 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700566 if tmpId:
567 main.log.info( "Added intent with id: " + tmpId )
568 intentIds.append( tmpId )
569 else:
570 main.log.error( "addHostIntent returned: " +
571 repr( tmpId ) )
572 else:
573 main.log.error( "Error, getHost() failed for h" + str( i ) +
574 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700575 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700576 main.log.warn( "Hosts output: " )
577 try:
578 main.log.warn( json.dumps( json.loads( hosts ),
579 sort_keys=True,
580 indent=4,
581 separators=( ',', ': ' ) ) )
582 except ( ValueError, TypeError ):
583 main.log.warn( repr( hosts ) )
584 hostResult = main.FALSE
585 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
586 onpass="Found a host id for each host",
587 onfail="Error looking up host ids" )
588
589 intentStart = time.time()
590 onosIds = main.ONOScli1.getAllIntentsId()
591 main.log.info( "Submitted intents: " + str( intentIds ) )
592 main.log.info( "Intents in ONOS: " + str( onosIds ) )
593 for intent in intentIds:
594 if intent in onosIds:
595 pass # intent submitted is in onos
596 else:
597 intentAddResult = False
598 if intentAddResult:
599 intentStop = time.time()
600 else:
601 intentStop = None
602 # Print the intent states
603 intents = main.ONOScli1.intents()
604 intentStates = []
605 installedCheck = True
606 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
607 count = 0
608 try:
609 for intent in json.loads( intents ):
610 state = intent.get( 'state', None )
611 if "INSTALLED" not in state:
612 installedCheck = False
613 intentId = intent.get( 'id', None )
614 intentStates.append( ( intentId, state ) )
615 except ( ValueError, TypeError ):
616 main.log.exception( "Error parsing intents" )
617 # add submitted intents not in the store
618 tmplist = [ i for i, s in intentStates ]
619 missingIntents = False
620 for i in intentIds:
621 if i not in tmplist:
622 intentStates.append( ( i, " - " ) )
623 missingIntents = True
624 intentStates.sort()
625 for i, s in intentStates:
626 count += 1
627 main.log.info( "%-6s%-15s%-15s" %
628 ( str( count ), str( i ), str( s ) ) )
629 leaders = main.ONOScli1.leaders()
630 try:
631 missing = False
632 if leaders:
633 parsedLeaders = json.loads( leaders )
634 main.log.warn( json.dumps( parsedLeaders,
635 sort_keys=True,
636 indent=4,
637 separators=( ',', ': ' ) ) )
638 # check for all intent partitions
639 topics = []
640 for i in range( 14 ):
641 topics.append( "intent-partition-" + str( i ) )
642 main.log.debug( topics )
643 ONOStopics = [ j['topic'] for j in parsedLeaders ]
644 for topic in topics:
645 if topic not in ONOStopics:
646 main.log.error( "Error: " + topic +
647 " not in leaders" )
648 missing = True
649 else:
650 main.log.error( "leaders() returned None" )
651 except ( ValueError, TypeError ):
652 main.log.exception( "Error parsing leaders" )
653 main.log.error( repr( leaders ) )
654 # Check all nodes
655 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700656 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700657 response = node.leaders( jsonFormat=False)
658 main.log.warn( str( node.name ) + " leaders output: \n" +
659 str( response ) )
660
661 partitions = main.ONOScli1.partitions()
662 try:
663 if partitions :
664 parsedPartitions = json.loads( partitions )
665 main.log.warn( json.dumps( parsedPartitions,
666 sort_keys=True,
667 indent=4,
668 separators=( ',', ': ' ) ) )
669 # TODO check for a leader in all paritions
670 # TODO check for consistency among nodes
671 else:
672 main.log.error( "partitions() returned None" )
673 except ( ValueError, TypeError ):
674 main.log.exception( "Error parsing partitions" )
675 main.log.error( repr( partitions ) )
676 pendingMap = main.ONOScli1.pendingMap()
677 try:
678 if pendingMap :
679 parsedPending = json.loads( pendingMap )
680 main.log.warn( json.dumps( parsedPending,
681 sort_keys=True,
682 indent=4,
683 separators=( ',', ': ' ) ) )
684 # TODO check something here?
685 else:
686 main.log.error( "pendingMap() returned None" )
687 except ( ValueError, TypeError ):
688 main.log.exception( "Error parsing pending map" )
689 main.log.error( repr( pendingMap ) )
690
691 intentAddResult = bool( intentAddResult and not missingIntents and
692 installedCheck )
693 if not intentAddResult:
694 main.log.error( "Error in pushing host intents to ONOS" )
695
696 main.step( "Intent Anti-Entropy dispersion" )
697 for i in range(100):
698 correct = True
699 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700700 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700701 onosIds = []
702 ids = cli.getAllIntentsId()
703 onosIds.append( ids )
704 main.log.debug( "Intents in " + cli.name + ": " +
705 str( sorted( onosIds ) ) )
706 if sorted( ids ) != sorted( intentIds ):
707 main.log.warn( "Set of intent IDs doesn't match" )
708 correct = False
709 break
710 else:
711 intents = json.loads( cli.intents() )
712 for intent in intents:
713 if intent[ 'state' ] != "INSTALLED":
714 main.log.warn( "Intent " + intent[ 'id' ] +
715 " is " + intent[ 'state' ] )
716 correct = False
717 break
718 if correct:
719 break
720 else:
721 time.sleep(1)
722 if not intentStop:
723 intentStop = time.time()
724 global gossipTime
725 gossipTime = intentStop - intentStart
726 main.log.info( "It took about " + str( gossipTime ) +
727 " seconds for all intents to appear in each node" )
728 # FIXME: make this time configurable/calculate based off of number of
729 # nodes and gossip rounds
730 utilities.assert_greater_equals(
731 expect=40, actual=gossipTime,
732 onpass="ECM anti-entropy for intents worked within " +
733 "expected time",
734 onfail="Intent ECM anti-entropy took too long" )
735 if gossipTime <= 40:
736 intentAddResult = True
737
738 if not intentAddResult or "key" in pendingMap:
739 import time
740 installedCheck = True
741 main.log.info( "Sleeping 60 seconds to see if intents are found" )
742 time.sleep( 60 )
743 onosIds = main.ONOScli1.getAllIntentsId()
744 main.log.info( "Submitted intents: " + str( intentIds ) )
745 main.log.info( "Intents in ONOS: " + str( onosIds ) )
746 # Print the intent states
747 intents = main.ONOScli1.intents()
748 intentStates = []
749 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
750 count = 0
751 try:
752 for intent in json.loads( intents ):
753 # Iter through intents of a node
754 state = intent.get( 'state', None )
755 if "INSTALLED" not in state:
756 installedCheck = False
757 intentId = intent.get( 'id', None )
758 intentStates.append( ( intentId, state ) )
759 except ( ValueError, TypeError ):
760 main.log.exception( "Error parsing intents" )
761 # add submitted intents not in the store
762 tmplist = [ i for i, s in intentStates ]
763 for i in intentIds:
764 if i not in tmplist:
765 intentStates.append( ( i, " - " ) )
766 intentStates.sort()
767 for i, s in intentStates:
768 count += 1
769 main.log.info( "%-6s%-15s%-15s" %
770 ( str( count ), str( i ), str( s ) ) )
771 leaders = main.ONOScli1.leaders()
772 try:
773 missing = False
774 if leaders:
775 parsedLeaders = json.loads( leaders )
776 main.log.warn( json.dumps( parsedLeaders,
777 sort_keys=True,
778 indent=4,
779 separators=( ',', ': ' ) ) )
780 # check for all intent partitions
781 # check for election
782 topics = []
783 for i in range( 14 ):
784 topics.append( "intent-partition-" + str( i ) )
785 # FIXME: this should only be after we start the app
786 topics.append( "org.onosproject.election" )
787 main.log.debug( topics )
788 ONOStopics = [ j['topic'] for j in parsedLeaders ]
789 for topic in topics:
790 if topic not in ONOStopics:
791 main.log.error( "Error: " + topic +
792 " not in leaders" )
793 missing = True
794 else:
795 main.log.error( "leaders() returned None" )
796 except ( ValueError, TypeError ):
797 main.log.exception( "Error parsing leaders" )
798 main.log.error( repr( leaders ) )
799 # Check all nodes
800 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700801 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700802 response = node.leaders( jsonFormat=False)
803 main.log.warn( str( node.name ) + " leaders output: \n" +
804 str( response ) )
805
806 partitions = main.ONOScli1.partitions()
807 try:
808 if partitions :
809 parsedPartitions = json.loads( partitions )
810 main.log.warn( json.dumps( parsedPartitions,
811 sort_keys=True,
812 indent=4,
813 separators=( ',', ': ' ) ) )
814 # TODO check for a leader in all paritions
815 # TODO check for consistency among nodes
816 else:
817 main.log.error( "partitions() returned None" )
818 except ( ValueError, TypeError ):
819 main.log.exception( "Error parsing partitions" )
820 main.log.error( repr( partitions ) )
821 pendingMap = main.ONOScli1.pendingMap()
822 try:
823 if pendingMap :
824 parsedPending = json.loads( pendingMap )
825 main.log.warn( json.dumps( parsedPending,
826 sort_keys=True,
827 indent=4,
828 separators=( ',', ': ' ) ) )
829 # TODO check something here?
830 else:
831 main.log.error( "pendingMap() returned None" )
832 except ( ValueError, TypeError ):
833 main.log.exception( "Error parsing pending map" )
834 main.log.error( repr( pendingMap ) )
835
836 def CASE4( self, main ):
837 """
838 Ping across added host intents
839 """
840 import json
841 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700842 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700843 assert main, "main not defined"
844 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700845 assert main.CLIs, "main.CLIs not defined"
846 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700847 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700848 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700849 "functionality and check the state of " +\
850 "the intent"
851 main.step( "Ping across added host intents" )
852 PingResult = main.TRUE
853 for i in range( 8, 18 ):
854 ping = main.Mininet1.pingHost( src="h" + str( i ),
855 target="h" + str( i + 10 ) )
856 PingResult = PingResult and ping
857 if ping == main.FALSE:
858 main.log.warn( "Ping failed between h" + str( i ) +
859 " and h" + str( i + 10 ) )
860 elif ping == main.TRUE:
861 main.log.info( "Ping test passed!" )
862 # Don't set PingResult or you'd override failures
863 if PingResult == main.FALSE:
864 main.log.error(
865 "Intents have not been installed correctly, pings failed." )
866 # TODO: pretty print
867 main.log.warn( "ONOS1 intents: " )
868 try:
869 tmpIntents = main.ONOScli1.intents()
870 main.log.warn( json.dumps( json.loads( tmpIntents ),
871 sort_keys=True,
872 indent=4,
873 separators=( ',', ': ' ) ) )
874 except ( ValueError, TypeError ):
875 main.log.warn( repr( tmpIntents ) )
876 utilities.assert_equals(
877 expect=main.TRUE,
878 actual=PingResult,
879 onpass="Intents have been installed correctly and pings work",
880 onfail="Intents have not been installed correctly, pings failed." )
881
882 main.step( "Check Intent state" )
883 installedCheck = False
884 loopCount = 0
885 while not installedCheck and loopCount < 40:
886 installedCheck = True
887 # Print the intent states
888 intents = main.ONOScli1.intents()
889 intentStates = []
890 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
891 count = 0
892 # Iter through intents of a node
893 try:
894 for intent in json.loads( intents ):
895 state = intent.get( 'state', None )
896 if "INSTALLED" not in state:
897 installedCheck = False
898 intentId = intent.get( 'id', None )
899 intentStates.append( ( intentId, state ) )
900 except ( ValueError, TypeError ):
901 main.log.exception( "Error parsing intents." )
902 # Print states
903 intentStates.sort()
904 for i, s in intentStates:
905 count += 1
906 main.log.info( "%-6s%-15s%-15s" %
907 ( str( count ), str( i ), str( s ) ) )
908 if not installedCheck:
909 time.sleep( 1 )
910 loopCount += 1
911 utilities.assert_equals( expect=True, actual=installedCheck,
912 onpass="Intents are all INSTALLED",
913 onfail="Intents are not all in " +
914 "INSTALLED state" )
915
916 main.step( "Check leadership of topics" )
917 leaders = main.ONOScli1.leaders()
918 topicCheck = main.TRUE
919 try:
920 if leaders:
921 parsedLeaders = json.loads( leaders )
922 main.log.warn( json.dumps( parsedLeaders,
923 sort_keys=True,
924 indent=4,
925 separators=( ',', ': ' ) ) )
926 # check for all intent partitions
927 # check for election
928 # TODO: Look at Devices as topics now that it uses this system
929 topics = []
930 for i in range( 14 ):
931 topics.append( "intent-partition-" + str( i ) )
932 # FIXME: this should only be after we start the app
933 # FIXME: topics.append( "org.onosproject.election" )
934 # Print leaders output
935 main.log.debug( topics )
936 ONOStopics = [ j['topic'] for j in parsedLeaders ]
937 for topic in topics:
938 if topic not in ONOStopics:
939 main.log.error( "Error: " + topic +
940 " not in leaders" )
941 topicCheck = main.FALSE
942 else:
943 main.log.error( "leaders() returned None" )
944 topicCheck = main.FALSE
945 except ( ValueError, TypeError ):
946 topicCheck = main.FALSE
947 main.log.exception( "Error parsing leaders" )
948 main.log.error( repr( leaders ) )
949 # TODO: Check for a leader of these topics
950 # Check all nodes
951 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700952 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700953 response = node.leaders( jsonFormat=False)
954 main.log.warn( str( node.name ) + " leaders output: \n" +
955 str( response ) )
956
957 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
958 onpass="intent Partitions is in leaders",
959 onfail="Some topics were lost " )
960 # Print partitions
961 partitions = main.ONOScli1.partitions()
962 try:
963 if partitions :
964 parsedPartitions = json.loads( partitions )
965 main.log.warn( json.dumps( parsedPartitions,
966 sort_keys=True,
967 indent=4,
968 separators=( ',', ': ' ) ) )
969 # TODO check for a leader in all paritions
970 # TODO check for consistency among nodes
971 else:
972 main.log.error( "partitions() returned None" )
973 except ( ValueError, TypeError ):
974 main.log.exception( "Error parsing partitions" )
975 main.log.error( repr( partitions ) )
976 # Print Pending Map
977 pendingMap = main.ONOScli1.pendingMap()
978 try:
979 if pendingMap :
980 parsedPending = json.loads( pendingMap )
981 main.log.warn( json.dumps( parsedPending,
982 sort_keys=True,
983 indent=4,
984 separators=( ',', ': ' ) ) )
985 # TODO check something here?
986 else:
987 main.log.error( "pendingMap() returned None" )
988 except ( ValueError, TypeError ):
989 main.log.exception( "Error parsing pending map" )
990 main.log.error( repr( pendingMap ) )
991
992 if not installedCheck:
993 main.log.info( "Waiting 60 seconds to see if the state of " +
994 "intents change" )
995 time.sleep( 60 )
996 # Print the intent states
997 intents = main.ONOScli1.intents()
998 intentStates = []
999 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1000 count = 0
1001 # Iter through intents of a node
1002 try:
1003 for intent in json.loads( intents ):
1004 state = intent.get( 'state', None )
1005 if "INSTALLED" not in state:
1006 installedCheck = False
1007 intentId = intent.get( 'id', None )
1008 intentStates.append( ( intentId, state ) )
1009 except ( ValueError, TypeError ):
1010 main.log.exception( "Error parsing intents." )
1011 intentStates.sort()
1012 for i, s in intentStates:
1013 count += 1
1014 main.log.info( "%-6s%-15s%-15s" %
1015 ( str( count ), str( i ), str( s ) ) )
1016 leaders = main.ONOScli1.leaders()
1017 try:
1018 missing = False
1019 if leaders:
1020 parsedLeaders = json.loads( leaders )
1021 main.log.warn( json.dumps( parsedLeaders,
1022 sort_keys=True,
1023 indent=4,
1024 separators=( ',', ': ' ) ) )
1025 # check for all intent partitions
1026 # check for election
1027 topics = []
1028 for i in range( 14 ):
1029 topics.append( "intent-partition-" + str( i ) )
1030 # FIXME: this should only be after we start the app
1031 topics.append( "org.onosproject.election" )
1032 main.log.debug( topics )
1033 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1034 for topic in topics:
1035 if topic not in ONOStopics:
1036 main.log.error( "Error: " + topic +
1037 " not in leaders" )
1038 missing = True
1039 else:
1040 main.log.error( "leaders() returned None" )
1041 except ( ValueError, TypeError ):
1042 main.log.exception( "Error parsing leaders" )
1043 main.log.error( repr( leaders ) )
1044 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001045 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001046 response = node.leaders( jsonFormat=False)
1047 main.log.warn( str( node.name ) + " leaders output: \n" +
1048 str( response ) )
1049
1050 partitions = main.ONOScli1.partitions()
1051 try:
1052 if partitions :
1053 parsedPartitions = json.loads( partitions )
1054 main.log.warn( json.dumps( parsedPartitions,
1055 sort_keys=True,
1056 indent=4,
1057 separators=( ',', ': ' ) ) )
1058 # TODO check for a leader in all paritions
1059 # TODO check for consistency among nodes
1060 else:
1061 main.log.error( "partitions() returned None" )
1062 except ( ValueError, TypeError ):
1063 main.log.exception( "Error parsing partitions" )
1064 main.log.error( repr( partitions ) )
1065 pendingMap = main.ONOScli1.pendingMap()
1066 try:
1067 if pendingMap :
1068 parsedPending = json.loads( pendingMap )
1069 main.log.warn( json.dumps( parsedPending,
1070 sort_keys=True,
1071 indent=4,
1072 separators=( ',', ': ' ) ) )
1073 # TODO check something here?
1074 else:
1075 main.log.error( "pendingMap() returned None" )
1076 except ( ValueError, TypeError ):
1077 main.log.exception( "Error parsing pending map" )
1078 main.log.error( repr( pendingMap ) )
1079 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001080 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001081 main.step( "Wait a minute then ping again" )
1082 # the wait is above
1083 PingResult = main.TRUE
1084 for i in range( 8, 18 ):
1085 ping = main.Mininet1.pingHost( src="h" + str( i ),
1086 target="h" + str( i + 10 ) )
1087 PingResult = PingResult and ping
1088 if ping == main.FALSE:
1089 main.log.warn( "Ping failed between h" + str( i ) +
1090 " and h" + str( i + 10 ) )
1091 elif ping == main.TRUE:
1092 main.log.info( "Ping test passed!" )
1093 # Don't set PingResult or you'd override failures
1094 if PingResult == main.FALSE:
1095 main.log.error(
1096 "Intents have not been installed correctly, pings failed." )
1097 # TODO: pretty print
1098 main.log.warn( "ONOS1 intents: " )
1099 try:
1100 tmpIntents = main.ONOScli1.intents()
1101 main.log.warn( json.dumps( json.loads( tmpIntents ),
1102 sort_keys=True,
1103 indent=4,
1104 separators=( ',', ': ' ) ) )
1105 except ( ValueError, TypeError ):
1106 main.log.warn( repr( tmpIntents ) )
1107 utilities.assert_equals(
1108 expect=main.TRUE,
1109 actual=PingResult,
1110 onpass="Intents have been installed correctly and pings work",
1111 onfail="Intents have not been installed correctly, pings failed." )
1112
1113 def CASE5( self, main ):
1114 """
1115 Reading state of ONOS
1116 """
1117 import json
1118 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001119 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001120 assert main, "main not defined"
1121 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001122 assert main.CLIs, "main.CLIs not defined"
1123 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001124
1125 main.case( "Setting up and gathering data for current state" )
1126 # The general idea for this test case is to pull the state of
1127 # ( intents,flows, topology,... ) from each ONOS node
1128 # We can then compare them with each other and also with past states
1129
1130 main.step( "Check that each switch has a master" )
1131 global mastershipState
1132 mastershipState = '[]'
1133
1134 # Assert that each device has a master
1135 rolesNotNull = main.TRUE
1136 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001137 for i in range( main.numCtrls ):
1138 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001139 name="rolesNotNull-" + str( i ),
1140 args=[] )
1141 threads.append( t )
1142 t.start()
1143
1144 for t in threads:
1145 t.join()
1146 rolesNotNull = rolesNotNull and t.result
1147 utilities.assert_equals(
1148 expect=main.TRUE,
1149 actual=rolesNotNull,
1150 onpass="Each device has a master",
1151 onfail="Some devices don't have a master assigned" )
1152
1153 main.step( "Get the Mastership of each switch from each controller" )
1154 ONOSMastership = []
1155 mastershipCheck = main.FALSE
1156 consistentMastership = True
1157 rolesResults = True
1158 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001159 for i in range( main.numCtrls ):
1160 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001161 name="roles-" + str( i ),
1162 args=[] )
1163 threads.append( t )
1164 t.start()
1165
1166 for t in threads:
1167 t.join()
1168 ONOSMastership.append( t.result )
1169
Jon Halle1a3b752015-07-22 13:02:46 -07001170 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001171 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1172 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1173 " roles" )
1174 main.log.warn(
1175 "ONOS" + str( i + 1 ) + " mastership response: " +
1176 repr( ONOSMastership[i] ) )
1177 rolesResults = False
1178 utilities.assert_equals(
1179 expect=True,
1180 actual=rolesResults,
1181 onpass="No error in reading roles output",
1182 onfail="Error in reading roles from ONOS" )
1183
1184 main.step( "Check for consistency in roles from each controller" )
1185 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1186 main.log.info(
1187 "Switch roles are consistent across all ONOS nodes" )
1188 else:
1189 consistentMastership = False
1190 utilities.assert_equals(
1191 expect=True,
1192 actual=consistentMastership,
1193 onpass="Switch roles are consistent across all ONOS nodes",
1194 onfail="ONOS nodes have different views of switch roles" )
1195
1196 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001197 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001198 try:
1199 main.log.warn(
1200 "ONOS" + str( i + 1 ) + " roles: ",
1201 json.dumps(
1202 json.loads( ONOSMastership[ i ] ),
1203 sort_keys=True,
1204 indent=4,
1205 separators=( ',', ': ' ) ) )
1206 except ( ValueError, TypeError ):
1207 main.log.warn( repr( ONOSMastership[ i ] ) )
1208 elif rolesResults and consistentMastership:
1209 mastershipCheck = main.TRUE
1210 mastershipState = ONOSMastership[ 0 ]
1211
1212 main.step( "Get the intents from each controller" )
1213 global intentState
1214 intentState = []
1215 ONOSIntents = []
1216 intentCheck = main.FALSE
1217 consistentIntents = True
1218 intentsResults = True
1219 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001220 for i in range( main.numCtrls ):
1221 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001222 name="intents-" + str( i ),
1223 args=[],
1224 kwargs={ 'jsonFormat': True } )
1225 threads.append( t )
1226 t.start()
1227
1228 for t in threads:
1229 t.join()
1230 ONOSIntents.append( t.result )
1231
Jon Halle1a3b752015-07-22 13:02:46 -07001232 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001233 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1234 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1235 " intents" )
1236 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1237 repr( ONOSIntents[ i ] ) )
1238 intentsResults = False
1239 utilities.assert_equals(
1240 expect=True,
1241 actual=intentsResults,
1242 onpass="No error in reading intents output",
1243 onfail="Error in reading intents from ONOS" )
1244
1245 main.step( "Check for consistency in Intents from each controller" )
1246 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1247 main.log.info( "Intents are consistent across all ONOS " +
1248 "nodes" )
1249 else:
1250 consistentIntents = False
1251 main.log.error( "Intents not consistent" )
1252 utilities.assert_equals(
1253 expect=True,
1254 actual=consistentIntents,
1255 onpass="Intents are consistent across all ONOS nodes",
1256 onfail="ONOS nodes have different views of intents" )
1257
1258 if intentsResults:
1259 # Try to make it easy to figure out what is happening
1260 #
1261 # Intent ONOS1 ONOS2 ...
1262 # 0x01 INSTALLED INSTALLING
1263 # ... ... ...
1264 # ... ... ...
1265 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001266 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001267 title += " " * 10 + "ONOS" + str( n + 1 )
1268 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001269 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001270 keys = []
1271 try:
1272 # Get the set of all intent keys
1273 for nodeStr in ONOSIntents:
1274 node = json.loads( nodeStr )
1275 for intent in node:
1276 keys.append( intent.get( 'id' ) )
1277 keys = set( keys )
1278 # For each intent key, print the state on each node
1279 for key in keys:
1280 row = "%-13s" % key
1281 for nodeStr in ONOSIntents:
1282 node = json.loads( nodeStr )
1283 for intent in node:
1284 if intent.get( 'id', "Error" ) == key:
1285 row += "%-15s" % intent.get( 'state' )
1286 main.log.warn( row )
1287 # End of intent state table
1288 except ValueError as e:
1289 main.log.exception( e )
1290 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1291
1292 if intentsResults and not consistentIntents:
1293 # print the json objects
1294 n = len(ONOSIntents)
1295 main.log.debug( "ONOS" + str( n ) + " intents: " )
1296 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1297 sort_keys=True,
1298 indent=4,
1299 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001300 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001301 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1302 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1303 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1304 sort_keys=True,
1305 indent=4,
1306 separators=( ',', ': ' ) ) )
1307 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001308 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001309 str( n ) + " intents" )
1310 elif intentsResults and consistentIntents:
1311 intentCheck = main.TRUE
1312 intentState = ONOSIntents[ 0 ]
1313
1314 main.step( "Get the flows from each controller" )
1315 global flowState
1316 flowState = []
1317 ONOSFlows = []
1318 ONOSFlowsJson = []
1319 flowCheck = main.FALSE
1320 consistentFlows = True
1321 flowsResults = True
1322 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001323 for i in range( main.numCtrls ):
1324 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001325 name="flows-" + str( i ),
1326 args=[],
1327 kwargs={ 'jsonFormat': True } )
1328 threads.append( t )
1329 t.start()
1330
1331 # NOTE: Flows command can take some time to run
1332 time.sleep(30)
1333 for t in threads:
1334 t.join()
1335 result = t.result
1336 ONOSFlows.append( result )
1337
Jon Halle1a3b752015-07-22 13:02:46 -07001338 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001339 num = str( i + 1 )
1340 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1341 main.log.error( "Error in getting ONOS" + num + " flows" )
1342 main.log.warn( "ONOS" + num + " flows response: " +
1343 repr( ONOSFlows[ i ] ) )
1344 flowsResults = False
1345 ONOSFlowsJson.append( None )
1346 else:
1347 try:
1348 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1349 except ( ValueError, TypeError ):
1350 # FIXME: change this to log.error?
1351 main.log.exception( "Error in parsing ONOS" + num +
1352 " response as json." )
1353 main.log.error( repr( ONOSFlows[ i ] ) )
1354 ONOSFlowsJson.append( None )
1355 flowsResults = False
1356 utilities.assert_equals(
1357 expect=True,
1358 actual=flowsResults,
1359 onpass="No error in reading flows output",
1360 onfail="Error in reading flows from ONOS" )
1361
1362 main.step( "Check for consistency in Flows from each controller" )
1363 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1364 if all( tmp ):
1365 main.log.info( "Flow count is consistent across all ONOS nodes" )
1366 else:
1367 consistentFlows = False
1368 utilities.assert_equals(
1369 expect=True,
1370 actual=consistentFlows,
1371 onpass="The flow count is consistent across all ONOS nodes",
1372 onfail="ONOS nodes have different flow counts" )
1373
1374 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001375 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001376 try:
1377 main.log.warn(
1378 "ONOS" + str( i + 1 ) + " flows: " +
1379 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1380 indent=4, separators=( ',', ': ' ) ) )
1381 except ( ValueError, TypeError ):
1382 main.log.warn(
1383 "ONOS" + str( i + 1 ) + " flows: " +
1384 repr( ONOSFlows[ i ] ) )
1385 elif flowsResults and consistentFlows:
1386 flowCheck = main.TRUE
1387 flowState = ONOSFlows[ 0 ]
1388
1389 main.step( "Get the OF Table entries" )
1390 global flows
1391 flows = []
1392 for i in range( 1, 29 ):
Jon Hall9043c902015-07-30 14:23:44 -07001393 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001394 if flowCheck == main.FALSE:
1395 for table in flows:
1396 main.log.warn( table )
1397 # TODO: Compare switch flow tables with ONOS flow tables
1398
1399 main.step( "Start continuous pings" )
1400 main.Mininet2.pingLong(
1401 src=main.params[ 'PING' ][ 'source1' ],
1402 target=main.params[ 'PING' ][ 'target1' ],
1403 pingTime=500 )
1404 main.Mininet2.pingLong(
1405 src=main.params[ 'PING' ][ 'source2' ],
1406 target=main.params[ 'PING' ][ 'target2' ],
1407 pingTime=500 )
1408 main.Mininet2.pingLong(
1409 src=main.params[ 'PING' ][ 'source3' ],
1410 target=main.params[ 'PING' ][ 'target3' ],
1411 pingTime=500 )
1412 main.Mininet2.pingLong(
1413 src=main.params[ 'PING' ][ 'source4' ],
1414 target=main.params[ 'PING' ][ 'target4' ],
1415 pingTime=500 )
1416 main.Mininet2.pingLong(
1417 src=main.params[ 'PING' ][ 'source5' ],
1418 target=main.params[ 'PING' ][ 'target5' ],
1419 pingTime=500 )
1420 main.Mininet2.pingLong(
1421 src=main.params[ 'PING' ][ 'source6' ],
1422 target=main.params[ 'PING' ][ 'target6' ],
1423 pingTime=500 )
1424 main.Mininet2.pingLong(
1425 src=main.params[ 'PING' ][ 'source7' ],
1426 target=main.params[ 'PING' ][ 'target7' ],
1427 pingTime=500 )
1428 main.Mininet2.pingLong(
1429 src=main.params[ 'PING' ][ 'source8' ],
1430 target=main.params[ 'PING' ][ 'target8' ],
1431 pingTime=500 )
1432 main.Mininet2.pingLong(
1433 src=main.params[ 'PING' ][ 'source9' ],
1434 target=main.params[ 'PING' ][ 'target9' ],
1435 pingTime=500 )
1436 main.Mininet2.pingLong(
1437 src=main.params[ 'PING' ][ 'source10' ],
1438 target=main.params[ 'PING' ][ 'target10' ],
1439 pingTime=500 )
1440
1441 main.step( "Collecting topology information from ONOS" )
1442 devices = []
1443 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001444 for i in range( main.numCtrls ):
1445 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001446 name="devices-" + str( i ),
1447 args=[ ] )
1448 threads.append( t )
1449 t.start()
1450
1451 for t in threads:
1452 t.join()
1453 devices.append( t.result )
1454 hosts = []
1455 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001456 for i in range( main.numCtrls ):
1457 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001458 name="hosts-" + str( i ),
1459 args=[ ] )
1460 threads.append( t )
1461 t.start()
1462
1463 for t in threads:
1464 t.join()
1465 try:
1466 hosts.append( json.loads( t.result ) )
1467 except ( ValueError, TypeError ):
1468 # FIXME: better handling of this, print which node
1469 # Maybe use thread name?
1470 main.log.exception( "Error parsing json output of hosts" )
1471 # FIXME: should this be an empty json object instead?
1472 hosts.append( None )
1473
1474 ports = []
1475 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001476 for i in range( main.numCtrls ):
1477 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001478 name="ports-" + str( i ),
1479 args=[ ] )
1480 threads.append( t )
1481 t.start()
1482
1483 for t in threads:
1484 t.join()
1485 ports.append( t.result )
1486 links = []
1487 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001488 for i in range( main.numCtrls ):
1489 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001490 name="links-" + str( i ),
1491 args=[ ] )
1492 threads.append( t )
1493 t.start()
1494
1495 for t in threads:
1496 t.join()
1497 links.append( t.result )
1498 clusters = []
1499 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001500 for i in range( main.numCtrls ):
1501 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001502 name="clusters-" + str( i ),
1503 args=[ ] )
1504 threads.append( t )
1505 t.start()
1506
1507 for t in threads:
1508 t.join()
1509 clusters.append( t.result )
1510 # Compare json objects for hosts and dataplane clusters
1511
1512 # hosts
1513 main.step( "Host view is consistent across ONOS nodes" )
1514 consistentHostsResult = main.TRUE
1515 for controller in range( len( hosts ) ):
1516 controllerStr = str( controller + 1 )
1517 if "Error" not in hosts[ controller ]:
1518 if hosts[ controller ] == hosts[ 0 ]:
1519 continue
1520 else: # hosts not consistent
1521 main.log.error( "hosts from ONOS" +
1522 controllerStr +
1523 " is inconsistent with ONOS1" )
1524 main.log.warn( repr( hosts[ controller ] ) )
1525 consistentHostsResult = main.FALSE
1526
1527 else:
1528 main.log.error( "Error in getting ONOS hosts from ONOS" +
1529 controllerStr )
1530 consistentHostsResult = main.FALSE
1531 main.log.warn( "ONOS" + controllerStr +
1532 " hosts response: " +
1533 repr( hosts[ controller ] ) )
1534 utilities.assert_equals(
1535 expect=main.TRUE,
1536 actual=consistentHostsResult,
1537 onpass="Hosts view is consistent across all ONOS nodes",
1538 onfail="ONOS nodes have different views of hosts" )
1539
1540 main.step( "Each host has an IP address" )
1541 ipResult = main.TRUE
1542 for controller in range( 0, len( hosts ) ):
1543 controllerStr = str( controller + 1 )
1544 for host in hosts[ controller ]:
1545 if not host.get( 'ipAddresses', [ ] ):
1546 main.log.error( "DEBUG:Error with host ips on controller" +
1547 controllerStr + ": " + str( host ) )
1548 ipResult = main.FALSE
1549 utilities.assert_equals(
1550 expect=main.TRUE,
1551 actual=ipResult,
1552 onpass="The ips of the hosts aren't empty",
1553 onfail="The ip of at least one host is missing" )
1554
1555 # Strongly connected clusters of devices
1556 main.step( "Cluster view is consistent across ONOS nodes" )
1557 consistentClustersResult = main.TRUE
1558 for controller in range( len( clusters ) ):
1559 controllerStr = str( controller + 1 )
1560 if "Error" not in clusters[ controller ]:
1561 if clusters[ controller ] == clusters[ 0 ]:
1562 continue
1563 else: # clusters not consistent
1564 main.log.error( "clusters from ONOS" + controllerStr +
1565 " is inconsistent with ONOS1" )
1566 consistentClustersResult = main.FALSE
1567
1568 else:
1569 main.log.error( "Error in getting dataplane clusters " +
1570 "from ONOS" + controllerStr )
1571 consistentClustersResult = main.FALSE
1572 main.log.warn( "ONOS" + controllerStr +
1573 " clusters response: " +
1574 repr( clusters[ controller ] ) )
1575 utilities.assert_equals(
1576 expect=main.TRUE,
1577 actual=consistentClustersResult,
1578 onpass="Clusters view is consistent across all ONOS nodes",
1579 onfail="ONOS nodes have different views of clusters" )
1580 # there should always only be one cluster
1581 main.step( "Cluster view correct across ONOS nodes" )
1582 try:
1583 numClusters = len( json.loads( clusters[ 0 ] ) )
1584 except ( ValueError, TypeError ):
1585 main.log.exception( "Error parsing clusters[0]: " +
1586 repr( clusters[ 0 ] ) )
1587 clusterResults = main.FALSE
1588 if numClusters == 1:
1589 clusterResults = main.TRUE
1590 utilities.assert_equals(
1591 expect=1,
1592 actual=numClusters,
1593 onpass="ONOS shows 1 SCC",
1594 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1595
1596 main.step( "Comparing ONOS topology to MN" )
1597 devicesResults = main.TRUE
1598 linksResults = main.TRUE
1599 hostsResults = main.TRUE
1600 mnSwitches = main.Mininet1.getSwitches()
1601 mnLinks = main.Mininet1.getLinks()
1602 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001603 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001604 controllerStr = str( controller + 1 )
1605 if devices[ controller ] and ports[ controller ] and\
1606 "Error" not in devices[ controller ] and\
1607 "Error" not in ports[ controller ]:
1608
1609 currentDevicesResult = main.Mininet1.compareSwitches(
1610 mnSwitches,
1611 json.loads( devices[ controller ] ),
1612 json.loads( ports[ controller ] ) )
1613 else:
1614 currentDevicesResult = main.FALSE
1615 utilities.assert_equals( expect=main.TRUE,
1616 actual=currentDevicesResult,
1617 onpass="ONOS" + controllerStr +
1618 " Switches view is correct",
1619 onfail="ONOS" + controllerStr +
1620 " Switches view is incorrect" )
1621 if links[ controller ] and "Error" not in links[ controller ]:
1622 currentLinksResult = main.Mininet1.compareLinks(
1623 mnSwitches, mnLinks,
1624 json.loads( links[ controller ] ) )
1625 else:
1626 currentLinksResult = main.FALSE
1627 utilities.assert_equals( expect=main.TRUE,
1628 actual=currentLinksResult,
1629 onpass="ONOS" + controllerStr +
1630 " links view is correct",
1631 onfail="ONOS" + controllerStr +
1632 " links view is incorrect" )
1633
1634 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1635 currentHostsResult = main.Mininet1.compareHosts(
1636 mnHosts,
1637 hosts[ controller ] )
1638 else:
1639 currentHostsResult = main.FALSE
1640 utilities.assert_equals( expect=main.TRUE,
1641 actual=currentHostsResult,
1642 onpass="ONOS" + controllerStr +
1643 " hosts exist in Mininet",
1644 onfail="ONOS" + controllerStr +
1645 " hosts don't match Mininet" )
1646
1647 devicesResults = devicesResults and currentDevicesResult
1648 linksResults = linksResults and currentLinksResult
1649 hostsResults = hostsResults and currentHostsResult
1650
1651 main.step( "Device information is correct" )
1652 utilities.assert_equals(
1653 expect=main.TRUE,
1654 actual=devicesResults,
1655 onpass="Device information is correct",
1656 onfail="Device information is incorrect" )
1657
1658 main.step( "Links are correct" )
1659 utilities.assert_equals(
1660 expect=main.TRUE,
1661 actual=linksResults,
1662 onpass="Link are correct",
1663 onfail="Links are incorrect" )
1664
1665 main.step( "Hosts are correct" )
1666 utilities.assert_equals(
1667 expect=main.TRUE,
1668 actual=hostsResults,
1669 onpass="Hosts are correct",
1670 onfail="Hosts are incorrect" )
1671
1672 def CASE6( self, main ):
1673 """
1674 The Failure case. Since this is the Sanity test, we do nothing.
1675 """
1676 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001677 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001678 assert main, "main not defined"
1679 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001680 assert main.CLIs, "main.CLIs not defined"
1681 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001682 main.case( "Wait 60 seconds instead of inducing a failure" )
1683 time.sleep( 60 )
1684 utilities.assert_equals(
1685 expect=main.TRUE,
1686 actual=main.TRUE,
1687 onpass="Sleeping 60 seconds",
1688 onfail="Something is terribly wrong with my math" )
1689
1690 def CASE7( self, main ):
1691 """
1692 Check state after ONOS failure
1693 """
1694 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001695 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001696 assert main, "main not defined"
1697 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001698 assert main.CLIs, "main.CLIs not defined"
1699 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001700 main.case( "Running ONOS Constant State Tests" )
1701
1702 main.step( "Check that each switch has a master" )
1703 # Assert that each device has a master
1704 rolesNotNull = main.TRUE
1705 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001706 for i in range( main.numCtrls ):
1707 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001708 name="rolesNotNull-" + str( i ),
1709 args=[ ] )
1710 threads.append( t )
1711 t.start()
1712
1713 for t in threads:
1714 t.join()
1715 rolesNotNull = rolesNotNull and t.result
1716 utilities.assert_equals(
1717 expect=main.TRUE,
1718 actual=rolesNotNull,
1719 onpass="Each device has a master",
1720 onfail="Some devices don't have a master assigned" )
1721
1722 main.step( "Read device roles from ONOS" )
1723 ONOSMastership = []
1724 mastershipCheck = main.FALSE
1725 consistentMastership = True
1726 rolesResults = True
1727 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001728 for i in range( main.numCtrls ):
1729 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001730 name="roles-" + str( i ),
1731 args=[] )
1732 threads.append( t )
1733 t.start()
1734
1735 for t in threads:
1736 t.join()
1737 ONOSMastership.append( t.result )
1738
Jon Halle1a3b752015-07-22 13:02:46 -07001739 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001740 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1741 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1742 " roles" )
1743 main.log.warn(
1744 "ONOS" + str( i + 1 ) + " mastership response: " +
1745 repr( ONOSMastership[i] ) )
1746 rolesResults = False
1747 utilities.assert_equals(
1748 expect=True,
1749 actual=rolesResults,
1750 onpass="No error in reading roles output",
1751 onfail="Error in reading roles from ONOS" )
1752
1753 main.step( "Check for consistency in roles from each controller" )
1754 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1755 main.log.info(
1756 "Switch roles are consistent across all ONOS nodes" )
1757 else:
1758 consistentMastership = False
1759 utilities.assert_equals(
1760 expect=True,
1761 actual=consistentMastership,
1762 onpass="Switch roles are consistent across all ONOS nodes",
1763 onfail="ONOS nodes have different views of switch roles" )
1764
1765 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001766 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001767 main.log.warn(
1768 "ONOS" + str( i + 1 ) + " roles: ",
1769 json.dumps(
1770 json.loads( ONOSMastership[ i ] ),
1771 sort_keys=True,
1772 indent=4,
1773 separators=( ',', ': ' ) ) )
1774 elif rolesResults and not consistentMastership:
1775 mastershipCheck = main.TRUE
1776
1777 description2 = "Compare switch roles from before failure"
1778 main.step( description2 )
1779 try:
1780 currentJson = json.loads( ONOSMastership[0] )
1781 oldJson = json.loads( mastershipState )
1782 except ( ValueError, TypeError ):
1783 main.log.exception( "Something is wrong with parsing " +
1784 "ONOSMastership[0] or mastershipState" )
1785 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1786 main.log.error( "mastershipState" + repr( mastershipState ) )
1787 main.cleanup()
1788 main.exit()
1789 mastershipCheck = main.TRUE
1790 for i in range( 1, 29 ):
1791 switchDPID = str(
1792 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1793 current = [ switch[ 'master' ] for switch in currentJson
1794 if switchDPID in switch[ 'id' ] ]
1795 old = [ switch[ 'master' ] for switch in oldJson
1796 if switchDPID in switch[ 'id' ] ]
1797 if current == old:
1798 mastershipCheck = mastershipCheck and main.TRUE
1799 else:
1800 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1801 mastershipCheck = main.FALSE
1802 utilities.assert_equals(
1803 expect=main.TRUE,
1804 actual=mastershipCheck,
1805 onpass="Mastership of Switches was not changed",
1806 onfail="Mastership of some switches changed" )
1807 mastershipCheck = mastershipCheck and consistentMastership
1808
1809 main.step( "Get the intents and compare across all nodes" )
1810 ONOSIntents = []
1811 intentCheck = main.FALSE
1812 consistentIntents = True
1813 intentsResults = True
1814 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001815 for i in range( main.numCtrls ):
1816 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001817 name="intents-" + str( i ),
1818 args=[],
1819 kwargs={ 'jsonFormat': True } )
1820 threads.append( t )
1821 t.start()
1822
1823 for t in threads:
1824 t.join()
1825 ONOSIntents.append( t.result )
1826
Jon Halle1a3b752015-07-22 13:02:46 -07001827 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001828 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1829 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1830 " intents" )
1831 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1832 repr( ONOSIntents[ i ] ) )
1833 intentsResults = False
1834 utilities.assert_equals(
1835 expect=True,
1836 actual=intentsResults,
1837 onpass="No error in reading intents output",
1838 onfail="Error in reading intents from ONOS" )
1839
1840 main.step( "Check for consistency in Intents from each controller" )
1841 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1842 main.log.info( "Intents are consistent across all ONOS " +
1843 "nodes" )
1844 else:
1845 consistentIntents = False
1846
1847 # Try to make it easy to figure out what is happening
1848 #
1849 # Intent ONOS1 ONOS2 ...
1850 # 0x01 INSTALLED INSTALLING
1851 # ... ... ...
1852 # ... ... ...
1853 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001854 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001855 title += " " * 10 + "ONOS" + str( n + 1 )
1856 main.log.warn( title )
1857 # get all intent keys in the cluster
1858 keys = []
1859 for nodeStr in ONOSIntents:
1860 node = json.loads( nodeStr )
1861 for intent in node:
1862 keys.append( intent.get( 'id' ) )
1863 keys = set( keys )
1864 for key in keys:
1865 row = "%-13s" % key
1866 for nodeStr in ONOSIntents:
1867 node = json.loads( nodeStr )
1868 for intent in node:
1869 if intent.get( 'id' ) == key:
1870 row += "%-15s" % intent.get( 'state' )
1871 main.log.warn( row )
1872 # End table view
1873
1874 utilities.assert_equals(
1875 expect=True,
1876 actual=consistentIntents,
1877 onpass="Intents are consistent across all ONOS nodes",
1878 onfail="ONOS nodes have different views of intents" )
1879 intentStates = []
1880 for node in ONOSIntents: # Iter through ONOS nodes
1881 nodeStates = []
1882 # Iter through intents of a node
1883 try:
1884 for intent in json.loads( node ):
1885 nodeStates.append( intent[ 'state' ] )
1886 except ( ValueError, TypeError ):
1887 main.log.exception( "Error in parsing intents" )
1888 main.log.error( repr( node ) )
1889 intentStates.append( nodeStates )
1890 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1891 main.log.info( dict( out ) )
1892
1893 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001894 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001895 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1896 main.log.warn( json.dumps(
1897 json.loads( ONOSIntents[ i ] ),
1898 sort_keys=True,
1899 indent=4,
1900 separators=( ',', ': ' ) ) )
1901 elif intentsResults and consistentIntents:
1902 intentCheck = main.TRUE
1903
1904 # NOTE: Store has no durability, so intents are lost across system
1905 # restarts
1906 main.step( "Compare current intents with intents before the failure" )
1907 # NOTE: this requires case 5 to pass for intentState to be set.
1908 # maybe we should stop the test if that fails?
1909 sameIntents = main.FALSE
1910 if intentState and intentState == ONOSIntents[ 0 ]:
1911 sameIntents = main.TRUE
1912 main.log.info( "Intents are consistent with before failure" )
1913 # TODO: possibly the states have changed? we may need to figure out
1914 # what the acceptable states are
1915 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1916 sameIntents = main.TRUE
1917 try:
1918 before = json.loads( intentState )
1919 after = json.loads( ONOSIntents[ 0 ] )
1920 for intent in before:
1921 if intent not in after:
1922 sameIntents = main.FALSE
1923 main.log.debug( "Intent is not currently in ONOS " +
1924 "(at least in the same form):" )
1925 main.log.debug( json.dumps( intent ) )
1926 except ( ValueError, TypeError ):
1927 main.log.exception( "Exception printing intents" )
1928 main.log.debug( repr( ONOSIntents[0] ) )
1929 main.log.debug( repr( intentState ) )
1930 if sameIntents == main.FALSE:
1931 try:
1932 main.log.debug( "ONOS intents before: " )
1933 main.log.debug( json.dumps( json.loads( intentState ),
1934 sort_keys=True, indent=4,
1935 separators=( ',', ': ' ) ) )
1936 main.log.debug( "Current ONOS intents: " )
1937 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1938 sort_keys=True, indent=4,
1939 separators=( ',', ': ' ) ) )
1940 except ( ValueError, TypeError ):
1941 main.log.exception( "Exception printing intents" )
1942 main.log.debug( repr( ONOSIntents[0] ) )
1943 main.log.debug( repr( intentState ) )
1944 utilities.assert_equals(
1945 expect=main.TRUE,
1946 actual=sameIntents,
1947 onpass="Intents are consistent with before failure",
1948 onfail="The Intents changed during failure" )
1949 intentCheck = intentCheck and sameIntents
1950
1951 main.step( "Get the OF Table entries and compare to before " +
1952 "component failure" )
1953 FlowTables = main.TRUE
1954 flows2 = []
1955 for i in range( 28 ):
1956 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall9043c902015-07-30 14:23:44 -07001957 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001958 flows2.append( tmpFlows )
Jon Hall9043c902015-07-30 14:23:44 -07001959 tempResult = main.Mininet1.flowComp(
Jon Hall5cf14d52015-07-16 12:15:19 -07001960 flow1=flows[ i ],
1961 flow2=tmpFlows )
1962 FlowTables = FlowTables and tempResult
1963 if FlowTables == main.FALSE:
1964 main.log.info( "Differences in flow table for switch: s" +
1965 str( i + 1 ) )
1966 utilities.assert_equals(
1967 expect=main.TRUE,
1968 actual=FlowTables,
1969 onpass="No changes were found in the flow tables",
1970 onfail="Changes were found in the flow tables" )
1971
1972 main.Mininet2.pingLongKill()
1973 '''
1974 main.step( "Check the continuous pings to ensure that no packets " +
1975 "were dropped during component failure" )
1976 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1977 main.params[ 'TESTONIP' ] )
1978 LossInPings = main.FALSE
1979 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1980 for i in range( 8, 18 ):
1981 main.log.info(
1982 "Checking for a loss in pings along flow from s" +
1983 str( i ) )
1984 LossInPings = main.Mininet2.checkForLoss(
1985 "/tmp/ping.h" +
1986 str( i ) ) or LossInPings
1987 if LossInPings == main.TRUE:
1988 main.log.info( "Loss in ping detected" )
1989 elif LossInPings == main.ERROR:
1990 main.log.info( "There are multiple mininet process running" )
1991 elif LossInPings == main.FALSE:
1992 main.log.info( "No Loss in the pings" )
1993 main.log.info( "No loss of dataplane connectivity" )
1994 utilities.assert_equals(
1995 expect=main.FALSE,
1996 actual=LossInPings,
1997 onpass="No Loss of connectivity",
1998 onfail="Loss of dataplane connectivity detected" )
1999 '''
2000
2001 main.step( "Leadership Election is still functional" )
2002 # Test of LeadershipElection
2003 # NOTE: this only works for the sanity test. In case of failures,
2004 # leader will likely change
Jon Halle1a3b752015-07-22 13:02:46 -07002005 leader = main.nodes[ 0 ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002006 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002007 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002008 leaderN = cli.electionTestLeader()
2009 # verify leader is ONOS1
2010 if leaderN == leader:
2011 # all is well
2012 # NOTE: In failure scenario, this could be a new node, maybe
2013 # check != ONOS1
2014 pass
2015 elif leaderN == main.FALSE:
2016 # error in response
2017 main.log.error( "Something is wrong with " +
2018 "electionTestLeader function, check the" +
2019 " error logs" )
2020 leaderResult = main.FALSE
2021 elif leader != leaderN:
2022 leaderResult = main.FALSE
2023 main.log.error( cli.name + " sees " + str( leaderN ) +
2024 " as the leader of the election app. " +
2025 "Leader should be " + str( leader ) )
2026 utilities.assert_equals(
2027 expect=main.TRUE,
2028 actual=leaderResult,
2029 onpass="Leadership election passed",
2030 onfail="Something went wrong with Leadership election" )
2031
2032 def CASE8( self, main ):
2033 """
2034 Compare topo
2035 """
2036 import json
2037 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002038 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002039 assert main, "main not defined"
2040 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002041 assert main.CLIs, "main.CLIs not defined"
2042 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002043
2044 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002045 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002046 " and ONOS"
2047
2048 main.step( "Comparing ONOS topology to MN" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002049 topoResult = main.FALSE
2050 elapsed = 0
2051 count = 0
2052 main.step( "Collecting topology information from ONOS" )
2053 startTime = time.time()
2054 # Give time for Gossip to work
2055 while topoResult == main.FALSE and elapsed < 60:
Jon Hall96091e62015-09-21 17:34:17 -07002056 devicesResults = main.TRUE
2057 linksResults = main.TRUE
2058 hostsResults = main.TRUE
2059 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002060 count += 1
2061 cliStart = time.time()
2062 devices = []
2063 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002064 for i in range( main.numCtrls ):
2065 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002066 name="devices-" + str( i ),
2067 args=[ ] )
2068 threads.append( t )
2069 t.start()
2070
2071 for t in threads:
2072 t.join()
2073 devices.append( t.result )
2074 hosts = []
2075 ipResult = main.TRUE
2076 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002077 for i in range( main.numCtrls ):
2078 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07002079 name="hosts-" + str( i ),
2080 args=[ ] )
2081 threads.append( t )
2082 t.start()
2083
2084 for t in threads:
2085 t.join()
2086 try:
2087 hosts.append( json.loads( t.result ) )
2088 except ( ValueError, TypeError ):
2089 main.log.exception( "Error parsing hosts results" )
2090 main.log.error( repr( t.result ) )
2091 for controller in range( 0, len( hosts ) ):
2092 controllerStr = str( controller + 1 )
2093 for host in hosts[ controller ]:
2094 if host is None or host.get( 'ipAddresses', [] ) == []:
2095 main.log.error(
2096 "DEBUG:Error with host ipAddresses on controller" +
2097 controllerStr + ": " + str( host ) )
2098 ipResult = main.FALSE
2099 ports = []
2100 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002101 for i in range( main.numCtrls ):
2102 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002103 name="ports-" + str( i ),
2104 args=[ ] )
2105 threads.append( t )
2106 t.start()
2107
2108 for t in threads:
2109 t.join()
2110 ports.append( t.result )
2111 links = []
2112 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002113 for i in range( main.numCtrls ):
2114 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002115 name="links-" + str( i ),
2116 args=[ ] )
2117 threads.append( t )
2118 t.start()
2119
2120 for t in threads:
2121 t.join()
2122 links.append( t.result )
2123 clusters = []
2124 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002125 for i in range( main.numCtrls ):
2126 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002127 name="clusters-" + str( i ),
2128 args=[ ] )
2129 threads.append( t )
2130 t.start()
2131
2132 for t in threads:
2133 t.join()
2134 clusters.append( t.result )
2135
2136 elapsed = time.time() - startTime
2137 cliTime = time.time() - cliStart
2138 print "Elapsed time: " + str( elapsed )
2139 print "CLI time: " + str( cliTime )
2140
2141 mnSwitches = main.Mininet1.getSwitches()
2142 mnLinks = main.Mininet1.getLinks()
2143 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002144 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002145 controllerStr = str( controller + 1 )
2146 if devices[ controller ] and ports[ controller ] and\
2147 "Error" not in devices[ controller ] and\
2148 "Error" not in ports[ controller ]:
2149
2150 currentDevicesResult = main.Mininet1.compareSwitches(
2151 mnSwitches,
2152 json.loads( devices[ controller ] ),
2153 json.loads( ports[ controller ] ) )
2154 else:
2155 currentDevicesResult = main.FALSE
2156 utilities.assert_equals( expect=main.TRUE,
2157 actual=currentDevicesResult,
2158 onpass="ONOS" + controllerStr +
2159 " Switches view is correct",
2160 onfail="ONOS" + controllerStr +
2161 " Switches view is incorrect" )
2162
2163 if links[ controller ] and "Error" not in links[ controller ]:
2164 currentLinksResult = main.Mininet1.compareLinks(
2165 mnSwitches, mnLinks,
2166 json.loads( links[ controller ] ) )
2167 else:
2168 currentLinksResult = main.FALSE
2169 utilities.assert_equals( expect=main.TRUE,
2170 actual=currentLinksResult,
2171 onpass="ONOS" + controllerStr +
2172 " links view is correct",
2173 onfail="ONOS" + controllerStr +
2174 " links view is incorrect" )
2175
2176 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2177 currentHostsResult = main.Mininet1.compareHosts(
2178 mnHosts,
2179 hosts[ controller ] )
2180 else:
2181 currentHostsResult = main.FALSE
2182 utilities.assert_equals( expect=main.TRUE,
2183 actual=currentHostsResult,
2184 onpass="ONOS" + controllerStr +
2185 " hosts exist in Mininet",
2186 onfail="ONOS" + controllerStr +
2187 " hosts don't match Mininet" )
2188 # CHECKING HOST ATTACHMENT POINTS
2189 hostAttachment = True
2190 zeroHosts = False
2191 # FIXME: topo-HA/obelisk specific mappings:
2192 # key is mac and value is dpid
2193 mappings = {}
2194 for i in range( 1, 29 ): # hosts 1 through 28
2195 # set up correct variables:
2196 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2197 if i == 1:
2198 deviceId = "1000".zfill(16)
2199 elif i == 2:
2200 deviceId = "2000".zfill(16)
2201 elif i == 3:
2202 deviceId = "3000".zfill(16)
2203 elif i == 4:
2204 deviceId = "3004".zfill(16)
2205 elif i == 5:
2206 deviceId = "5000".zfill(16)
2207 elif i == 6:
2208 deviceId = "6000".zfill(16)
2209 elif i == 7:
2210 deviceId = "6007".zfill(16)
2211 elif i >= 8 and i <= 17:
2212 dpid = '3' + str( i ).zfill( 3 )
2213 deviceId = dpid.zfill(16)
2214 elif i >= 18 and i <= 27:
2215 dpid = '6' + str( i ).zfill( 3 )
2216 deviceId = dpid.zfill(16)
2217 elif i == 28:
2218 deviceId = "2800".zfill(16)
2219 mappings[ macId ] = deviceId
2220 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2221 if hosts[ controller ] == []:
2222 main.log.warn( "There are no hosts discovered" )
2223 zeroHosts = True
2224 else:
2225 for host in hosts[ controller ]:
2226 mac = None
2227 location = None
2228 device = None
2229 port = None
2230 try:
2231 mac = host.get( 'mac' )
2232 assert mac, "mac field could not be found for this host object"
2233
2234 location = host.get( 'location' )
2235 assert location, "location field could not be found for this host object"
2236
2237 # Trim the protocol identifier off deviceId
2238 device = str( location.get( 'elementId' ) ).split(':')[1]
2239 assert device, "elementId field could not be found for this host location object"
2240
2241 port = location.get( 'port' )
2242 assert port, "port field could not be found for this host location object"
2243
2244 # Now check if this matches where they should be
2245 if mac and device and port:
2246 if str( port ) != "1":
2247 main.log.error( "The attachment port is incorrect for " +
2248 "host " + str( mac ) +
2249 ". Expected: 1 Actual: " + str( port) )
2250 hostAttachment = False
2251 if device != mappings[ str( mac ) ]:
2252 main.log.error( "The attachment device is incorrect for " +
2253 "host " + str( mac ) +
2254 ". Expected: " + mappings[ str( mac ) ] +
2255 " Actual: " + device )
2256 hostAttachment = False
2257 else:
2258 hostAttachment = False
2259 except AssertionError:
2260 main.log.exception( "Json object not as expected" )
2261 main.log.error( repr( host ) )
2262 hostAttachment = False
2263 else:
2264 main.log.error( "No hosts json output or \"Error\"" +
2265 " in output. hosts = " +
2266 repr( hosts[ controller ] ) )
2267 if zeroHosts is False:
2268 hostAttachment = True
2269
2270 # END CHECKING HOST ATTACHMENT POINTS
2271 devicesResults = devicesResults and currentDevicesResult
2272 linksResults = linksResults and currentLinksResult
2273 hostsResults = hostsResults and currentHostsResult
2274 hostAttachmentResults = hostAttachmentResults and\
2275 hostAttachment
2276 topoResult = ( devicesResults and linksResults
2277 and hostsResults and ipResult and
2278 hostAttachmentResults )
2279
2280 # Compare json objects for hosts and dataplane clusters
2281
2282 # hosts
2283 main.step( "Hosts view is consistent across all ONOS nodes" )
2284 consistentHostsResult = main.TRUE
2285 for controller in range( len( hosts ) ):
2286 controllerStr = str( controller + 1 )
2287 if "Error" not in hosts[ controller ]:
2288 if hosts[ controller ] == hosts[ 0 ]:
2289 continue
2290 else: # hosts not consistent
2291 main.log.error( "hosts from ONOS" + controllerStr +
2292 " is inconsistent with ONOS1" )
2293 main.log.warn( repr( hosts[ controller ] ) )
2294 consistentHostsResult = main.FALSE
2295
2296 else:
2297 main.log.error( "Error in getting ONOS hosts from ONOS" +
2298 controllerStr )
2299 consistentHostsResult = main.FALSE
2300 main.log.warn( "ONOS" + controllerStr +
2301 " hosts response: " +
2302 repr( hosts[ controller ] ) )
2303 utilities.assert_equals(
2304 expect=main.TRUE,
2305 actual=consistentHostsResult,
2306 onpass="Hosts view is consistent across all ONOS nodes",
2307 onfail="ONOS nodes have different views of hosts" )
2308
2309 main.step( "Hosts information is correct" )
2310 hostsResults = hostsResults and ipResult
2311 utilities.assert_equals(
2312 expect=main.TRUE,
2313 actual=hostsResults,
2314 onpass="Host information is correct",
2315 onfail="Host information is incorrect" )
2316
2317 main.step( "Host attachment points to the network" )
2318 utilities.assert_equals(
2319 expect=True,
2320 actual=hostAttachmentResults,
2321 onpass="Hosts are correctly attached to the network",
2322 onfail="ONOS did not correctly attach hosts to the network" )
2323
2324 # Strongly connected clusters of devices
2325 main.step( "Clusters view is consistent across all ONOS nodes" )
2326 consistentClustersResult = main.TRUE
2327 for controller in range( len( clusters ) ):
2328 controllerStr = str( controller + 1 )
2329 if "Error" not in clusters[ controller ]:
2330 if clusters[ controller ] == clusters[ 0 ]:
2331 continue
2332 else: # clusters not consistent
2333 main.log.error( "clusters from ONOS" +
2334 controllerStr +
2335 " is inconsistent with ONOS1" )
2336 consistentClustersResult = main.FALSE
2337
2338 else:
2339 main.log.error( "Error in getting dataplane clusters " +
2340 "from ONOS" + controllerStr )
2341 consistentClustersResult = main.FALSE
2342 main.log.warn( "ONOS" + controllerStr +
2343 " clusters response: " +
2344 repr( clusters[ controller ] ) )
2345 utilities.assert_equals(
2346 expect=main.TRUE,
2347 actual=consistentClustersResult,
2348 onpass="Clusters view is consistent across all ONOS nodes",
2349 onfail="ONOS nodes have different views of clusters" )
2350
2351 main.step( "There is only one SCC" )
2352 # there should always only be one cluster
2353 try:
2354 numClusters = len( json.loads( clusters[ 0 ] ) )
2355 except ( ValueError, TypeError ):
2356 main.log.exception( "Error parsing clusters[0]: " +
2357 repr( clusters[0] ) )
2358 clusterResults = main.FALSE
2359 if numClusters == 1:
2360 clusterResults = main.TRUE
2361 utilities.assert_equals(
2362 expect=1,
2363 actual=numClusters,
2364 onpass="ONOS shows 1 SCC",
2365 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2366
2367 topoResult = ( devicesResults and linksResults
2368 and hostsResults and consistentHostsResult
2369 and consistentClustersResult and clusterResults
2370 and ipResult and hostAttachmentResults )
2371
2372 topoResult = topoResult and int( count <= 2 )
2373 note = "note it takes about " + str( int( cliTime ) ) + \
2374 " seconds for the test to make all the cli calls to fetch " +\
2375 "the topology from each ONOS instance"
2376 main.log.info(
2377 "Very crass estimate for topology discovery/convergence( " +
2378 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2379 str( count ) + " tries" )
2380
2381 main.step( "Device information is correct" )
2382 utilities.assert_equals(
2383 expect=main.TRUE,
2384 actual=devicesResults,
2385 onpass="Device information is correct",
2386 onfail="Device information is incorrect" )
2387
2388 main.step( "Links are correct" )
2389 utilities.assert_equals(
2390 expect=main.TRUE,
2391 actual=linksResults,
2392 onpass="Link are correct",
2393 onfail="Links are incorrect" )
2394
2395 main.step( "Hosts are correct" )
2396 utilities.assert_equals(
2397 expect=main.TRUE,
2398 actual=hostsResults,
2399 onpass="Hosts are correct",
2400 onfail="Hosts are incorrect" )
2401
2402 # FIXME: move this to an ONOS state case
2403 main.step( "Checking ONOS nodes" )
2404 nodesOutput = []
2405 nodeResults = main.TRUE
2406 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002407 for i in range( main.numCtrls ):
2408 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002409 name="nodes-" + str( i ),
2410 args=[ ] )
2411 threads.append( t )
2412 t.start()
2413
2414 for t in threads:
2415 t.join()
2416 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002417 ips = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002418 for i in nodesOutput:
2419 try:
2420 current = json.loads( i )
2421 for node in current:
2422 currentResult = main.FALSE
2423 if node['ip'] in ips: # node in nodes() output is in cell
2424 if node['state'] == 'ACTIVE':
2425 currentResult = main.TRUE
2426 else:
2427 main.log.error( "Error in ONOS node availability" )
2428 main.log.error(
2429 json.dumps( current,
2430 sort_keys=True,
2431 indent=4,
2432 separators=( ',', ': ' ) ) )
2433 break
2434 nodeResults = nodeResults and currentResult
2435 except ( ValueError, TypeError ):
2436 main.log.error( "Error parsing nodes output" )
2437 main.log.warn( repr( i ) )
2438 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2439 onpass="Nodes check successful",
2440 onfail="Nodes check NOT successful" )
2441
2442 def CASE9( self, main ):
2443 """
2444 Link s3-s28 down
2445 """
2446 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002447 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002448 assert main, "main not defined"
2449 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002450 assert main.CLIs, "main.CLIs not defined"
2451 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002452 # NOTE: You should probably run a topology check after this
2453
2454 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2455
2456 description = "Turn off a link to ensure that Link Discovery " +\
2457 "is working properly"
2458 main.case( description )
2459
2460 main.step( "Kill Link between s3 and s28" )
2461 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2462 main.log.info( "Waiting " + str( linkSleep ) +
2463 " seconds for link down to be discovered" )
2464 time.sleep( linkSleep )
2465 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2466 onpass="Link down successful",
2467 onfail="Failed to bring link down" )
2468 # TODO do some sort of check here
2469
2470 def CASE10( self, main ):
2471 """
2472 Link s3-s28 up
2473 """
2474 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002475 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002476 assert main, "main not defined"
2477 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002478 assert main.CLIs, "main.CLIs not defined"
2479 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002480 # NOTE: You should probably run a topology check after this
2481
2482 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2483
2484 description = "Restore a link to ensure that Link Discovery is " + \
2485 "working properly"
2486 main.case( description )
2487
2488 main.step( "Bring link between s3 and s28 back up" )
2489 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2490 main.log.info( "Waiting " + str( linkSleep ) +
2491 " seconds for link up to be discovered" )
2492 time.sleep( linkSleep )
2493 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2494 onpass="Link up successful",
2495 onfail="Failed to bring link up" )
2496 # TODO do some sort of check here
2497
2498 def CASE11( self, main ):
2499 """
2500 Switch Down
2501 """
2502 # NOTE: You should probably run a topology check after this
2503 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002504 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002505 assert main, "main not defined"
2506 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002507 assert main.CLIs, "main.CLIs not defined"
2508 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002509
2510 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2511
2512 description = "Killing a switch to ensure it is discovered correctly"
2513 main.case( description )
2514 switch = main.params[ 'kill' ][ 'switch' ]
2515 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2516
2517 # TODO: Make this switch parameterizable
2518 main.step( "Kill " + switch )
2519 main.log.info( "Deleting " + switch )
2520 main.Mininet1.delSwitch( switch )
2521 main.log.info( "Waiting " + str( switchSleep ) +
2522 " seconds for switch down to be discovered" )
2523 time.sleep( switchSleep )
2524 device = main.ONOScli1.getDevice( dpid=switchDPID )
2525 # Peek at the deleted switch
2526 main.log.warn( str( device ) )
2527 result = main.FALSE
2528 if device and device[ 'available' ] is False:
2529 result = main.TRUE
2530 utilities.assert_equals( expect=main.TRUE, actual=result,
2531 onpass="Kill switch successful",
2532 onfail="Failed to kill switch?" )
2533
2534 def CASE12( self, main ):
2535 """
2536 Switch Up
2537 """
2538 # NOTE: You should probably run a topology check after this
2539 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002540 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002541 assert main, "main not defined"
2542 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002543 assert main.CLIs, "main.CLIs not defined"
2544 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002545 assert ONOS1Port, "ONOS1Port not defined"
2546 assert ONOS2Port, "ONOS2Port not defined"
2547 assert ONOS3Port, "ONOS3Port not defined"
2548 assert ONOS4Port, "ONOS4Port not defined"
2549 assert ONOS5Port, "ONOS5Port not defined"
2550 assert ONOS6Port, "ONOS6Port not defined"
2551 assert ONOS7Port, "ONOS7Port not defined"
2552
2553 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2554 switch = main.params[ 'kill' ][ 'switch' ]
2555 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2556 links = main.params[ 'kill' ][ 'links' ].split()
2557 description = "Adding a switch to ensure it is discovered correctly"
2558 main.case( description )
2559
2560 main.step( "Add back " + switch )
2561 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2562 for peer in links:
2563 main.Mininet1.addLink( switch, peer )
2564 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002565 for i in range( main.numCtrls ):
2566 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002567 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2568 main.log.info( "Waiting " + str( switchSleep ) +
2569 " seconds for switch up to be discovered" )
2570 time.sleep( switchSleep )
2571 device = main.ONOScli1.getDevice( dpid=switchDPID )
2572 # Peek at the deleted switch
2573 main.log.warn( str( device ) )
2574 result = main.FALSE
2575 if device and device[ 'available' ]:
2576 result = main.TRUE
2577 utilities.assert_equals( expect=main.TRUE, actual=result,
2578 onpass="add switch successful",
2579 onfail="Failed to add switch?" )
2580
2581 def CASE13( self, main ):
2582 """
2583 Clean up
2584 """
2585 import os
2586 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002587 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002588 assert main, "main not defined"
2589 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002590 assert main.CLIs, "main.CLIs not defined"
2591 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002592
2593 # printing colors to terminal
2594 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2595 'blue': '\033[94m', 'green': '\033[92m',
2596 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2597 main.case( "Test Cleanup" )
2598 main.step( "Killing tcpdumps" )
2599 main.Mininet2.stopTcpdump()
2600
2601 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002602 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002603 main.step( "Copying MN pcap and ONOS log files to test station" )
2604 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2605 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002606 # NOTE: MN Pcap file is being saved to logdir.
2607 # We scp this file as MN and TestON aren't necessarily the same vm
2608
2609 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002610 # TODO: Load these from params
2611 # NOTE: must end in /
2612 logFolder = "/opt/onos/log/"
2613 logFiles = [ "karaf.log", "karaf.log.1" ]
2614 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002615 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002616 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002617 dstName = main.logdir + "/" + node.name + "-" + f
2618 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2619 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002620 # std*.log's
2621 # NOTE: must end in /
2622 logFolder = "/opt/onos/var/"
2623 logFiles = [ "stderr.log", "stdout.log" ]
2624 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002625 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002626 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002627 dstName = main.logdir + "/" + node.name + "-" + f
2628 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2629 logFolder + f, dstName )
2630 else:
2631 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002632
2633 main.step( "Stopping Mininet" )
2634 mnResult = main.Mininet1.stopNet()
2635 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2636 onpass="Mininet stopped",
2637 onfail="MN cleanup NOT successful" )
2638
2639 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002640 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002641 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2642 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002643
2644 try:
2645 timerLog = open( main.logdir + "/Timers.csv", 'w')
2646 # Overwrite with empty line and close
2647 labels = "Gossip Intents"
2648 data = str( gossipTime )
2649 timerLog.write( labels + "\n" + data )
2650 timerLog.close()
2651 except NameError, e:
2652 main.log.exception(e)
2653
2654 def CASE14( self, main ):
2655 """
2656 start election app on all onos nodes
2657 """
Jon Halle1a3b752015-07-22 13:02:46 -07002658 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002659 assert main, "main not defined"
2660 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002661 assert main.CLIs, "main.CLIs not defined"
2662 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002663
2664 main.case("Start Leadership Election app")
2665 main.step( "Install leadership election app" )
2666 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2667 utilities.assert_equals(
2668 expect=main.TRUE,
2669 actual=appResult,
2670 onpass="Election app installed",
2671 onfail="Something went wrong with installing Leadership election" )
2672
2673 main.step( "Run for election on each node" )
2674 leaderResult = main.TRUE
2675 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002676 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002677 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002678 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002679 leader = cli.electionTestLeader()
2680 if leader is None or leader == main.FALSE:
2681 main.log.error( cli.name + ": Leader for the election app " +
2682 "should be an ONOS node, instead got '" +
2683 str( leader ) + "'" )
2684 leaderResult = main.FALSE
2685 leaders.append( leader )
2686 utilities.assert_equals(
2687 expect=main.TRUE,
2688 actual=leaderResult,
2689 onpass="Successfully ran for leadership",
2690 onfail="Failed to run for leadership" )
2691
2692 main.step( "Check that each node shows the same leader" )
2693 sameLeader = main.TRUE
2694 if len( set( leaders ) ) != 1:
2695 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002696 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002697 str( leaders ) )
2698 utilities.assert_equals(
2699 expect=main.TRUE,
2700 actual=sameLeader,
2701 onpass="Leadership is consistent for the election topic",
2702 onfail="Nodes have different leaders" )
2703
2704 def CASE15( self, main ):
2705 """
2706 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002707 15.1 Run election on each node
2708 15.2 Check that each node has the same leaders and candidates
2709 15.3 Find current leader and withdraw
2710 15.4 Check that a new node was elected leader
2711 15.5 Check that that new leader was the candidate of old leader
2712 15.6 Run for election on old leader
2713 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2714 15.8 Make sure that the old leader was added to the candidate list
2715
2716 old and new variable prefixes refer to data from before vs after
2717 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002718 """
2719 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002720 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002721 assert main, "main not defined"
2722 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002723 assert main.CLIs, "main.CLIs not defined"
2724 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002725
acsmars3a72bde2015-09-02 14:16:22 -07002726 description = "Check that Leadership Election App is still functional"
Jon Hall5cf14d52015-07-16 12:15:19 -07002727 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07002728 # NOTE: Need to re-run since being a canidate is not persistant
2729 # TODO: add check for "Command not found:" in the driver, this
2730 # means the election test app isn't loaded
Jon Hall5cf14d52015-07-16 12:15:19 -07002731
acsmars71adceb2015-08-31 15:09:26 -07002732 oldLeaders = [] # leaders by node before withdrawl from candidates
2733 newLeaders = [] # leaders by node after withdrawl from candidates
2734 oldAllCandidates = [] # list of lists of each nodes' candidates before
2735 newAllCandidates = [] # list of lists of each nodes' candidates after
2736 oldCandidates = [] # list of candidates from node 0 before withdrawl
2737 newCandidates = [] # list of candidates from node 0 after withdrawl
2738 oldLeader = '' # the old leader from oldLeaders, None if not same
2739 newLeader = '' # the new leaders fron newLoeaders, None if not same
2740 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2741 expectNoLeader = False # True when there is only one leader
2742 if main.numCtrls == 1:
2743 expectNoLeader = True
2744
2745 main.step( "Run for election on each node" )
2746 electionResult = main.TRUE
2747
2748 for cli in main.CLIs: # run test election on each node
2749 if cli.electionTestRun() == main.FALSE:
2750 electionResult = main.FALSE
2751
Jon Hall5cf14d52015-07-16 12:15:19 -07002752 utilities.assert_equals(
2753 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002754 actual=electionResult,
2755 onpass="All nodes successfully ran for leadership",
2756 onfail="At least one node failed to run for leadership" )
2757
acsmars3a72bde2015-09-02 14:16:22 -07002758 if electionResult == main.FALSE:
2759 main.log.error(
2760 "Skipping Test Case because Election Test isn't loaded" )
2761 main.skipCase()
2762
acsmars71adceb2015-08-31 15:09:26 -07002763 main.step( "Check that each node shows the same leader and candidates" )
2764 sameResult = main.TRUE
2765 failMessage = "Nodes have different leaders"
2766 for cli in main.CLIs:
2767 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2768 oldAllCandidates.append( node )
2769 oldLeaders.append( node[ 0 ] )
2770 oldCandidates = oldAllCandidates[ 0 ]
2771
2772 # Check that each node has the same leader. Defines oldLeader
2773 if len( set( oldLeaders ) ) != 1:
2774 sameResult = main.FALSE
2775 main.log.error( "More than one leader present:" + str( oldLeaders ) )
2776 oldLeader = None
2777 else:
2778 oldLeader = oldLeaders[ 0 ]
2779
2780 # Check that each node's candidate list is the same
2781 for candidates in oldAllCandidates:
2782 if set( candidates ) != set( oldCandidates ):
2783 sameResult = main.FALSE
2784 failMessage += "and candidates"
2785
2786 utilities.assert_equals(
2787 expect=main.TRUE,
2788 actual=sameResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002789 onpass="Leadership is consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002790 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002791
2792 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002793 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002794 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002795 if oldLeader is None:
2796 main.log.error( "Leadership isn't consistent." )
2797 withdrawResult = main.FALSE
2798 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002799 for i in range( len( main.CLIs ) ):
acsmars71adceb2015-08-31 15:09:26 -07002800 if oldLeader == main.nodes[ i ].ip_address:
2801 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002802 break
2803 else: # FOR/ELSE statement
2804 main.log.error( "Leader election, could not find current leader" )
2805 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002806 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002807 utilities.assert_equals(
2808 expect=main.TRUE,
2809 actual=withdrawResult,
2810 onpass="Node was withdrawn from election",
2811 onfail="Node was not withdrawn from election" )
2812
acsmars71adceb2015-08-31 15:09:26 -07002813 main.step( "Check that a new node was elected leader" )
2814
Jon Hall5cf14d52015-07-16 12:15:19 -07002815 # FIXME: use threads
acsmars71adceb2015-08-31 15:09:26 -07002816 newLeaderResult = main.TRUE
2817 failMessage = "Nodes have different leaders"
2818
2819 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002820 for cli in main.CLIs:
acsmars71adceb2015-08-31 15:09:26 -07002821 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2822 # elections might no have finished yet
2823 if node[ 0 ] == 'none' and not expectNoLeader:
2824 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2825 "sure elections are complete." )
2826 time.sleep(5)
2827 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2828 # election still isn't done or there is a problem
2829 if node[ 0 ] == 'none':
2830 main.log.error( "No leader was elected on at least 1 node" )
2831 newLeaderResult = main.FALSE
2832 newAllCandidates.append( node )
2833 newLeaders.append( node[ 0 ] )
2834 newCandidates = newAllCandidates[ 0 ]
2835
2836 # Check that each node has the same leader. Defines newLeader
2837 if len( set( newLeaders ) ) != 1:
2838 newLeaderResult = main.FALSE
2839 main.log.error( "Nodes have different leaders: " +
2840 str( newLeaders ) )
2841 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002842 else:
acsmars71adceb2015-08-31 15:09:26 -07002843 newLeader = newLeaders[ 0 ]
2844
2845 # Check that each node's candidate list is the same
2846 for candidates in newAllCandidates:
2847 if set( candidates ) != set( newCandidates ):
2848 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002849 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002850
2851 # Check that the new leader is not the older leader, which was withdrawn
2852 if newLeader == oldLeader:
2853 newLeaderResult = main.FALSE
2854 main.log.error( "All nodes still see old leader: " + oldLeader +
2855 " as the current leader" )
2856
Jon Hall5cf14d52015-07-16 12:15:19 -07002857 utilities.assert_equals(
2858 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002859 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002860 onpass="Leadership election passed",
2861 onfail="Something went wrong with Leadership election" )
2862
acsmars71adceb2015-08-31 15:09:26 -07002863 main.step( "Check that that new leader was the candidate of old leader")
2864 # candidates[ 2 ] should be come the top candidate after withdrawl
2865 correctCandidateResult = main.TRUE
2866 if expectNoLeader:
2867 if newLeader == 'none':
2868 main.log.info( "No leader expected. None found. Pass" )
2869 correctCandidateResult = main.TRUE
2870 else:
2871 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2872 correctCandidateResult = main.FALSE
2873 elif newLeader != oldCandidates[ 2 ]:
2874 correctCandidateResult = main.FALSE
2875 main.log.error( "Candidate " + newLeader + " was elected. " +
2876 oldCandidates[ 2 ] + " should have had priority." )
2877
2878 utilities.assert_equals(
2879 expect=main.TRUE,
2880 actual=correctCandidateResult,
2881 onpass="Correct Candidate Elected",
2882 onfail="Incorrect Candidate Elected" )
2883
Jon Hall5cf14d52015-07-16 12:15:19 -07002884 main.step( "Run for election on old leader( just so everyone " +
2885 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002886 if oldLeaderCLI is not None:
2887 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002888 else:
acsmars71adceb2015-08-31 15:09:26 -07002889 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002890 runResult = main.FALSE
2891 utilities.assert_equals(
2892 expect=main.TRUE,
2893 actual=runResult,
2894 onpass="App re-ran for election",
2895 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002896 main.step(
2897 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002898 # verify leader didn't just change
acsmars71adceb2015-08-31 15:09:26 -07002899 positionResult = main.TRUE
2900 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
2901
2902 # Reset and reuse the new candidate and leaders lists
2903 newAllCandidates = []
2904 newCandidates = []
2905 newLeaders = []
2906 for cli in main.CLIs:
2907 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2908 if oldLeader not in node: # election might no have finished yet
2909 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2910 "be sure elections are complete" )
2911 time.sleep(5)
2912 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2913 if oldLeader not in node: # election still isn't done, errors
2914 main.log.error(
2915 "Old leader was not elected on at least one node" )
2916 positionResult = main.FALSE
2917 newAllCandidates.append( node )
2918 newLeaders.append( node[ 0 ] )
2919 newCandidates = newAllCandidates[ 0 ]
2920
2921 # Check that each node has the same leader. Defines newLeader
2922 if len( set( newLeaders ) ) != 1:
2923 positionResult = main.FALSE
2924 main.log.error( "Nodes have different leaders: " +
2925 str( newLeaders ) )
2926 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002927 else:
acsmars71adceb2015-08-31 15:09:26 -07002928 newLeader = newLeaders[ 0 ]
2929
2930 # Check that each node's candidate list is the same
2931 for candidates in newAllCandidates:
2932 if set( candidates ) != set( newCandidates ):
2933 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002934 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002935
2936 # Check that the re-elected node is last on the candidate List
2937 if oldLeader != newCandidates[ -1 ]:
2938 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
2939 str( newCandidates ) )
2940 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002941
2942 utilities.assert_equals(
2943 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002944 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002945 onpass="Old leader successfully re-ran for election",
2946 onfail="Something went wrong with Leadership election after " +
2947 "the old leader re-ran for election" )
2948
2949 def CASE16( self, main ):
2950 """
2951 Install Distributed Primitives app
2952 """
2953 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002954 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002955 assert main, "main not defined"
2956 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002957 assert main.CLIs, "main.CLIs not defined"
2958 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002959
2960 # Variables for the distributed primitives tests
2961 global pCounterName
2962 global iCounterName
2963 global pCounterValue
2964 global iCounterValue
2965 global onosSet
2966 global onosSetName
2967 pCounterName = "TestON-Partitions"
2968 iCounterName = "TestON-inMemory"
2969 pCounterValue = 0
2970 iCounterValue = 0
2971 onosSet = set([])
2972 onosSetName = "TestON-set"
2973
2974 description = "Install Primitives app"
2975 main.case( description )
2976 main.step( "Install Primitives app" )
2977 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002978 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002979 utilities.assert_equals( expect=main.TRUE,
2980 actual=appResults,
2981 onpass="Primitives app activated",
2982 onfail="Primitives app not activated" )
2983 time.sleep( 5 ) # To allow all nodes to activate
2984
2985 def CASE17( self, main ):
2986 """
2987 Check for basic functionality with distributed primitives
2988 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002989 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002990 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002991 assert main, "main not defined"
2992 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002993 assert main.CLIs, "main.CLIs not defined"
2994 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002995 assert pCounterName, "pCounterName not defined"
2996 assert iCounterName, "iCounterName not defined"
2997 assert onosSetName, "onosSetName not defined"
2998 # NOTE: assert fails if value is 0/None/Empty/False
2999 try:
3000 pCounterValue
3001 except NameError:
3002 main.log.error( "pCounterValue not defined, setting to 0" )
3003 pCounterValue = 0
3004 try:
3005 iCounterValue
3006 except NameError:
3007 main.log.error( "iCounterValue not defined, setting to 0" )
3008 iCounterValue = 0
3009 try:
3010 onosSet
3011 except NameError:
3012 main.log.error( "onosSet not defined, setting to empty Set" )
3013 onosSet = set([])
3014 # Variables for the distributed primitives tests. These are local only
3015 addValue = "a"
3016 addAllValue = "a b c d e f"
3017 retainValue = "c d e f"
3018
3019 description = "Check for basic functionality with distributed " +\
3020 "primitives"
3021 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003022 main.caseExplanation = "Test the methods of the distributed " +\
3023 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003024 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003025 # Partitioned counters
3026 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003027 pCounters = []
3028 threads = []
3029 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003030 for i in range( main.numCtrls ):
3031 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3032 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003033 args=[ pCounterName ] )
3034 pCounterValue += 1
3035 addedPValues.append( pCounterValue )
3036 threads.append( t )
3037 t.start()
3038
3039 for t in threads:
3040 t.join()
3041 pCounters.append( t.result )
3042 # Check that counter incremented numController times
3043 pCounterResults = True
3044 for i in addedPValues:
3045 tmpResult = i in pCounters
3046 pCounterResults = pCounterResults and tmpResult
3047 if not tmpResult:
3048 main.log.error( str( i ) + " is not in partitioned "
3049 "counter incremented results" )
3050 utilities.assert_equals( expect=True,
3051 actual=pCounterResults,
3052 onpass="Default counter incremented",
3053 onfail="Error incrementing default" +
3054 " counter" )
3055
Jon Halle1a3b752015-07-22 13:02:46 -07003056 main.step( "Get then Increment a default counter on each node" )
3057 pCounters = []
3058 threads = []
3059 addedPValues = []
3060 for i in range( main.numCtrls ):
3061 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3062 name="counterGetAndAdd-" + str( i ),
3063 args=[ pCounterName ] )
3064 addedPValues.append( pCounterValue )
3065 pCounterValue += 1
3066 threads.append( t )
3067 t.start()
3068
3069 for t in threads:
3070 t.join()
3071 pCounters.append( t.result )
3072 # Check that counter incremented numController times
3073 pCounterResults = True
3074 for i in addedPValues:
3075 tmpResult = i in pCounters
3076 pCounterResults = pCounterResults and tmpResult
3077 if not tmpResult:
3078 main.log.error( str( i ) + " is not in partitioned "
3079 "counter incremented results" )
3080 utilities.assert_equals( expect=True,
3081 actual=pCounterResults,
3082 onpass="Default counter incremented",
3083 onfail="Error incrementing default" +
3084 " counter" )
3085
3086 main.step( "Counters we added have the correct values" )
3087 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3088 utilities.assert_equals( expect=main.TRUE,
3089 actual=incrementCheck,
3090 onpass="Added counters are correct",
3091 onfail="Added counters are incorrect" )
3092
3093 main.step( "Add -8 to then get a default counter on each node" )
3094 pCounters = []
3095 threads = []
3096 addedPValues = []
3097 for i in range( main.numCtrls ):
3098 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3099 name="counterIncrement-" + str( i ),
3100 args=[ pCounterName ],
3101 kwargs={ "delta": -8 } )
3102 pCounterValue += -8
3103 addedPValues.append( pCounterValue )
3104 threads.append( t )
3105 t.start()
3106
3107 for t in threads:
3108 t.join()
3109 pCounters.append( t.result )
3110 # Check that counter incremented numController times
3111 pCounterResults = True
3112 for i in addedPValues:
3113 tmpResult = i in pCounters
3114 pCounterResults = pCounterResults and tmpResult
3115 if not tmpResult:
3116 main.log.error( str( i ) + " is not in partitioned "
3117 "counter incremented results" )
3118 utilities.assert_equals( expect=True,
3119 actual=pCounterResults,
3120 onpass="Default counter incremented",
3121 onfail="Error incrementing default" +
3122 " counter" )
3123
3124 main.step( "Add 5 to then get a default counter on each node" )
3125 pCounters = []
3126 threads = []
3127 addedPValues = []
3128 for i in range( main.numCtrls ):
3129 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3130 name="counterIncrement-" + str( i ),
3131 args=[ pCounterName ],
3132 kwargs={ "delta": 5 } )
3133 pCounterValue += 5
3134 addedPValues.append( pCounterValue )
3135 threads.append( t )
3136 t.start()
3137
3138 for t in threads:
3139 t.join()
3140 pCounters.append( t.result )
3141 # Check that counter incremented numController times
3142 pCounterResults = True
3143 for i in addedPValues:
3144 tmpResult = i in pCounters
3145 pCounterResults = pCounterResults and tmpResult
3146 if not tmpResult:
3147 main.log.error( str( i ) + " is not in partitioned "
3148 "counter incremented results" )
3149 utilities.assert_equals( expect=True,
3150 actual=pCounterResults,
3151 onpass="Default counter incremented",
3152 onfail="Error incrementing default" +
3153 " counter" )
3154
3155 main.step( "Get then add 5 to a default counter on each node" )
3156 pCounters = []
3157 threads = []
3158 addedPValues = []
3159 for i in range( main.numCtrls ):
3160 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3161 name="counterIncrement-" + str( i ),
3162 args=[ pCounterName ],
3163 kwargs={ "delta": 5 } )
3164 addedPValues.append( pCounterValue )
3165 pCounterValue += 5
3166 threads.append( t )
3167 t.start()
3168
3169 for t in threads:
3170 t.join()
3171 pCounters.append( t.result )
3172 # Check that counter incremented numController times
3173 pCounterResults = True
3174 for i in addedPValues:
3175 tmpResult = i in pCounters
3176 pCounterResults = pCounterResults and tmpResult
3177 if not tmpResult:
3178 main.log.error( str( i ) + " is not in partitioned "
3179 "counter incremented results" )
3180 utilities.assert_equals( expect=True,
3181 actual=pCounterResults,
3182 onpass="Default counter incremented",
3183 onfail="Error incrementing default" +
3184 " counter" )
3185
3186 main.step( "Counters we added have the correct values" )
3187 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3188 utilities.assert_equals( expect=main.TRUE,
3189 actual=incrementCheck,
3190 onpass="Added counters are correct",
3191 onfail="Added counters are incorrect" )
3192
3193 # In-Memory counters
3194 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003195 iCounters = []
3196 addedIValues = []
3197 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003198 for i in range( main.numCtrls ):
3199 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003200 name="icounterIncrement-" + str( i ),
3201 args=[ iCounterName ],
3202 kwargs={ "inMemory": True } )
3203 iCounterValue += 1
3204 addedIValues.append( iCounterValue )
3205 threads.append( t )
3206 t.start()
3207
3208 for t in threads:
3209 t.join()
3210 iCounters.append( t.result )
3211 # Check that counter incremented numController times
3212 iCounterResults = True
3213 for i in addedIValues:
3214 tmpResult = i in iCounters
3215 iCounterResults = iCounterResults and tmpResult
3216 if not tmpResult:
3217 main.log.error( str( i ) + " is not in the in-memory "
3218 "counter incremented results" )
3219 utilities.assert_equals( expect=True,
3220 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003221 onpass="In-memory counter incremented",
3222 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003223 " counter" )
3224
Jon Halle1a3b752015-07-22 13:02:46 -07003225 main.step( "Get then Increment a in-memory counter on each node" )
3226 iCounters = []
3227 threads = []
3228 addedIValues = []
3229 for i in range( main.numCtrls ):
3230 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3231 name="counterGetAndAdd-" + str( i ),
3232 args=[ iCounterName ],
3233 kwargs={ "inMemory": True } )
3234 addedIValues.append( iCounterValue )
3235 iCounterValue += 1
3236 threads.append( t )
3237 t.start()
3238
3239 for t in threads:
3240 t.join()
3241 iCounters.append( t.result )
3242 # Check that counter incremented numController times
3243 iCounterResults = True
3244 for i in addedIValues:
3245 tmpResult = i in iCounters
3246 iCounterResults = iCounterResults and tmpResult
3247 if not tmpResult:
3248 main.log.error( str( i ) + " is not in in-memory "
3249 "counter incremented results" )
3250 utilities.assert_equals( expect=True,
3251 actual=iCounterResults,
3252 onpass="In-memory counter incremented",
3253 onfail="Error incrementing in-memory" +
3254 " counter" )
3255
3256 main.step( "Counters we added have the correct values" )
3257 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3258 utilities.assert_equals( expect=main.TRUE,
3259 actual=incrementCheck,
3260 onpass="Added counters are correct",
3261 onfail="Added counters are incorrect" )
3262
3263 main.step( "Add -8 to then get a in-memory counter on each node" )
3264 iCounters = []
3265 threads = []
3266 addedIValues = []
3267 for i in range( main.numCtrls ):
3268 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3269 name="counterIncrement-" + str( i ),
3270 args=[ iCounterName ],
3271 kwargs={ "delta": -8, "inMemory": True } )
3272 iCounterValue += -8
3273 addedIValues.append( iCounterValue )
3274 threads.append( t )
3275 t.start()
3276
3277 for t in threads:
3278 t.join()
3279 iCounters.append( t.result )
3280 # Check that counter incremented numController times
3281 iCounterResults = True
3282 for i in addedIValues:
3283 tmpResult = i in iCounters
3284 iCounterResults = iCounterResults and tmpResult
3285 if not tmpResult:
3286 main.log.error( str( i ) + " is not in in-memory "
3287 "counter incremented results" )
3288 utilities.assert_equals( expect=True,
3289 actual=pCounterResults,
3290 onpass="In-memory counter incremented",
3291 onfail="Error incrementing in-memory" +
3292 " counter" )
3293
3294 main.step( "Add 5 to then get a in-memory counter on each node" )
3295 iCounters = []
3296 threads = []
3297 addedIValues = []
3298 for i in range( main.numCtrls ):
3299 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3300 name="counterIncrement-" + str( i ),
3301 args=[ iCounterName ],
3302 kwargs={ "delta": 5, "inMemory": True } )
3303 iCounterValue += 5
3304 addedIValues.append( iCounterValue )
3305 threads.append( t )
3306 t.start()
3307
3308 for t in threads:
3309 t.join()
3310 iCounters.append( t.result )
3311 # Check that counter incremented numController times
3312 iCounterResults = True
3313 for i in addedIValues:
3314 tmpResult = i in iCounters
3315 iCounterResults = iCounterResults and tmpResult
3316 if not tmpResult:
3317 main.log.error( str( i ) + " is not in in-memory "
3318 "counter incremented results" )
3319 utilities.assert_equals( expect=True,
3320 actual=pCounterResults,
3321 onpass="In-memory counter incremented",
3322 onfail="Error incrementing in-memory" +
3323 " counter" )
3324
3325 main.step( "Get then add 5 to a in-memory counter on each node" )
3326 iCounters = []
3327 threads = []
3328 addedIValues = []
3329 for i in range( main.numCtrls ):
3330 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3331 name="counterIncrement-" + str( i ),
3332 args=[ iCounterName ],
3333 kwargs={ "delta": 5, "inMemory": True } )
3334 addedIValues.append( iCounterValue )
3335 iCounterValue += 5
3336 threads.append( t )
3337 t.start()
3338
3339 for t in threads:
3340 t.join()
3341 iCounters.append( t.result )
3342 # Check that counter incremented numController times
3343 iCounterResults = True
3344 for i in addedIValues:
3345 tmpResult = i in iCounters
3346 iCounterResults = iCounterResults and tmpResult
3347 if not tmpResult:
3348 main.log.error( str( i ) + " is not in in-memory "
3349 "counter incremented results" )
3350 utilities.assert_equals( expect=True,
3351 actual=iCounterResults,
3352 onpass="In-memory counter incremented",
3353 onfail="Error incrementing in-memory" +
3354 " counter" )
3355
3356 main.step( "Counters we added have the correct values" )
3357 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3358 utilities.assert_equals( expect=main.TRUE,
3359 actual=incrementCheck,
3360 onpass="Added counters are correct",
3361 onfail="Added counters are incorrect" )
3362
Jon Hall5cf14d52015-07-16 12:15:19 -07003363 main.step( "Check counters are consistant across nodes" )
3364 onosCounters = []
3365 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003366 for i in range( main.numCtrls ):
3367 t = main.Thread( target=main.CLIs[i].counters,
Jon Hall5cf14d52015-07-16 12:15:19 -07003368 name="counters-" + str( i ) )
3369 threads.append( t )
3370 t.start()
3371 for t in threads:
3372 t.join()
3373 onosCounters.append( t.result )
3374 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3375 if all( tmp ):
3376 main.log.info( "Counters are consistent across all nodes" )
3377 consistentCounterResults = main.TRUE
3378 else:
3379 main.log.error( "Counters are not consistent across all nodes" )
3380 consistentCounterResults = main.FALSE
3381 utilities.assert_equals( expect=main.TRUE,
3382 actual=consistentCounterResults,
3383 onpass="ONOS counters are consistent " +
3384 "across nodes",
3385 onfail="ONOS Counters are inconsistent " +
3386 "across nodes" )
3387
3388 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003389 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3390 incrementCheck = incrementCheck and \
3391 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003392 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003393 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003394 onpass="Added counters are correct",
3395 onfail="Added counters are incorrect" )
Jon Halle1a3b752015-07-22 13:02:46 -07003396
Jon Hall5cf14d52015-07-16 12:15:19 -07003397 # DISTRIBUTED SETS
3398 main.step( "Distributed Set get" )
3399 size = len( onosSet )
3400 getResponses = []
3401 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003402 for i in range( main.numCtrls ):
3403 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003404 name="setTestGet-" + str( i ),
3405 args=[ onosSetName ] )
3406 threads.append( t )
3407 t.start()
3408 for t in threads:
3409 t.join()
3410 getResponses.append( t.result )
3411
3412 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003413 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003414 if isinstance( getResponses[ i ], list):
3415 current = set( getResponses[ i ] )
3416 if len( current ) == len( getResponses[ i ] ):
3417 # no repeats
3418 if onosSet != current:
3419 main.log.error( "ONOS" + str( i + 1 ) +
3420 " has incorrect view" +
3421 " of set " + onosSetName + ":\n" +
3422 str( getResponses[ i ] ) )
3423 main.log.debug( "Expected: " + str( onosSet ) )
3424 main.log.debug( "Actual: " + str( current ) )
3425 getResults = main.FALSE
3426 else:
3427 # error, set is not a set
3428 main.log.error( "ONOS" + str( i + 1 ) +
3429 " has repeat elements in" +
3430 " set " + onosSetName + ":\n" +
3431 str( getResponses[ i ] ) )
3432 getResults = main.FALSE
3433 elif getResponses[ i ] == main.ERROR:
3434 getResults = main.FALSE
3435 utilities.assert_equals( expect=main.TRUE,
3436 actual=getResults,
3437 onpass="Set elements are correct",
3438 onfail="Set elements are incorrect" )
3439
3440 main.step( "Distributed Set size" )
3441 sizeResponses = []
3442 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003443 for i in range( main.numCtrls ):
3444 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003445 name="setTestSize-" + str( i ),
3446 args=[ onosSetName ] )
3447 threads.append( t )
3448 t.start()
3449 for t in threads:
3450 t.join()
3451 sizeResponses.append( t.result )
3452
3453 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003454 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003455 if size != sizeResponses[ i ]:
3456 sizeResults = main.FALSE
3457 main.log.error( "ONOS" + str( i + 1 ) +
3458 " expected a size of " + str( size ) +
3459 " for set " + onosSetName +
3460 " but got " + str( sizeResponses[ i ] ) )
3461 utilities.assert_equals( expect=main.TRUE,
3462 actual=sizeResults,
3463 onpass="Set sizes are correct",
3464 onfail="Set sizes are incorrect" )
3465
3466 main.step( "Distributed Set add()" )
3467 onosSet.add( addValue )
3468 addResponses = []
3469 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003470 for i in range( main.numCtrls ):
3471 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003472 name="setTestAdd-" + str( i ),
3473 args=[ onosSetName, addValue ] )
3474 threads.append( t )
3475 t.start()
3476 for t in threads:
3477 t.join()
3478 addResponses.append( t.result )
3479
3480 # main.TRUE = successfully changed the set
3481 # main.FALSE = action resulted in no change in set
3482 # main.ERROR - Some error in executing the function
3483 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003484 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003485 if addResponses[ i ] == main.TRUE:
3486 # All is well
3487 pass
3488 elif addResponses[ i ] == main.FALSE:
3489 # Already in set, probably fine
3490 pass
3491 elif addResponses[ i ] == main.ERROR:
3492 # Error in execution
3493 addResults = main.FALSE
3494 else:
3495 # unexpected result
3496 addResults = main.FALSE
3497 if addResults != main.TRUE:
3498 main.log.error( "Error executing set add" )
3499
3500 # Check if set is still correct
3501 size = len( onosSet )
3502 getResponses = []
3503 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003504 for i in range( main.numCtrls ):
3505 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003506 name="setTestGet-" + str( i ),
3507 args=[ onosSetName ] )
3508 threads.append( t )
3509 t.start()
3510 for t in threads:
3511 t.join()
3512 getResponses.append( t.result )
3513 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003514 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003515 if isinstance( getResponses[ i ], list):
3516 current = set( getResponses[ i ] )
3517 if len( current ) == len( getResponses[ i ] ):
3518 # no repeats
3519 if onosSet != current:
3520 main.log.error( "ONOS" + str( i + 1 ) +
3521 " has incorrect view" +
3522 " of set " + onosSetName + ":\n" +
3523 str( getResponses[ i ] ) )
3524 main.log.debug( "Expected: " + str( onosSet ) )
3525 main.log.debug( "Actual: " + str( current ) )
3526 getResults = main.FALSE
3527 else:
3528 # error, set is not a set
3529 main.log.error( "ONOS" + str( i + 1 ) +
3530 " has repeat elements in" +
3531 " set " + onosSetName + ":\n" +
3532 str( getResponses[ i ] ) )
3533 getResults = main.FALSE
3534 elif getResponses[ i ] == main.ERROR:
3535 getResults = main.FALSE
3536 sizeResponses = []
3537 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003538 for i in range( main.numCtrls ):
3539 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003540 name="setTestSize-" + str( i ),
3541 args=[ onosSetName ] )
3542 threads.append( t )
3543 t.start()
3544 for t in threads:
3545 t.join()
3546 sizeResponses.append( t.result )
3547 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003548 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003549 if size != sizeResponses[ i ]:
3550 sizeResults = main.FALSE
3551 main.log.error( "ONOS" + str( i + 1 ) +
3552 " expected a size of " + str( size ) +
3553 " for set " + onosSetName +
3554 " but got " + str( sizeResponses[ i ] ) )
3555 addResults = addResults and getResults and sizeResults
3556 utilities.assert_equals( expect=main.TRUE,
3557 actual=addResults,
3558 onpass="Set add correct",
3559 onfail="Set add was incorrect" )
3560
3561 main.step( "Distributed Set addAll()" )
3562 onosSet.update( addAllValue.split() )
3563 addResponses = []
3564 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003565 for i in range( main.numCtrls ):
3566 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003567 name="setTestAddAll-" + str( i ),
3568 args=[ onosSetName, addAllValue ] )
3569 threads.append( t )
3570 t.start()
3571 for t in threads:
3572 t.join()
3573 addResponses.append( t.result )
3574
3575 # main.TRUE = successfully changed the set
3576 # main.FALSE = action resulted in no change in set
3577 # main.ERROR - Some error in executing the function
3578 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003579 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003580 if addResponses[ i ] == main.TRUE:
3581 # All is well
3582 pass
3583 elif addResponses[ i ] == main.FALSE:
3584 # Already in set, probably fine
3585 pass
3586 elif addResponses[ i ] == main.ERROR:
3587 # Error in execution
3588 addAllResults = main.FALSE
3589 else:
3590 # unexpected result
3591 addAllResults = main.FALSE
3592 if addAllResults != main.TRUE:
3593 main.log.error( "Error executing set addAll" )
3594
3595 # Check if set is still correct
3596 size = len( onosSet )
3597 getResponses = []
3598 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003599 for i in range( main.numCtrls ):
3600 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003601 name="setTestGet-" + str( i ),
3602 args=[ onosSetName ] )
3603 threads.append( t )
3604 t.start()
3605 for t in threads:
3606 t.join()
3607 getResponses.append( t.result )
3608 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003609 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003610 if isinstance( getResponses[ i ], list):
3611 current = set( getResponses[ i ] )
3612 if len( current ) == len( getResponses[ i ] ):
3613 # no repeats
3614 if onosSet != current:
3615 main.log.error( "ONOS" + str( i + 1 ) +
3616 " has incorrect view" +
3617 " of set " + onosSetName + ":\n" +
3618 str( getResponses[ i ] ) )
3619 main.log.debug( "Expected: " + str( onosSet ) )
3620 main.log.debug( "Actual: " + str( current ) )
3621 getResults = main.FALSE
3622 else:
3623 # error, set is not a set
3624 main.log.error( "ONOS" + str( i + 1 ) +
3625 " has repeat elements in" +
3626 " set " + onosSetName + ":\n" +
3627 str( getResponses[ i ] ) )
3628 getResults = main.FALSE
3629 elif getResponses[ i ] == main.ERROR:
3630 getResults = main.FALSE
3631 sizeResponses = []
3632 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003633 for i in range( main.numCtrls ):
3634 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003635 name="setTestSize-" + str( i ),
3636 args=[ onosSetName ] )
3637 threads.append( t )
3638 t.start()
3639 for t in threads:
3640 t.join()
3641 sizeResponses.append( t.result )
3642 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003643 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003644 if size != sizeResponses[ i ]:
3645 sizeResults = main.FALSE
3646 main.log.error( "ONOS" + str( i + 1 ) +
3647 " expected a size of " + str( size ) +
3648 " for set " + onosSetName +
3649 " but got " + str( sizeResponses[ i ] ) )
3650 addAllResults = addAllResults and getResults and sizeResults
3651 utilities.assert_equals( expect=main.TRUE,
3652 actual=addAllResults,
3653 onpass="Set addAll correct",
3654 onfail="Set addAll was incorrect" )
3655
3656 main.step( "Distributed Set contains()" )
3657 containsResponses = []
3658 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003659 for i in range( main.numCtrls ):
3660 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003661 name="setContains-" + str( i ),
3662 args=[ onosSetName ],
3663 kwargs={ "values": addValue } )
3664 threads.append( t )
3665 t.start()
3666 for t in threads:
3667 t.join()
3668 # NOTE: This is the tuple
3669 containsResponses.append( t.result )
3670
3671 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003672 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003673 if containsResponses[ i ] == main.ERROR:
3674 containsResults = main.FALSE
3675 else:
3676 containsResults = containsResults and\
3677 containsResponses[ i ][ 1 ]
3678 utilities.assert_equals( expect=main.TRUE,
3679 actual=containsResults,
3680 onpass="Set contains is functional",
3681 onfail="Set contains failed" )
3682
3683 main.step( "Distributed Set containsAll()" )
3684 containsAllResponses = []
3685 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003686 for i in range( main.numCtrls ):
3687 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003688 name="setContainsAll-" + str( i ),
3689 args=[ onosSetName ],
3690 kwargs={ "values": addAllValue } )
3691 threads.append( t )
3692 t.start()
3693 for t in threads:
3694 t.join()
3695 # NOTE: This is the tuple
3696 containsAllResponses.append( t.result )
3697
3698 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003699 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003700 if containsResponses[ i ] == main.ERROR:
3701 containsResults = main.FALSE
3702 else:
3703 containsResults = containsResults and\
3704 containsResponses[ i ][ 1 ]
3705 utilities.assert_equals( expect=main.TRUE,
3706 actual=containsAllResults,
3707 onpass="Set containsAll is functional",
3708 onfail="Set containsAll failed" )
3709
3710 main.step( "Distributed Set remove()" )
3711 onosSet.remove( addValue )
3712 removeResponses = []
3713 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003714 for i in range( main.numCtrls ):
3715 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003716 name="setTestRemove-" + str( i ),
3717 args=[ onosSetName, addValue ] )
3718 threads.append( t )
3719 t.start()
3720 for t in threads:
3721 t.join()
3722 removeResponses.append( t.result )
3723
3724 # main.TRUE = successfully changed the set
3725 # main.FALSE = action resulted in no change in set
3726 # main.ERROR - Some error in executing the function
3727 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003728 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003729 if removeResponses[ i ] == main.TRUE:
3730 # All is well
3731 pass
3732 elif removeResponses[ i ] == main.FALSE:
3733 # not in set, probably fine
3734 pass
3735 elif removeResponses[ i ] == main.ERROR:
3736 # Error in execution
3737 removeResults = main.FALSE
3738 else:
3739 # unexpected result
3740 removeResults = main.FALSE
3741 if removeResults != main.TRUE:
3742 main.log.error( "Error executing set remove" )
3743
3744 # Check if set is still correct
3745 size = len( onosSet )
3746 getResponses = []
3747 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003748 for i in range( main.numCtrls ):
3749 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003750 name="setTestGet-" + str( i ),
3751 args=[ onosSetName ] )
3752 threads.append( t )
3753 t.start()
3754 for t in threads:
3755 t.join()
3756 getResponses.append( t.result )
3757 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003758 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003759 if isinstance( getResponses[ i ], list):
3760 current = set( getResponses[ i ] )
3761 if len( current ) == len( getResponses[ i ] ):
3762 # no repeats
3763 if onosSet != current:
3764 main.log.error( "ONOS" + str( i + 1 ) +
3765 " has incorrect view" +
3766 " of set " + onosSetName + ":\n" +
3767 str( getResponses[ i ] ) )
3768 main.log.debug( "Expected: " + str( onosSet ) )
3769 main.log.debug( "Actual: " + str( current ) )
3770 getResults = main.FALSE
3771 else:
3772 # error, set is not a set
3773 main.log.error( "ONOS" + str( i + 1 ) +
3774 " has repeat elements in" +
3775 " set " + onosSetName + ":\n" +
3776 str( getResponses[ i ] ) )
3777 getResults = main.FALSE
3778 elif getResponses[ i ] == main.ERROR:
3779 getResults = main.FALSE
3780 sizeResponses = []
3781 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003782 for i in range( main.numCtrls ):
3783 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003784 name="setTestSize-" + str( i ),
3785 args=[ onosSetName ] )
3786 threads.append( t )
3787 t.start()
3788 for t in threads:
3789 t.join()
3790 sizeResponses.append( t.result )
3791 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003792 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003793 if size != sizeResponses[ i ]:
3794 sizeResults = main.FALSE
3795 main.log.error( "ONOS" + str( i + 1 ) +
3796 " expected a size of " + str( size ) +
3797 " for set " + onosSetName +
3798 " but got " + str( sizeResponses[ i ] ) )
3799 removeResults = removeResults and getResults and sizeResults
3800 utilities.assert_equals( expect=main.TRUE,
3801 actual=removeResults,
3802 onpass="Set remove correct",
3803 onfail="Set remove was incorrect" )
3804
3805 main.step( "Distributed Set removeAll()" )
3806 onosSet.difference_update( addAllValue.split() )
3807 removeAllResponses = []
3808 threads = []
3809 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003810 for i in range( main.numCtrls ):
3811 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003812 name="setTestRemoveAll-" + str( i ),
3813 args=[ onosSetName, addAllValue ] )
3814 threads.append( t )
3815 t.start()
3816 for t in threads:
3817 t.join()
3818 removeAllResponses.append( t.result )
3819 except Exception, e:
3820 main.log.exception(e)
3821
3822 # main.TRUE = successfully changed the set
3823 # main.FALSE = action resulted in no change in set
3824 # main.ERROR - Some error in executing the function
3825 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003826 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003827 if removeAllResponses[ i ] == main.TRUE:
3828 # All is well
3829 pass
3830 elif removeAllResponses[ i ] == main.FALSE:
3831 # not in set, probably fine
3832 pass
3833 elif removeAllResponses[ i ] == main.ERROR:
3834 # Error in execution
3835 removeAllResults = main.FALSE
3836 else:
3837 # unexpected result
3838 removeAllResults = main.FALSE
3839 if removeAllResults != main.TRUE:
3840 main.log.error( "Error executing set removeAll" )
3841
3842 # Check if set is still correct
3843 size = len( onosSet )
3844 getResponses = []
3845 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003846 for i in range( main.numCtrls ):
3847 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003848 name="setTestGet-" + str( i ),
3849 args=[ onosSetName ] )
3850 threads.append( t )
3851 t.start()
3852 for t in threads:
3853 t.join()
3854 getResponses.append( t.result )
3855 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003856 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003857 if isinstance( getResponses[ i ], list):
3858 current = set( getResponses[ i ] )
3859 if len( current ) == len( getResponses[ i ] ):
3860 # no repeats
3861 if onosSet != current:
3862 main.log.error( "ONOS" + str( i + 1 ) +
3863 " has incorrect view" +
3864 " of set " + onosSetName + ":\n" +
3865 str( getResponses[ i ] ) )
3866 main.log.debug( "Expected: " + str( onosSet ) )
3867 main.log.debug( "Actual: " + str( current ) )
3868 getResults = main.FALSE
3869 else:
3870 # error, set is not a set
3871 main.log.error( "ONOS" + str( i + 1 ) +
3872 " has repeat elements in" +
3873 " set " + onosSetName + ":\n" +
3874 str( getResponses[ i ] ) )
3875 getResults = main.FALSE
3876 elif getResponses[ i ] == main.ERROR:
3877 getResults = main.FALSE
3878 sizeResponses = []
3879 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003880 for i in range( main.numCtrls ):
3881 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003882 name="setTestSize-" + str( i ),
3883 args=[ onosSetName ] )
3884 threads.append( t )
3885 t.start()
3886 for t in threads:
3887 t.join()
3888 sizeResponses.append( t.result )
3889 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003890 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003891 if size != sizeResponses[ i ]:
3892 sizeResults = main.FALSE
3893 main.log.error( "ONOS" + str( i + 1 ) +
3894 " expected a size of " + str( size ) +
3895 " for set " + onosSetName +
3896 " but got " + str( sizeResponses[ i ] ) )
3897 removeAllResults = removeAllResults and getResults and sizeResults
3898 utilities.assert_equals( expect=main.TRUE,
3899 actual=removeAllResults,
3900 onpass="Set removeAll correct",
3901 onfail="Set removeAll was incorrect" )
3902
3903 main.step( "Distributed Set addAll()" )
3904 onosSet.update( addAllValue.split() )
3905 addResponses = []
3906 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003907 for i in range( main.numCtrls ):
3908 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003909 name="setTestAddAll-" + str( i ),
3910 args=[ onosSetName, addAllValue ] )
3911 threads.append( t )
3912 t.start()
3913 for t in threads:
3914 t.join()
3915 addResponses.append( t.result )
3916
3917 # main.TRUE = successfully changed the set
3918 # main.FALSE = action resulted in no change in set
3919 # main.ERROR - Some error in executing the function
3920 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003921 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003922 if addResponses[ i ] == main.TRUE:
3923 # All is well
3924 pass
3925 elif addResponses[ i ] == main.FALSE:
3926 # Already in set, probably fine
3927 pass
3928 elif addResponses[ i ] == main.ERROR:
3929 # Error in execution
3930 addAllResults = main.FALSE
3931 else:
3932 # unexpected result
3933 addAllResults = main.FALSE
3934 if addAllResults != main.TRUE:
3935 main.log.error( "Error executing set addAll" )
3936
3937 # Check if set is still correct
3938 size = len( onosSet )
3939 getResponses = []
3940 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003941 for i in range( main.numCtrls ):
3942 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003943 name="setTestGet-" + str( i ),
3944 args=[ onosSetName ] )
3945 threads.append( t )
3946 t.start()
3947 for t in threads:
3948 t.join()
3949 getResponses.append( t.result )
3950 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003951 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003952 if isinstance( getResponses[ i ], list):
3953 current = set( getResponses[ i ] )
3954 if len( current ) == len( getResponses[ i ] ):
3955 # no repeats
3956 if onosSet != current:
3957 main.log.error( "ONOS" + str( i + 1 ) +
3958 " has incorrect view" +
3959 " of set " + onosSetName + ":\n" +
3960 str( getResponses[ i ] ) )
3961 main.log.debug( "Expected: " + str( onosSet ) )
3962 main.log.debug( "Actual: " + str( current ) )
3963 getResults = main.FALSE
3964 else:
3965 # error, set is not a set
3966 main.log.error( "ONOS" + str( i + 1 ) +
3967 " has repeat elements in" +
3968 " set " + onosSetName + ":\n" +
3969 str( getResponses[ i ] ) )
3970 getResults = main.FALSE
3971 elif getResponses[ i ] == main.ERROR:
3972 getResults = main.FALSE
3973 sizeResponses = []
3974 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003975 for i in range( main.numCtrls ):
3976 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003977 name="setTestSize-" + str( i ),
3978 args=[ onosSetName ] )
3979 threads.append( t )
3980 t.start()
3981 for t in threads:
3982 t.join()
3983 sizeResponses.append( t.result )
3984 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003985 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003986 if size != sizeResponses[ i ]:
3987 sizeResults = main.FALSE
3988 main.log.error( "ONOS" + str( i + 1 ) +
3989 " expected a size of " + str( size ) +
3990 " for set " + onosSetName +
3991 " but got " + str( sizeResponses[ i ] ) )
3992 addAllResults = addAllResults and getResults and sizeResults
3993 utilities.assert_equals( expect=main.TRUE,
3994 actual=addAllResults,
3995 onpass="Set addAll correct",
3996 onfail="Set addAll was incorrect" )
3997
3998 main.step( "Distributed Set clear()" )
3999 onosSet.clear()
4000 clearResponses = []
4001 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004002 for i in range( main.numCtrls ):
4003 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004004 name="setTestClear-" + str( i ),
4005 args=[ onosSetName, " "], # Values doesn't matter
4006 kwargs={ "clear": True } )
4007 threads.append( t )
4008 t.start()
4009 for t in threads:
4010 t.join()
4011 clearResponses.append( t.result )
4012
4013 # main.TRUE = successfully changed the set
4014 # main.FALSE = action resulted in no change in set
4015 # main.ERROR - Some error in executing the function
4016 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004017 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004018 if clearResponses[ i ] == main.TRUE:
4019 # All is well
4020 pass
4021 elif clearResponses[ i ] == main.FALSE:
4022 # Nothing set, probably fine
4023 pass
4024 elif clearResponses[ i ] == main.ERROR:
4025 # Error in execution
4026 clearResults = main.FALSE
4027 else:
4028 # unexpected result
4029 clearResults = main.FALSE
4030 if clearResults != main.TRUE:
4031 main.log.error( "Error executing set clear" )
4032
4033 # Check if set is still correct
4034 size = len( onosSet )
4035 getResponses = []
4036 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004037 for i in range( main.numCtrls ):
4038 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004039 name="setTestGet-" + str( i ),
4040 args=[ onosSetName ] )
4041 threads.append( t )
4042 t.start()
4043 for t in threads:
4044 t.join()
4045 getResponses.append( t.result )
4046 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004047 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004048 if isinstance( getResponses[ i ], list):
4049 current = set( getResponses[ i ] )
4050 if len( current ) == len( getResponses[ i ] ):
4051 # no repeats
4052 if onosSet != current:
4053 main.log.error( "ONOS" + str( i + 1 ) +
4054 " has incorrect view" +
4055 " of set " + onosSetName + ":\n" +
4056 str( getResponses[ i ] ) )
4057 main.log.debug( "Expected: " + str( onosSet ) )
4058 main.log.debug( "Actual: " + str( current ) )
4059 getResults = main.FALSE
4060 else:
4061 # error, set is not a set
4062 main.log.error( "ONOS" + str( i + 1 ) +
4063 " has repeat elements in" +
4064 " set " + onosSetName + ":\n" +
4065 str( getResponses[ i ] ) )
4066 getResults = main.FALSE
4067 elif getResponses[ i ] == main.ERROR:
4068 getResults = main.FALSE
4069 sizeResponses = []
4070 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004071 for i in range( main.numCtrls ):
4072 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004073 name="setTestSize-" + str( i ),
4074 args=[ onosSetName ] )
4075 threads.append( t )
4076 t.start()
4077 for t in threads:
4078 t.join()
4079 sizeResponses.append( t.result )
4080 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004081 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004082 if size != sizeResponses[ i ]:
4083 sizeResults = main.FALSE
4084 main.log.error( "ONOS" + str( i + 1 ) +
4085 " expected a size of " + str( size ) +
4086 " for set " + onosSetName +
4087 " but got " + str( sizeResponses[ i ] ) )
4088 clearResults = clearResults and getResults and sizeResults
4089 utilities.assert_equals( expect=main.TRUE,
4090 actual=clearResults,
4091 onpass="Set clear correct",
4092 onfail="Set clear was incorrect" )
4093
4094 main.step( "Distributed Set addAll()" )
4095 onosSet.update( addAllValue.split() )
4096 addResponses = []
4097 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004098 for i in range( main.numCtrls ):
4099 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004100 name="setTestAddAll-" + str( i ),
4101 args=[ onosSetName, addAllValue ] )
4102 threads.append( t )
4103 t.start()
4104 for t in threads:
4105 t.join()
4106 addResponses.append( t.result )
4107
4108 # main.TRUE = successfully changed the set
4109 # main.FALSE = action resulted in no change in set
4110 # main.ERROR - Some error in executing the function
4111 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004112 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004113 if addResponses[ i ] == main.TRUE:
4114 # All is well
4115 pass
4116 elif addResponses[ i ] == main.FALSE:
4117 # Already in set, probably fine
4118 pass
4119 elif addResponses[ i ] == main.ERROR:
4120 # Error in execution
4121 addAllResults = main.FALSE
4122 else:
4123 # unexpected result
4124 addAllResults = main.FALSE
4125 if addAllResults != main.TRUE:
4126 main.log.error( "Error executing set addAll" )
4127
4128 # Check if set is still correct
4129 size = len( onosSet )
4130 getResponses = []
4131 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004132 for i in range( main.numCtrls ):
4133 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004134 name="setTestGet-" + str( i ),
4135 args=[ onosSetName ] )
4136 threads.append( t )
4137 t.start()
4138 for t in threads:
4139 t.join()
4140 getResponses.append( t.result )
4141 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004142 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004143 if isinstance( getResponses[ i ], list):
4144 current = set( getResponses[ i ] )
4145 if len( current ) == len( getResponses[ i ] ):
4146 # no repeats
4147 if onosSet != current:
4148 main.log.error( "ONOS" + str( i + 1 ) +
4149 " has incorrect view" +
4150 " of set " + onosSetName + ":\n" +
4151 str( getResponses[ i ] ) )
4152 main.log.debug( "Expected: " + str( onosSet ) )
4153 main.log.debug( "Actual: " + str( current ) )
4154 getResults = main.FALSE
4155 else:
4156 # error, set is not a set
4157 main.log.error( "ONOS" + str( i + 1 ) +
4158 " has repeat elements in" +
4159 " set " + onosSetName + ":\n" +
4160 str( getResponses[ i ] ) )
4161 getResults = main.FALSE
4162 elif getResponses[ i ] == main.ERROR:
4163 getResults = main.FALSE
4164 sizeResponses = []
4165 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004166 for i in range( main.numCtrls ):
4167 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004168 name="setTestSize-" + str( i ),
4169 args=[ onosSetName ] )
4170 threads.append( t )
4171 t.start()
4172 for t in threads:
4173 t.join()
4174 sizeResponses.append( t.result )
4175 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004176 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004177 if size != sizeResponses[ i ]:
4178 sizeResults = main.FALSE
4179 main.log.error( "ONOS" + str( i + 1 ) +
4180 " expected a size of " + str( size ) +
4181 " for set " + onosSetName +
4182 " but got " + str( sizeResponses[ i ] ) )
4183 addAllResults = addAllResults and getResults and sizeResults
4184 utilities.assert_equals( expect=main.TRUE,
4185 actual=addAllResults,
4186 onpass="Set addAll correct",
4187 onfail="Set addAll was incorrect" )
4188
4189 main.step( "Distributed Set retain()" )
4190 onosSet.intersection_update( retainValue.split() )
4191 retainResponses = []
4192 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004193 for i in range( main.numCtrls ):
4194 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004195 name="setTestRetain-" + str( i ),
4196 args=[ onosSetName, retainValue ],
4197 kwargs={ "retain": True } )
4198 threads.append( t )
4199 t.start()
4200 for t in threads:
4201 t.join()
4202 retainResponses.append( t.result )
4203
4204 # main.TRUE = successfully changed the set
4205 # main.FALSE = action resulted in no change in set
4206 # main.ERROR - Some error in executing the function
4207 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004208 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004209 if retainResponses[ i ] == main.TRUE:
4210 # All is well
4211 pass
4212 elif retainResponses[ i ] == main.FALSE:
4213 # Already in set, probably fine
4214 pass
4215 elif retainResponses[ i ] == main.ERROR:
4216 # Error in execution
4217 retainResults = main.FALSE
4218 else:
4219 # unexpected result
4220 retainResults = main.FALSE
4221 if retainResults != main.TRUE:
4222 main.log.error( "Error executing set retain" )
4223
4224 # Check if set is still correct
4225 size = len( onosSet )
4226 getResponses = []
4227 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004228 for i in range( main.numCtrls ):
4229 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004230 name="setTestGet-" + str( i ),
4231 args=[ onosSetName ] )
4232 threads.append( t )
4233 t.start()
4234 for t in threads:
4235 t.join()
4236 getResponses.append( t.result )
4237 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004238 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004239 if isinstance( getResponses[ i ], list):
4240 current = set( getResponses[ i ] )
4241 if len( current ) == len( getResponses[ i ] ):
4242 # no repeats
4243 if onosSet != current:
4244 main.log.error( "ONOS" + str( i + 1 ) +
4245 " has incorrect view" +
4246 " of set " + onosSetName + ":\n" +
4247 str( getResponses[ i ] ) )
4248 main.log.debug( "Expected: " + str( onosSet ) )
4249 main.log.debug( "Actual: " + str( current ) )
4250 getResults = main.FALSE
4251 else:
4252 # error, set is not a set
4253 main.log.error( "ONOS" + str( i + 1 ) +
4254 " has repeat elements in" +
4255 " set " + onosSetName + ":\n" +
4256 str( getResponses[ i ] ) )
4257 getResults = main.FALSE
4258 elif getResponses[ i ] == main.ERROR:
4259 getResults = main.FALSE
4260 sizeResponses = []
4261 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004262 for i in range( main.numCtrls ):
4263 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004264 name="setTestSize-" + str( i ),
4265 args=[ onosSetName ] )
4266 threads.append( t )
4267 t.start()
4268 for t in threads:
4269 t.join()
4270 sizeResponses.append( t.result )
4271 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004272 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004273 if size != sizeResponses[ i ]:
4274 sizeResults = main.FALSE
4275 main.log.error( "ONOS" + str( i + 1 ) +
4276 " expected a size of " +
4277 str( size ) + " for set " + onosSetName +
4278 " but got " + str( sizeResponses[ i ] ) )
4279 retainResults = retainResults and getResults and sizeResults
4280 utilities.assert_equals( expect=main.TRUE,
4281 actual=retainResults,
4282 onpass="Set retain correct",
4283 onfail="Set retain was incorrect" )
4284
Jon Hall2a5002c2015-08-21 16:49:11 -07004285 # Transactional maps
4286 main.step( "Partitioned Transactional maps put" )
4287 tMapValue = "Testing"
4288 numKeys = 100
4289 putResult = True
4290 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4291 if len( putResponses ) == 100:
4292 for i in putResponses:
4293 if putResponses[ i ][ 'value' ] != tMapValue:
4294 putResult = False
4295 else:
4296 putResult = False
4297 if not putResult:
4298 main.log.debug( "Put response values: " + str( putResponses ) )
4299 utilities.assert_equals( expect=True,
4300 actual=putResult,
4301 onpass="Partitioned Transactional Map put successful",
4302 onfail="Partitioned Transactional Map put values are incorrect" )
4303
4304 main.step( "Partitioned Transactional maps get" )
4305 getCheck = True
4306 for n in range( 1, numKeys + 1 ):
4307 getResponses = []
4308 threads = []
4309 valueCheck = True
4310 for i in range( main.numCtrls ):
4311 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4312 name="TMap-get-" + str( i ),
4313 args=[ "Key" + str ( n ) ] )
4314 threads.append( t )
4315 t.start()
4316 for t in threads:
4317 t.join()
4318 getResponses.append( t.result )
4319 for node in getResponses:
4320 if node != tMapValue:
4321 valueCheck = False
4322 if not valueCheck:
4323 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4324 main.log.warn( getResponses )
4325 getCheck = getCheck and valueCheck
4326 utilities.assert_equals( expect=True,
4327 actual=getCheck,
4328 onpass="Partitioned Transactional Map get values were correct",
4329 onfail="Partitioned Transactional Map values incorrect" )
4330
4331 main.step( "In-memory Transactional maps put" )
4332 tMapValue = "Testing"
4333 numKeys = 100
4334 putResult = True
4335 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4336 if len( putResponses ) == 100:
4337 for i in putResponses:
4338 if putResponses[ i ][ 'value' ] != tMapValue:
4339 putResult = False
4340 else:
4341 putResult = False
4342 if not putResult:
4343 main.log.debug( "Put response values: " + str( putResponses ) )
4344 utilities.assert_equals( expect=True,
4345 actual=putResult,
4346 onpass="In-Memory Transactional Map put successful",
4347 onfail="In-Memory Transactional Map put values are incorrect" )
4348
4349 main.step( "In-Memory Transactional maps get" )
4350 getCheck = True
4351 for n in range( 1, numKeys + 1 ):
4352 getResponses = []
4353 threads = []
4354 valueCheck = True
4355 for i in range( main.numCtrls ):
4356 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4357 name="TMap-get-" + str( i ),
4358 args=[ "Key" + str ( n ) ],
4359 kwargs={ "inMemory": True } )
4360 threads.append( t )
4361 t.start()
4362 for t in threads:
4363 t.join()
4364 getResponses.append( t.result )
4365 for node in getResponses:
4366 if node != tMapValue:
4367 valueCheck = False
4368 if not valueCheck:
4369 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4370 main.log.warn( getResponses )
4371 getCheck = getCheck and valueCheck
4372 utilities.assert_equals( expect=True,
4373 actual=getCheck,
4374 onpass="In-Memory Transactional Map get values were correct",
4375 onfail="In-Memory Transactional Map values incorrect" )