blob: c64e05e66499dbe4240ab089f29e018f148ae19c [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
28class HAsanity:
29
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall5cf14d52015-07-16 12:15:19 -070052 main.log.info( "ONOS HA Sanity test - initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070071 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79
80 # FIXME: just get controller port from params?
81 # TODO: do we really need all these?
82 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
83 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
84 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
85 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
86 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
87 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
88 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
89
Jon Halle1a3b752015-07-22 13:02:46 -070090 try:
91 fileName = "Counters"
92 path = main.params[ 'imports' ][ 'path' ]
93 main.Counters = imp.load_source( fileName,
94 path + fileName + ".py" )
95 except Exception as e:
96 main.log.exception( e )
97 main.cleanup()
98 main.exit()
99
100 main.CLIs = []
101 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700102 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700103 for i in range( 1, main.numCtrls + 1 ):
104 try:
105 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
106 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
107 ipList.append( main.nodes[ -1 ].ip_address )
108 except AttributeError:
109 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700110
111 main.step( "Create cell file" )
112 cellAppString = main.params[ 'ENV' ][ 'appString' ]
113 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
114 main.Mininet1.ip_address,
115 cellAppString, ipList )
116 main.step( "Applying cell variable to environment" )
117 cellResult = main.ONOSbench.setCell( cellName )
118 verifyResult = main.ONOSbench.verifyCell()
119
120 # FIXME:this is short term fix
121 main.log.info( "Removing raft logs" )
122 main.ONOSbench.onosRemoveRaftLogs()
123
124 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700125 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700126 main.ONOSbench.onosUninstall( node.ip_address )
127
128 # Make sure ONOS is DEAD
129 main.log.info( "Killing any ONOS processes" )
130 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700131 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700132 killed = main.ONOSbench.onosKill( node.ip_address )
133 killResults = killResults and killed
134
135 cleanInstallResult = main.TRUE
136 gitPullResult = main.TRUE
137
138 main.step( "Starting Mininet" )
139 # scp topo file to mininet
140 # TODO: move to params?
141 topoName = "obelisk.py"
142 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700143 main.ONOSbench.scp( main.Mininet1,
144 filePath + topoName,
145 main.Mininet1.home,
146 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700147 mnResult = main.Mininet1.startNet( )
148 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
149 onpass="Mininet Started",
150 onfail="Error starting Mininet" )
151
152 main.step( "Git checkout and pull " + gitBranch )
153 if PULLCODE:
154 main.ONOSbench.gitCheckout( gitBranch )
155 gitPullResult = main.ONOSbench.gitPull()
156 # values of 1 or 3 are good
157 utilities.assert_lesser( expect=0, actual=gitPullResult,
158 onpass="Git pull successful",
159 onfail="Git pull failed" )
160 main.ONOSbench.getVersion( report=True )
161
162 main.step( "Using mvn clean install" )
163 cleanInstallResult = main.TRUE
164 if PULLCODE and gitPullResult == main.TRUE:
165 cleanInstallResult = main.ONOSbench.cleanInstall()
166 else:
167 main.log.warn( "Did not pull new code so skipping mvn " +
168 "clean install" )
169 utilities.assert_equals( expect=main.TRUE,
170 actual=cleanInstallResult,
171 onpass="MCI successful",
172 onfail="MCI failed" )
173 # GRAPHS
174 # NOTE: important params here:
175 # job = name of Jenkins job
176 # Plot Name = Plot-HA, only can be used if multiple plots
177 # index = The number of the graph under plot name
178 job = "HAsanity"
179 plotName = "Plot-HA"
180 graphs = '<ac:structured-macro ac:name="html">\n'
181 graphs += '<ac:plain-text-body><![CDATA[\n'
182 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
183 '/plot/' + plotName + '/getPlot?index=0' +\
184 '&width=500&height=300"' +\
185 'noborder="0" width="500" height="300" scrolling="yes" ' +\
186 'seamless="seamless"></iframe>\n'
187 graphs += ']]></ac:plain-text-body>\n'
188 graphs += '</ac:structured-macro>\n'
189 main.log.wiki(graphs)
190
191 main.step( "Creating ONOS package" )
192 packageResult = main.ONOSbench.onosPackage()
193 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
194 onpass="ONOS package successful",
195 onfail="ONOS package failed" )
196
197 main.step( "Installing ONOS package" )
198 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700199 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700200 tmpResult = main.ONOSbench.onosInstall( options="-f",
201 node=node.ip_address )
202 onosInstallResult = onosInstallResult and tmpResult
203 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
204 onpass="ONOS install successful",
205 onfail="ONOS install failed" )
206
207 main.step( "Checking if ONOS is up yet" )
208 for i in range( 2 ):
209 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700210 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700211 started = main.ONOSbench.isup( node.ip_address )
212 if not started:
213 main.log.error( node.name + " didn't start!" )
214 main.ONOSbench.onosStop( node.ip_address )
215 main.ONOSbench.onosStart( node.ip_address )
216 onosIsupResult = onosIsupResult and started
217 if onosIsupResult == main.TRUE:
218 break
219 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
220 onpass="ONOS startup successful",
221 onfail="ONOS startup failed" )
222
223 main.log.step( "Starting ONOS CLI sessions" )
224 cliResults = main.TRUE
225 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700226 for i in range( main.numCtrls ):
227 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700228 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700229 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 threads.append( t )
231 t.start()
232
233 for t in threads:
234 t.join()
235 cliResults = cliResults and t.result
236 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
237 onpass="ONOS cli startup successful",
238 onfail="ONOS cli startup failed" )
239
240 if main.params[ 'tcpdump' ].lower() == "true":
241 main.step( "Start Packet Capture MN" )
242 main.Mininet2.startTcpdump(
243 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
244 + "-MN.pcap",
245 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
246 port=main.params[ 'MNtcpdump' ][ 'port' ] )
247
248 main.step( "App Ids check" )
249 appCheck = main.TRUE
250 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700251 for i in range( main.numCtrls ):
252 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700253 name="appToIDCheck-" + str( i ),
254 args=[] )
255 threads.append( t )
256 t.start()
257
258 for t in threads:
259 t.join()
260 appCheck = appCheck and t.result
261 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700262 main.log.warn( main.CLIs[0].apps() )
263 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700264 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
265 onpass="App Ids seem to be correct",
266 onfail="Something is wrong with app Ids" )
267
268 if cliResults == main.FALSE:
269 main.log.error( "Failed to start ONOS, stopping test" )
270 main.cleanup()
271 main.exit()
272
273 def CASE2( self, main ):
274 """
275 Assign devices to controllers
276 """
277 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700278 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700279 assert main, "main not defined"
280 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700281 assert main.CLIs, "main.CLIs not defined"
282 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700283 assert ONOS1Port, "ONOS1Port not defined"
284 assert ONOS2Port, "ONOS2Port not defined"
285 assert ONOS3Port, "ONOS3Port not defined"
286 assert ONOS4Port, "ONOS4Port not defined"
287 assert ONOS5Port, "ONOS5Port not defined"
288 assert ONOS6Port, "ONOS6Port not defined"
289 assert ONOS7Port, "ONOS7Port not defined"
290
291 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700292 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700293 "and check that an ONOS node becomes the " +\
294 "master of the device."
295 main.step( "Assign switches to controllers" )
296
297 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700298 for i in range( main.numCtrls ):
299 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700300 swList = []
301 for i in range( 1, 29 ):
302 swList.append( "s" + str( i ) )
303 main.Mininet1.assignSwController( sw=swList, ip=ipList )
304
305 mastershipCheck = main.TRUE
306 for i in range( 1, 29 ):
307 response = main.Mininet1.getSwController( "s" + str( i ) )
308 try:
309 main.log.info( str( response ) )
310 except Exception:
311 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700312 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700313 if re.search( "tcp:" + node.ip_address, response ):
314 mastershipCheck = mastershipCheck and main.TRUE
315 else:
316 main.log.error( "Error, node " + node.ip_address + " is " +
317 "not in the list of controllers s" +
318 str( i ) + " is connecting to." )
319 mastershipCheck = main.FALSE
320 utilities.assert_equals(
321 expect=main.TRUE,
322 actual=mastershipCheck,
323 onpass="Switch mastership assigned correctly",
324 onfail="Switches not assigned correctly to controllers" )
325
326 def CASE21( self, main ):
327 """
328 Assign mastership to controllers
329 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700330 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700331 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700332 assert main, "main not defined"
333 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700334 assert main.CLIs, "main.CLIs not defined"
335 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700336 assert ONOS1Port, "ONOS1Port not defined"
337 assert ONOS2Port, "ONOS2Port not defined"
338 assert ONOS3Port, "ONOS3Port not defined"
339 assert ONOS4Port, "ONOS4Port not defined"
340 assert ONOS5Port, "ONOS5Port not defined"
341 assert ONOS6Port, "ONOS6Port not defined"
342 assert ONOS7Port, "ONOS7Port not defined"
343
344 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700345 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700346 "device. Then manually assign" +\
347 " mastership to specific ONOS nodes using" +\
348 " 'device-role'"
349 main.step( "Assign mastership of switches to specific controllers" )
350 # Manually assign mastership to the controller we want
351 roleCall = main.TRUE
352
353 ipList = [ ]
354 deviceList = []
355 try:
356 # Assign mastership to specific controllers. This assignment was
357 # determined for a 7 node cluser, but will work with any sized
358 # cluster
359 for i in range( 1, 29 ): # switches 1 through 28
360 # set up correct variables:
361 if i == 1:
362 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700363 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700364 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
365 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700366 c = 1 % main.numCtrls
367 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
369 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700370 c = 1 % main.numCtrls
371 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
373 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700374 c = 3 % main.numCtrls
375 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700376 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
377 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700378 c = 2 % main.numCtrls
379 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700380 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
381 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700382 c = 2 % main.numCtrls
383 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700384 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
385 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700386 c = 5 % main.numCtrls
387 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700388 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
389 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700390 c = 4 % main.numCtrls
391 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700392 dpid = '3' + str( i ).zfill( 3 )
393 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
394 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700395 c = 6 % main.numCtrls
396 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700397 dpid = '6' + str( i ).zfill( 3 )
398 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
399 elif i == 28:
400 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700401 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700402 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
403 else:
404 main.log.error( "You didn't write an else statement for " +
405 "switch s" + str( i ) )
406 roleCall = main.FALSE
407 # Assign switch
408 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
409 # TODO: make this controller dynamic
410 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
411 ip )
412 ipList.append( ip )
413 deviceList.append( deviceId )
414 except ( AttributeError, AssertionError ):
415 main.log.exception( "Something is wrong with ONOS device view" )
416 main.log.info( main.ONOScli1.devices() )
417 utilities.assert_equals(
418 expect=main.TRUE,
419 actual=roleCall,
420 onpass="Re-assigned switch mastership to designated controller",
421 onfail="Something wrong with deviceRole calls" )
422
423 main.step( "Check mastership was correctly assigned" )
424 roleCheck = main.TRUE
425 # NOTE: This is due to the fact that device mastership change is not
426 # atomic and is actually a multi step process
427 time.sleep( 5 )
428 for i in range( len( ipList ) ):
429 ip = ipList[i]
430 deviceId = deviceList[i]
431 # Check assignment
432 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
433 if ip in master:
434 roleCheck = roleCheck and main.TRUE
435 else:
436 roleCheck = roleCheck and main.FALSE
437 main.log.error( "Error, controller " + ip + " is not" +
438 " master " + "of device " +
439 str( deviceId ) + ". Master is " +
440 repr( master ) + "." )
441 utilities.assert_equals(
442 expect=main.TRUE,
443 actual=roleCheck,
444 onpass="Switches were successfully reassigned to designated " +
445 "controller",
446 onfail="Switches were not successfully reassigned" )
447
448 def CASE3( self, main ):
449 """
450 Assign intents
451 """
452 import time
453 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700454 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700455 assert main, "main not defined"
456 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700457 assert main.CLIs, "main.CLIs not defined"
458 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700460 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700461 "assign predetermined host-to-host intents." +\
462 " After installation, check that the intent" +\
463 " is distributed to all nodes and the state" +\
464 " is INSTALLED"
465
466 # install onos-app-fwd
467 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700468 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700469 utilities.assert_equals( expect=main.TRUE, actual=installResults,
470 onpass="Install fwd successful",
471 onfail="Install fwd failed" )
472
473 main.step( "Check app ids" )
474 appCheck = main.TRUE
475 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700476 for i in range( main.numCtrls ):
477 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700478 name="appToIDCheck-" + str( i ),
479 args=[] )
480 threads.append( t )
481 t.start()
482
483 for t in threads:
484 t.join()
485 appCheck = appCheck and t.result
486 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700487 main.log.warn( main.CLIs[0].apps() )
488 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700489 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
490 onpass="App Ids seem to be correct",
491 onfail="Something is wrong with app Ids" )
492
493 main.step( "Discovering Hosts( Via pingall for now )" )
494 # FIXME: Once we have a host discovery mechanism, use that instead
495 # REACTIVE FWD test
496 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700497 passMsg = "Reactive Pingall test passed"
498 time1 = time.time()
499 pingResult = main.Mininet1.pingall()
500 time2 = time.time()
501 if not pingResult:
502 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700503 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700504 passMsg += " on the second try"
505 utilities.assert_equals(
506 expect=main.TRUE,
507 actual=pingResult,
508 onpass= passMsg,
509 onfail="Reactive Pingall failed, " +
510 "one or more ping pairs failed" )
511 main.log.info( "Time for pingall: %2f seconds" %
512 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700513 # timeout for fwd flows
514 time.sleep( 11 )
515 # uninstall onos-app-fwd
516 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700517 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700518 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
519 onpass="Uninstall fwd successful",
520 onfail="Uninstall fwd failed" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700521
522 main.step( "Check app ids" )
523 threads = []
524 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700525 for i in range( main.numCtrls ):
526 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700527 name="appToIDCheck-" + str( i ),
528 args=[] )
529 threads.append( t )
530 t.start()
531
532 for t in threads:
533 t.join()
534 appCheck2 = appCheck2 and t.result
535 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700536 main.log.warn( main.CLIs[0].apps() )
537 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700538 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
539 onpass="App Ids seem to be correct",
540 onfail="Something is wrong with app Ids" )
541
542 main.step( "Add host intents via cli" )
543 intentIds = []
544 # TODO: move the host numbers to params
545 # Maybe look at all the paths we ping?
546 intentAddResult = True
547 hostResult = main.TRUE
548 for i in range( 8, 18 ):
549 main.log.info( "Adding host intent between h" + str( i ) +
550 " and h" + str( i + 10 ) )
551 host1 = "00:00:00:00:00:" + \
552 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
553 host2 = "00:00:00:00:00:" + \
554 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
555 # NOTE: getHost can return None
556 host1Dict = main.ONOScli1.getHost( host1 )
557 host2Dict = main.ONOScli1.getHost( host2 )
558 host1Id = None
559 host2Id = None
560 if host1Dict and host2Dict:
561 host1Id = host1Dict.get( 'id', None )
562 host2Id = host2Dict.get( 'id', None )
563 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700564 nodeNum = ( i % main.numCtrls )
565 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700566 if tmpId:
567 main.log.info( "Added intent with id: " + tmpId )
568 intentIds.append( tmpId )
569 else:
570 main.log.error( "addHostIntent returned: " +
571 repr( tmpId ) )
572 else:
573 main.log.error( "Error, getHost() failed for h" + str( i ) +
574 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700575 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700576 main.log.warn( "Hosts output: " )
577 try:
578 main.log.warn( json.dumps( json.loads( hosts ),
579 sort_keys=True,
580 indent=4,
581 separators=( ',', ': ' ) ) )
582 except ( ValueError, TypeError ):
583 main.log.warn( repr( hosts ) )
584 hostResult = main.FALSE
585 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
586 onpass="Found a host id for each host",
587 onfail="Error looking up host ids" )
588
589 intentStart = time.time()
590 onosIds = main.ONOScli1.getAllIntentsId()
591 main.log.info( "Submitted intents: " + str( intentIds ) )
592 main.log.info( "Intents in ONOS: " + str( onosIds ) )
593 for intent in intentIds:
594 if intent in onosIds:
595 pass # intent submitted is in onos
596 else:
597 intentAddResult = False
598 if intentAddResult:
599 intentStop = time.time()
600 else:
601 intentStop = None
602 # Print the intent states
603 intents = main.ONOScli1.intents()
604 intentStates = []
605 installedCheck = True
606 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
607 count = 0
608 try:
609 for intent in json.loads( intents ):
610 state = intent.get( 'state', None )
611 if "INSTALLED" not in state:
612 installedCheck = False
613 intentId = intent.get( 'id', None )
614 intentStates.append( ( intentId, state ) )
615 except ( ValueError, TypeError ):
616 main.log.exception( "Error parsing intents" )
617 # add submitted intents not in the store
618 tmplist = [ i for i, s in intentStates ]
619 missingIntents = False
620 for i in intentIds:
621 if i not in tmplist:
622 intentStates.append( ( i, " - " ) )
623 missingIntents = True
624 intentStates.sort()
625 for i, s in intentStates:
626 count += 1
627 main.log.info( "%-6s%-15s%-15s" %
628 ( str( count ), str( i ), str( s ) ) )
629 leaders = main.ONOScli1.leaders()
630 try:
631 missing = False
632 if leaders:
633 parsedLeaders = json.loads( leaders )
634 main.log.warn( json.dumps( parsedLeaders,
635 sort_keys=True,
636 indent=4,
637 separators=( ',', ': ' ) ) )
638 # check for all intent partitions
639 topics = []
640 for i in range( 14 ):
641 topics.append( "intent-partition-" + str( i ) )
642 main.log.debug( topics )
643 ONOStopics = [ j['topic'] for j in parsedLeaders ]
644 for topic in topics:
645 if topic not in ONOStopics:
646 main.log.error( "Error: " + topic +
647 " not in leaders" )
648 missing = True
649 else:
650 main.log.error( "leaders() returned None" )
651 except ( ValueError, TypeError ):
652 main.log.exception( "Error parsing leaders" )
653 main.log.error( repr( leaders ) )
654 # Check all nodes
655 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700656 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700657 response = node.leaders( jsonFormat=False)
658 main.log.warn( str( node.name ) + " leaders output: \n" +
659 str( response ) )
660
661 partitions = main.ONOScli1.partitions()
662 try:
663 if partitions :
664 parsedPartitions = json.loads( partitions )
665 main.log.warn( json.dumps( parsedPartitions,
666 sort_keys=True,
667 indent=4,
668 separators=( ',', ': ' ) ) )
669 # TODO check for a leader in all paritions
670 # TODO check for consistency among nodes
671 else:
672 main.log.error( "partitions() returned None" )
673 except ( ValueError, TypeError ):
674 main.log.exception( "Error parsing partitions" )
675 main.log.error( repr( partitions ) )
676 pendingMap = main.ONOScli1.pendingMap()
677 try:
678 if pendingMap :
679 parsedPending = json.loads( pendingMap )
680 main.log.warn( json.dumps( parsedPending,
681 sort_keys=True,
682 indent=4,
683 separators=( ',', ': ' ) ) )
684 # TODO check something here?
685 else:
686 main.log.error( "pendingMap() returned None" )
687 except ( ValueError, TypeError ):
688 main.log.exception( "Error parsing pending map" )
689 main.log.error( repr( pendingMap ) )
690
691 intentAddResult = bool( intentAddResult and not missingIntents and
692 installedCheck )
693 if not intentAddResult:
694 main.log.error( "Error in pushing host intents to ONOS" )
695
696 main.step( "Intent Anti-Entropy dispersion" )
697 for i in range(100):
698 correct = True
699 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700700 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700701 onosIds = []
702 ids = cli.getAllIntentsId()
703 onosIds.append( ids )
704 main.log.debug( "Intents in " + cli.name + ": " +
705 str( sorted( onosIds ) ) )
706 if sorted( ids ) != sorted( intentIds ):
707 main.log.warn( "Set of intent IDs doesn't match" )
708 correct = False
709 break
710 else:
711 intents = json.loads( cli.intents() )
712 for intent in intents:
713 if intent[ 'state' ] != "INSTALLED":
714 main.log.warn( "Intent " + intent[ 'id' ] +
715 " is " + intent[ 'state' ] )
716 correct = False
717 break
718 if correct:
719 break
720 else:
721 time.sleep(1)
722 if not intentStop:
723 intentStop = time.time()
724 global gossipTime
725 gossipTime = intentStop - intentStart
726 main.log.info( "It took about " + str( gossipTime ) +
727 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700728 gossipPeriod = int( main.params['timers']['gossip'] )
729 maxGossipTime = gossipPeriod * len( main.nodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700730 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700731 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700732 onpass="ECM anti-entropy for intents worked within " +
733 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700734 onfail="Intent ECM anti-entropy took too long. " +
735 "Expected time:{}, Actual time:{}".format( maxGossipTime,
736 gossipTime ) )
737 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700738 intentAddResult = True
739
740 if not intentAddResult or "key" in pendingMap:
741 import time
742 installedCheck = True
743 main.log.info( "Sleeping 60 seconds to see if intents are found" )
744 time.sleep( 60 )
745 onosIds = main.ONOScli1.getAllIntentsId()
746 main.log.info( "Submitted intents: " + str( intentIds ) )
747 main.log.info( "Intents in ONOS: " + str( onosIds ) )
748 # Print the intent states
749 intents = main.ONOScli1.intents()
750 intentStates = []
751 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
752 count = 0
753 try:
754 for intent in json.loads( intents ):
755 # Iter through intents of a node
756 state = intent.get( 'state', None )
757 if "INSTALLED" not in state:
758 installedCheck = False
759 intentId = intent.get( 'id', None )
760 intentStates.append( ( intentId, state ) )
761 except ( ValueError, TypeError ):
762 main.log.exception( "Error parsing intents" )
763 # add submitted intents not in the store
764 tmplist = [ i for i, s in intentStates ]
765 for i in intentIds:
766 if i not in tmplist:
767 intentStates.append( ( i, " - " ) )
768 intentStates.sort()
769 for i, s in intentStates:
770 count += 1
771 main.log.info( "%-6s%-15s%-15s" %
772 ( str( count ), str( i ), str( s ) ) )
773 leaders = main.ONOScli1.leaders()
774 try:
775 missing = False
776 if leaders:
777 parsedLeaders = json.loads( leaders )
778 main.log.warn( json.dumps( parsedLeaders,
779 sort_keys=True,
780 indent=4,
781 separators=( ',', ': ' ) ) )
782 # check for all intent partitions
783 # check for election
784 topics = []
785 for i in range( 14 ):
786 topics.append( "intent-partition-" + str( i ) )
787 # FIXME: this should only be after we start the app
788 topics.append( "org.onosproject.election" )
789 main.log.debug( topics )
790 ONOStopics = [ j['topic'] for j in parsedLeaders ]
791 for topic in topics:
792 if topic not in ONOStopics:
793 main.log.error( "Error: " + topic +
794 " not in leaders" )
795 missing = True
796 else:
797 main.log.error( "leaders() returned None" )
798 except ( ValueError, TypeError ):
799 main.log.exception( "Error parsing leaders" )
800 main.log.error( repr( leaders ) )
801 # Check all nodes
802 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700803 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700804 response = node.leaders( jsonFormat=False)
805 main.log.warn( str( node.name ) + " leaders output: \n" +
806 str( response ) )
807
808 partitions = main.ONOScli1.partitions()
809 try:
810 if partitions :
811 parsedPartitions = json.loads( partitions )
812 main.log.warn( json.dumps( parsedPartitions,
813 sort_keys=True,
814 indent=4,
815 separators=( ',', ': ' ) ) )
816 # TODO check for a leader in all paritions
817 # TODO check for consistency among nodes
818 else:
819 main.log.error( "partitions() returned None" )
820 except ( ValueError, TypeError ):
821 main.log.exception( "Error parsing partitions" )
822 main.log.error( repr( partitions ) )
823 pendingMap = main.ONOScli1.pendingMap()
824 try:
825 if pendingMap :
826 parsedPending = json.loads( pendingMap )
827 main.log.warn( json.dumps( parsedPending,
828 sort_keys=True,
829 indent=4,
830 separators=( ',', ': ' ) ) )
831 # TODO check something here?
832 else:
833 main.log.error( "pendingMap() returned None" )
834 except ( ValueError, TypeError ):
835 main.log.exception( "Error parsing pending map" )
836 main.log.error( repr( pendingMap ) )
837
838 def CASE4( self, main ):
839 """
840 Ping across added host intents
841 """
842 import json
843 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700844 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700845 assert main, "main not defined"
846 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700847 assert main.CLIs, "main.CLIs not defined"
848 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700849 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700850 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700851 "functionality and check the state of " +\
852 "the intent"
853 main.step( "Ping across added host intents" )
854 PingResult = main.TRUE
855 for i in range( 8, 18 ):
856 ping = main.Mininet1.pingHost( src="h" + str( i ),
857 target="h" + str( i + 10 ) )
858 PingResult = PingResult and ping
859 if ping == main.FALSE:
860 main.log.warn( "Ping failed between h" + str( i ) +
861 " and h" + str( i + 10 ) )
862 elif ping == main.TRUE:
863 main.log.info( "Ping test passed!" )
864 # Don't set PingResult or you'd override failures
865 if PingResult == main.FALSE:
866 main.log.error(
867 "Intents have not been installed correctly, pings failed." )
868 # TODO: pretty print
869 main.log.warn( "ONOS1 intents: " )
870 try:
871 tmpIntents = main.ONOScli1.intents()
872 main.log.warn( json.dumps( json.loads( tmpIntents ),
873 sort_keys=True,
874 indent=4,
875 separators=( ',', ': ' ) ) )
876 except ( ValueError, TypeError ):
877 main.log.warn( repr( tmpIntents ) )
878 utilities.assert_equals(
879 expect=main.TRUE,
880 actual=PingResult,
881 onpass="Intents have been installed correctly and pings work",
882 onfail="Intents have not been installed correctly, pings failed." )
883
884 main.step( "Check Intent state" )
885 installedCheck = False
886 loopCount = 0
887 while not installedCheck and loopCount < 40:
888 installedCheck = True
889 # Print the intent states
890 intents = main.ONOScli1.intents()
891 intentStates = []
892 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
893 count = 0
894 # Iter through intents of a node
895 try:
896 for intent in json.loads( intents ):
897 state = intent.get( 'state', None )
898 if "INSTALLED" not in state:
899 installedCheck = False
900 intentId = intent.get( 'id', None )
901 intentStates.append( ( intentId, state ) )
902 except ( ValueError, TypeError ):
903 main.log.exception( "Error parsing intents." )
904 # Print states
905 intentStates.sort()
906 for i, s in intentStates:
907 count += 1
908 main.log.info( "%-6s%-15s%-15s" %
909 ( str( count ), str( i ), str( s ) ) )
910 if not installedCheck:
911 time.sleep( 1 )
912 loopCount += 1
913 utilities.assert_equals( expect=True, actual=installedCheck,
914 onpass="Intents are all INSTALLED",
915 onfail="Intents are not all in " +
916 "INSTALLED state" )
917
918 main.step( "Check leadership of topics" )
919 leaders = main.ONOScli1.leaders()
920 topicCheck = main.TRUE
921 try:
922 if leaders:
923 parsedLeaders = json.loads( leaders )
924 main.log.warn( json.dumps( parsedLeaders,
925 sort_keys=True,
926 indent=4,
927 separators=( ',', ': ' ) ) )
928 # check for all intent partitions
929 # check for election
930 # TODO: Look at Devices as topics now that it uses this system
931 topics = []
932 for i in range( 14 ):
933 topics.append( "intent-partition-" + str( i ) )
934 # FIXME: this should only be after we start the app
935 # FIXME: topics.append( "org.onosproject.election" )
936 # Print leaders output
937 main.log.debug( topics )
938 ONOStopics = [ j['topic'] for j in parsedLeaders ]
939 for topic in topics:
940 if topic not in ONOStopics:
941 main.log.error( "Error: " + topic +
942 " not in leaders" )
943 topicCheck = main.FALSE
944 else:
945 main.log.error( "leaders() returned None" )
946 topicCheck = main.FALSE
947 except ( ValueError, TypeError ):
948 topicCheck = main.FALSE
949 main.log.exception( "Error parsing leaders" )
950 main.log.error( repr( leaders ) )
951 # TODO: Check for a leader of these topics
952 # Check all nodes
953 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700954 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700955 response = node.leaders( jsonFormat=False)
956 main.log.warn( str( node.name ) + " leaders output: \n" +
957 str( response ) )
958
959 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
960 onpass="intent Partitions is in leaders",
961 onfail="Some topics were lost " )
962 # Print partitions
963 partitions = main.ONOScli1.partitions()
964 try:
965 if partitions :
966 parsedPartitions = json.loads( partitions )
967 main.log.warn( json.dumps( parsedPartitions,
968 sort_keys=True,
969 indent=4,
970 separators=( ',', ': ' ) ) )
971 # TODO check for a leader in all paritions
972 # TODO check for consistency among nodes
973 else:
974 main.log.error( "partitions() returned None" )
975 except ( ValueError, TypeError ):
976 main.log.exception( "Error parsing partitions" )
977 main.log.error( repr( partitions ) )
978 # Print Pending Map
979 pendingMap = main.ONOScli1.pendingMap()
980 try:
981 if pendingMap :
982 parsedPending = json.loads( pendingMap )
983 main.log.warn( json.dumps( parsedPending,
984 sort_keys=True,
985 indent=4,
986 separators=( ',', ': ' ) ) )
987 # TODO check something here?
988 else:
989 main.log.error( "pendingMap() returned None" )
990 except ( ValueError, TypeError ):
991 main.log.exception( "Error parsing pending map" )
992 main.log.error( repr( pendingMap ) )
993
994 if not installedCheck:
995 main.log.info( "Waiting 60 seconds to see if the state of " +
996 "intents change" )
997 time.sleep( 60 )
998 # Print the intent states
999 intents = main.ONOScli1.intents()
1000 intentStates = []
1001 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1002 count = 0
1003 # Iter through intents of a node
1004 try:
1005 for intent in json.loads( intents ):
1006 state = intent.get( 'state', None )
1007 if "INSTALLED" not in state:
1008 installedCheck = False
1009 intentId = intent.get( 'id', None )
1010 intentStates.append( ( intentId, state ) )
1011 except ( ValueError, TypeError ):
1012 main.log.exception( "Error parsing intents." )
1013 intentStates.sort()
1014 for i, s in intentStates:
1015 count += 1
1016 main.log.info( "%-6s%-15s%-15s" %
1017 ( str( count ), str( i ), str( s ) ) )
1018 leaders = main.ONOScli1.leaders()
1019 try:
1020 missing = False
1021 if leaders:
1022 parsedLeaders = json.loads( leaders )
1023 main.log.warn( json.dumps( parsedLeaders,
1024 sort_keys=True,
1025 indent=4,
1026 separators=( ',', ': ' ) ) )
1027 # check for all intent partitions
1028 # check for election
1029 topics = []
1030 for i in range( 14 ):
1031 topics.append( "intent-partition-" + str( i ) )
1032 # FIXME: this should only be after we start the app
1033 topics.append( "org.onosproject.election" )
1034 main.log.debug( topics )
1035 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1036 for topic in topics:
1037 if topic not in ONOStopics:
1038 main.log.error( "Error: " + topic +
1039 " not in leaders" )
1040 missing = True
1041 else:
1042 main.log.error( "leaders() returned None" )
1043 except ( ValueError, TypeError ):
1044 main.log.exception( "Error parsing leaders" )
1045 main.log.error( repr( leaders ) )
1046 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001047 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001048 response = node.leaders( jsonFormat=False)
1049 main.log.warn( str( node.name ) + " leaders output: \n" +
1050 str( response ) )
1051
1052 partitions = main.ONOScli1.partitions()
1053 try:
1054 if partitions :
1055 parsedPartitions = json.loads( partitions )
1056 main.log.warn( json.dumps( parsedPartitions,
1057 sort_keys=True,
1058 indent=4,
1059 separators=( ',', ': ' ) ) )
1060 # TODO check for a leader in all paritions
1061 # TODO check for consistency among nodes
1062 else:
1063 main.log.error( "partitions() returned None" )
1064 except ( ValueError, TypeError ):
1065 main.log.exception( "Error parsing partitions" )
1066 main.log.error( repr( partitions ) )
1067 pendingMap = main.ONOScli1.pendingMap()
1068 try:
1069 if pendingMap :
1070 parsedPending = json.loads( pendingMap )
1071 main.log.warn( json.dumps( parsedPending,
1072 sort_keys=True,
1073 indent=4,
1074 separators=( ',', ': ' ) ) )
1075 # TODO check something here?
1076 else:
1077 main.log.error( "pendingMap() returned None" )
1078 except ( ValueError, TypeError ):
1079 main.log.exception( "Error parsing pending map" )
1080 main.log.error( repr( pendingMap ) )
1081 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001082 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001083 main.step( "Wait a minute then ping again" )
1084 # the wait is above
1085 PingResult = main.TRUE
1086 for i in range( 8, 18 ):
1087 ping = main.Mininet1.pingHost( src="h" + str( i ),
1088 target="h" + str( i + 10 ) )
1089 PingResult = PingResult and ping
1090 if ping == main.FALSE:
1091 main.log.warn( "Ping failed between h" + str( i ) +
1092 " and h" + str( i + 10 ) )
1093 elif ping == main.TRUE:
1094 main.log.info( "Ping test passed!" )
1095 # Don't set PingResult or you'd override failures
1096 if PingResult == main.FALSE:
1097 main.log.error(
1098 "Intents have not been installed correctly, pings failed." )
1099 # TODO: pretty print
1100 main.log.warn( "ONOS1 intents: " )
1101 try:
1102 tmpIntents = main.ONOScli1.intents()
1103 main.log.warn( json.dumps( json.loads( tmpIntents ),
1104 sort_keys=True,
1105 indent=4,
1106 separators=( ',', ': ' ) ) )
1107 except ( ValueError, TypeError ):
1108 main.log.warn( repr( tmpIntents ) )
1109 utilities.assert_equals(
1110 expect=main.TRUE,
1111 actual=PingResult,
1112 onpass="Intents have been installed correctly and pings work",
1113 onfail="Intents have not been installed correctly, pings failed." )
1114
1115 def CASE5( self, main ):
1116 """
1117 Reading state of ONOS
1118 """
1119 import json
1120 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001121 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001122 assert main, "main not defined"
1123 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001124 assert main.CLIs, "main.CLIs not defined"
1125 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001126
1127 main.case( "Setting up and gathering data for current state" )
1128 # The general idea for this test case is to pull the state of
1129 # ( intents,flows, topology,... ) from each ONOS node
1130 # We can then compare them with each other and also with past states
1131
1132 main.step( "Check that each switch has a master" )
1133 global mastershipState
1134 mastershipState = '[]'
1135
1136 # Assert that each device has a master
1137 rolesNotNull = main.TRUE
1138 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001139 for i in range( main.numCtrls ):
1140 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001141 name="rolesNotNull-" + str( i ),
1142 args=[] )
1143 threads.append( t )
1144 t.start()
1145
1146 for t in threads:
1147 t.join()
1148 rolesNotNull = rolesNotNull and t.result
1149 utilities.assert_equals(
1150 expect=main.TRUE,
1151 actual=rolesNotNull,
1152 onpass="Each device has a master",
1153 onfail="Some devices don't have a master assigned" )
1154
1155 main.step( "Get the Mastership of each switch from each controller" )
1156 ONOSMastership = []
1157 mastershipCheck = main.FALSE
1158 consistentMastership = True
1159 rolesResults = True
1160 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001161 for i in range( main.numCtrls ):
1162 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001163 name="roles-" + str( i ),
1164 args=[] )
1165 threads.append( t )
1166 t.start()
1167
1168 for t in threads:
1169 t.join()
1170 ONOSMastership.append( t.result )
1171
Jon Halle1a3b752015-07-22 13:02:46 -07001172 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001173 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1174 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1175 " roles" )
1176 main.log.warn(
1177 "ONOS" + str( i + 1 ) + " mastership response: " +
1178 repr( ONOSMastership[i] ) )
1179 rolesResults = False
1180 utilities.assert_equals(
1181 expect=True,
1182 actual=rolesResults,
1183 onpass="No error in reading roles output",
1184 onfail="Error in reading roles from ONOS" )
1185
1186 main.step( "Check for consistency in roles from each controller" )
1187 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1188 main.log.info(
1189 "Switch roles are consistent across all ONOS nodes" )
1190 else:
1191 consistentMastership = False
1192 utilities.assert_equals(
1193 expect=True,
1194 actual=consistentMastership,
1195 onpass="Switch roles are consistent across all ONOS nodes",
1196 onfail="ONOS nodes have different views of switch roles" )
1197
1198 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001199 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001200 try:
1201 main.log.warn(
1202 "ONOS" + str( i + 1 ) + " roles: ",
1203 json.dumps(
1204 json.loads( ONOSMastership[ i ] ),
1205 sort_keys=True,
1206 indent=4,
1207 separators=( ',', ': ' ) ) )
1208 except ( ValueError, TypeError ):
1209 main.log.warn( repr( ONOSMastership[ i ] ) )
1210 elif rolesResults and consistentMastership:
1211 mastershipCheck = main.TRUE
1212 mastershipState = ONOSMastership[ 0 ]
1213
1214 main.step( "Get the intents from each controller" )
1215 global intentState
1216 intentState = []
1217 ONOSIntents = []
1218 intentCheck = main.FALSE
1219 consistentIntents = True
1220 intentsResults = True
1221 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001222 for i in range( main.numCtrls ):
1223 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001224 name="intents-" + str( i ),
1225 args=[],
1226 kwargs={ 'jsonFormat': True } )
1227 threads.append( t )
1228 t.start()
1229
1230 for t in threads:
1231 t.join()
1232 ONOSIntents.append( t.result )
1233
Jon Halle1a3b752015-07-22 13:02:46 -07001234 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001235 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1236 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1237 " intents" )
1238 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1239 repr( ONOSIntents[ i ] ) )
1240 intentsResults = False
1241 utilities.assert_equals(
1242 expect=True,
1243 actual=intentsResults,
1244 onpass="No error in reading intents output",
1245 onfail="Error in reading intents from ONOS" )
1246
1247 main.step( "Check for consistency in Intents from each controller" )
1248 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1249 main.log.info( "Intents are consistent across all ONOS " +
1250 "nodes" )
1251 else:
1252 consistentIntents = False
1253 main.log.error( "Intents not consistent" )
1254 utilities.assert_equals(
1255 expect=True,
1256 actual=consistentIntents,
1257 onpass="Intents are consistent across all ONOS nodes",
1258 onfail="ONOS nodes have different views of intents" )
1259
1260 if intentsResults:
1261 # Try to make it easy to figure out what is happening
1262 #
1263 # Intent ONOS1 ONOS2 ...
1264 # 0x01 INSTALLED INSTALLING
1265 # ... ... ...
1266 # ... ... ...
1267 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001268 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001269 title += " " * 10 + "ONOS" + str( n + 1 )
1270 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001271 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001272 keys = []
1273 try:
1274 # Get the set of all intent keys
1275 for nodeStr in ONOSIntents:
1276 node = json.loads( nodeStr )
1277 for intent in node:
1278 keys.append( intent.get( 'id' ) )
1279 keys = set( keys )
1280 # For each intent key, print the state on each node
1281 for key in keys:
1282 row = "%-13s" % key
1283 for nodeStr in ONOSIntents:
1284 node = json.loads( nodeStr )
1285 for intent in node:
1286 if intent.get( 'id', "Error" ) == key:
1287 row += "%-15s" % intent.get( 'state' )
1288 main.log.warn( row )
1289 # End of intent state table
1290 except ValueError as e:
1291 main.log.exception( e )
1292 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1293
1294 if intentsResults and not consistentIntents:
1295 # print the json objects
1296 n = len(ONOSIntents)
1297 main.log.debug( "ONOS" + str( n ) + " intents: " )
1298 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1299 sort_keys=True,
1300 indent=4,
1301 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001302 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001303 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1304 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1305 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1306 sort_keys=True,
1307 indent=4,
1308 separators=( ',', ': ' ) ) )
1309 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001310 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001311 str( n ) + " intents" )
1312 elif intentsResults and consistentIntents:
1313 intentCheck = main.TRUE
1314 intentState = ONOSIntents[ 0 ]
1315
1316 main.step( "Get the flows from each controller" )
1317 global flowState
1318 flowState = []
1319 ONOSFlows = []
1320 ONOSFlowsJson = []
1321 flowCheck = main.FALSE
1322 consistentFlows = True
1323 flowsResults = True
1324 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001325 for i in range( main.numCtrls ):
1326 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001327 name="flows-" + str( i ),
1328 args=[],
1329 kwargs={ 'jsonFormat': True } )
1330 threads.append( t )
1331 t.start()
1332
1333 # NOTE: Flows command can take some time to run
1334 time.sleep(30)
1335 for t in threads:
1336 t.join()
1337 result = t.result
1338 ONOSFlows.append( result )
1339
Jon Halle1a3b752015-07-22 13:02:46 -07001340 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001341 num = str( i + 1 )
1342 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1343 main.log.error( "Error in getting ONOS" + num + " flows" )
1344 main.log.warn( "ONOS" + num + " flows response: " +
1345 repr( ONOSFlows[ i ] ) )
1346 flowsResults = False
1347 ONOSFlowsJson.append( None )
1348 else:
1349 try:
1350 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1351 except ( ValueError, TypeError ):
1352 # FIXME: change this to log.error?
1353 main.log.exception( "Error in parsing ONOS" + num +
1354 " response as json." )
1355 main.log.error( repr( ONOSFlows[ i ] ) )
1356 ONOSFlowsJson.append( None )
1357 flowsResults = False
1358 utilities.assert_equals(
1359 expect=True,
1360 actual=flowsResults,
1361 onpass="No error in reading flows output",
1362 onfail="Error in reading flows from ONOS" )
1363
1364 main.step( "Check for consistency in Flows from each controller" )
1365 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1366 if all( tmp ):
1367 main.log.info( "Flow count is consistent across all ONOS nodes" )
1368 else:
1369 consistentFlows = False
1370 utilities.assert_equals(
1371 expect=True,
1372 actual=consistentFlows,
1373 onpass="The flow count is consistent across all ONOS nodes",
1374 onfail="ONOS nodes have different flow counts" )
1375
1376 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001377 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001378 try:
1379 main.log.warn(
1380 "ONOS" + str( i + 1 ) + " flows: " +
1381 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1382 indent=4, separators=( ',', ': ' ) ) )
1383 except ( ValueError, TypeError ):
1384 main.log.warn(
1385 "ONOS" + str( i + 1 ) + " flows: " +
1386 repr( ONOSFlows[ i ] ) )
1387 elif flowsResults and consistentFlows:
1388 flowCheck = main.TRUE
1389 flowState = ONOSFlows[ 0 ]
1390
1391 main.step( "Get the OF Table entries" )
1392 global flows
1393 flows = []
1394 for i in range( 1, 29 ):
Jon Hall9043c902015-07-30 14:23:44 -07001395 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001396 if flowCheck == main.FALSE:
1397 for table in flows:
1398 main.log.warn( table )
1399 # TODO: Compare switch flow tables with ONOS flow tables
1400
1401 main.step( "Start continuous pings" )
1402 main.Mininet2.pingLong(
1403 src=main.params[ 'PING' ][ 'source1' ],
1404 target=main.params[ 'PING' ][ 'target1' ],
1405 pingTime=500 )
1406 main.Mininet2.pingLong(
1407 src=main.params[ 'PING' ][ 'source2' ],
1408 target=main.params[ 'PING' ][ 'target2' ],
1409 pingTime=500 )
1410 main.Mininet2.pingLong(
1411 src=main.params[ 'PING' ][ 'source3' ],
1412 target=main.params[ 'PING' ][ 'target3' ],
1413 pingTime=500 )
1414 main.Mininet2.pingLong(
1415 src=main.params[ 'PING' ][ 'source4' ],
1416 target=main.params[ 'PING' ][ 'target4' ],
1417 pingTime=500 )
1418 main.Mininet2.pingLong(
1419 src=main.params[ 'PING' ][ 'source5' ],
1420 target=main.params[ 'PING' ][ 'target5' ],
1421 pingTime=500 )
1422 main.Mininet2.pingLong(
1423 src=main.params[ 'PING' ][ 'source6' ],
1424 target=main.params[ 'PING' ][ 'target6' ],
1425 pingTime=500 )
1426 main.Mininet2.pingLong(
1427 src=main.params[ 'PING' ][ 'source7' ],
1428 target=main.params[ 'PING' ][ 'target7' ],
1429 pingTime=500 )
1430 main.Mininet2.pingLong(
1431 src=main.params[ 'PING' ][ 'source8' ],
1432 target=main.params[ 'PING' ][ 'target8' ],
1433 pingTime=500 )
1434 main.Mininet2.pingLong(
1435 src=main.params[ 'PING' ][ 'source9' ],
1436 target=main.params[ 'PING' ][ 'target9' ],
1437 pingTime=500 )
1438 main.Mininet2.pingLong(
1439 src=main.params[ 'PING' ][ 'source10' ],
1440 target=main.params[ 'PING' ][ 'target10' ],
1441 pingTime=500 )
1442
1443 main.step( "Collecting topology information from ONOS" )
1444 devices = []
1445 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001446 for i in range( main.numCtrls ):
1447 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001448 name="devices-" + str( i ),
1449 args=[ ] )
1450 threads.append( t )
1451 t.start()
1452
1453 for t in threads:
1454 t.join()
1455 devices.append( t.result )
1456 hosts = []
1457 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001458 for i in range( main.numCtrls ):
1459 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001460 name="hosts-" + str( i ),
1461 args=[ ] )
1462 threads.append( t )
1463 t.start()
1464
1465 for t in threads:
1466 t.join()
1467 try:
1468 hosts.append( json.loads( t.result ) )
1469 except ( ValueError, TypeError ):
1470 # FIXME: better handling of this, print which node
1471 # Maybe use thread name?
1472 main.log.exception( "Error parsing json output of hosts" )
1473 # FIXME: should this be an empty json object instead?
1474 hosts.append( None )
1475
1476 ports = []
1477 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001478 for i in range( main.numCtrls ):
1479 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001480 name="ports-" + str( i ),
1481 args=[ ] )
1482 threads.append( t )
1483 t.start()
1484
1485 for t in threads:
1486 t.join()
1487 ports.append( t.result )
1488 links = []
1489 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001490 for i in range( main.numCtrls ):
1491 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001492 name="links-" + str( i ),
1493 args=[ ] )
1494 threads.append( t )
1495 t.start()
1496
1497 for t in threads:
1498 t.join()
1499 links.append( t.result )
1500 clusters = []
1501 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001502 for i in range( main.numCtrls ):
1503 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001504 name="clusters-" + str( i ),
1505 args=[ ] )
1506 threads.append( t )
1507 t.start()
1508
1509 for t in threads:
1510 t.join()
1511 clusters.append( t.result )
1512 # Compare json objects for hosts and dataplane clusters
1513
1514 # hosts
1515 main.step( "Host view is consistent across ONOS nodes" )
1516 consistentHostsResult = main.TRUE
1517 for controller in range( len( hosts ) ):
1518 controllerStr = str( controller + 1 )
1519 if "Error" not in hosts[ controller ]:
1520 if hosts[ controller ] == hosts[ 0 ]:
1521 continue
1522 else: # hosts not consistent
1523 main.log.error( "hosts from ONOS" +
1524 controllerStr +
1525 " is inconsistent with ONOS1" )
1526 main.log.warn( repr( hosts[ controller ] ) )
1527 consistentHostsResult = main.FALSE
1528
1529 else:
1530 main.log.error( "Error in getting ONOS hosts from ONOS" +
1531 controllerStr )
1532 consistentHostsResult = main.FALSE
1533 main.log.warn( "ONOS" + controllerStr +
1534 " hosts response: " +
1535 repr( hosts[ controller ] ) )
1536 utilities.assert_equals(
1537 expect=main.TRUE,
1538 actual=consistentHostsResult,
1539 onpass="Hosts view is consistent across all ONOS nodes",
1540 onfail="ONOS nodes have different views of hosts" )
1541
1542 main.step( "Each host has an IP address" )
1543 ipResult = main.TRUE
1544 for controller in range( 0, len( hosts ) ):
1545 controllerStr = str( controller + 1 )
1546 for host in hosts[ controller ]:
1547 if not host.get( 'ipAddresses', [ ] ):
1548 main.log.error( "DEBUG:Error with host ips on controller" +
1549 controllerStr + ": " + str( host ) )
1550 ipResult = main.FALSE
1551 utilities.assert_equals(
1552 expect=main.TRUE,
1553 actual=ipResult,
1554 onpass="The ips of the hosts aren't empty",
1555 onfail="The ip of at least one host is missing" )
1556
1557 # Strongly connected clusters of devices
1558 main.step( "Cluster view is consistent across ONOS nodes" )
1559 consistentClustersResult = main.TRUE
1560 for controller in range( len( clusters ) ):
1561 controllerStr = str( controller + 1 )
1562 if "Error" not in clusters[ controller ]:
1563 if clusters[ controller ] == clusters[ 0 ]:
1564 continue
1565 else: # clusters not consistent
1566 main.log.error( "clusters from ONOS" + controllerStr +
1567 " is inconsistent with ONOS1" )
1568 consistentClustersResult = main.FALSE
1569
1570 else:
1571 main.log.error( "Error in getting dataplane clusters " +
1572 "from ONOS" + controllerStr )
1573 consistentClustersResult = main.FALSE
1574 main.log.warn( "ONOS" + controllerStr +
1575 " clusters response: " +
1576 repr( clusters[ controller ] ) )
1577 utilities.assert_equals(
1578 expect=main.TRUE,
1579 actual=consistentClustersResult,
1580 onpass="Clusters view is consistent across all ONOS nodes",
1581 onfail="ONOS nodes have different views of clusters" )
1582 # there should always only be one cluster
1583 main.step( "Cluster view correct across ONOS nodes" )
1584 try:
1585 numClusters = len( json.loads( clusters[ 0 ] ) )
1586 except ( ValueError, TypeError ):
1587 main.log.exception( "Error parsing clusters[0]: " +
1588 repr( clusters[ 0 ] ) )
1589 clusterResults = main.FALSE
1590 if numClusters == 1:
1591 clusterResults = main.TRUE
1592 utilities.assert_equals(
1593 expect=1,
1594 actual=numClusters,
1595 onpass="ONOS shows 1 SCC",
1596 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1597
1598 main.step( "Comparing ONOS topology to MN" )
1599 devicesResults = main.TRUE
1600 linksResults = main.TRUE
1601 hostsResults = main.TRUE
1602 mnSwitches = main.Mininet1.getSwitches()
1603 mnLinks = main.Mininet1.getLinks()
1604 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001605 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001606 controllerStr = str( controller + 1 )
1607 if devices[ controller ] and ports[ controller ] and\
1608 "Error" not in devices[ controller ] and\
1609 "Error" not in ports[ controller ]:
1610
1611 currentDevicesResult = main.Mininet1.compareSwitches(
1612 mnSwitches,
1613 json.loads( devices[ controller ] ),
1614 json.loads( ports[ controller ] ) )
1615 else:
1616 currentDevicesResult = main.FALSE
1617 utilities.assert_equals( expect=main.TRUE,
1618 actual=currentDevicesResult,
1619 onpass="ONOS" + controllerStr +
1620 " Switches view is correct",
1621 onfail="ONOS" + controllerStr +
1622 " Switches view is incorrect" )
1623 if links[ controller ] and "Error" not in links[ controller ]:
1624 currentLinksResult = main.Mininet1.compareLinks(
1625 mnSwitches, mnLinks,
1626 json.loads( links[ controller ] ) )
1627 else:
1628 currentLinksResult = main.FALSE
1629 utilities.assert_equals( expect=main.TRUE,
1630 actual=currentLinksResult,
1631 onpass="ONOS" + controllerStr +
1632 " links view is correct",
1633 onfail="ONOS" + controllerStr +
1634 " links view is incorrect" )
1635
1636 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1637 currentHostsResult = main.Mininet1.compareHosts(
1638 mnHosts,
1639 hosts[ controller ] )
1640 else:
1641 currentHostsResult = main.FALSE
1642 utilities.assert_equals( expect=main.TRUE,
1643 actual=currentHostsResult,
1644 onpass="ONOS" + controllerStr +
1645 " hosts exist in Mininet",
1646 onfail="ONOS" + controllerStr +
1647 " hosts don't match Mininet" )
1648
1649 devicesResults = devicesResults and currentDevicesResult
1650 linksResults = linksResults and currentLinksResult
1651 hostsResults = hostsResults and currentHostsResult
1652
1653 main.step( "Device information is correct" )
1654 utilities.assert_equals(
1655 expect=main.TRUE,
1656 actual=devicesResults,
1657 onpass="Device information is correct",
1658 onfail="Device information is incorrect" )
1659
1660 main.step( "Links are correct" )
1661 utilities.assert_equals(
1662 expect=main.TRUE,
1663 actual=linksResults,
1664 onpass="Link are correct",
1665 onfail="Links are incorrect" )
1666
1667 main.step( "Hosts are correct" )
1668 utilities.assert_equals(
1669 expect=main.TRUE,
1670 actual=hostsResults,
1671 onpass="Hosts are correct",
1672 onfail="Hosts are incorrect" )
1673
1674 def CASE6( self, main ):
1675 """
1676 The Failure case. Since this is the Sanity test, we do nothing.
1677 """
1678 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001679 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001680 assert main, "main not defined"
1681 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001682 assert main.CLIs, "main.CLIs not defined"
1683 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001684 main.case( "Wait 60 seconds instead of inducing a failure" )
1685 time.sleep( 60 )
1686 utilities.assert_equals(
1687 expect=main.TRUE,
1688 actual=main.TRUE,
1689 onpass="Sleeping 60 seconds",
1690 onfail="Something is terribly wrong with my math" )
1691
1692 def CASE7( self, main ):
1693 """
1694 Check state after ONOS failure
1695 """
1696 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001697 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001698 assert main, "main not defined"
1699 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001700 assert main.CLIs, "main.CLIs not defined"
1701 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001702 main.case( "Running ONOS Constant State Tests" )
1703
1704 main.step( "Check that each switch has a master" )
1705 # Assert that each device has a master
1706 rolesNotNull = main.TRUE
1707 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001708 for i in range( main.numCtrls ):
1709 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001710 name="rolesNotNull-" + str( i ),
1711 args=[ ] )
1712 threads.append( t )
1713 t.start()
1714
1715 for t in threads:
1716 t.join()
1717 rolesNotNull = rolesNotNull and t.result
1718 utilities.assert_equals(
1719 expect=main.TRUE,
1720 actual=rolesNotNull,
1721 onpass="Each device has a master",
1722 onfail="Some devices don't have a master assigned" )
1723
1724 main.step( "Read device roles from ONOS" )
1725 ONOSMastership = []
1726 mastershipCheck = main.FALSE
1727 consistentMastership = True
1728 rolesResults = True
1729 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001730 for i in range( main.numCtrls ):
1731 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001732 name="roles-" + str( i ),
1733 args=[] )
1734 threads.append( t )
1735 t.start()
1736
1737 for t in threads:
1738 t.join()
1739 ONOSMastership.append( t.result )
1740
Jon Halle1a3b752015-07-22 13:02:46 -07001741 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001742 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1743 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1744 " roles" )
1745 main.log.warn(
1746 "ONOS" + str( i + 1 ) + " mastership response: " +
1747 repr( ONOSMastership[i] ) )
1748 rolesResults = False
1749 utilities.assert_equals(
1750 expect=True,
1751 actual=rolesResults,
1752 onpass="No error in reading roles output",
1753 onfail="Error in reading roles from ONOS" )
1754
1755 main.step( "Check for consistency in roles from each controller" )
1756 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1757 main.log.info(
1758 "Switch roles are consistent across all ONOS nodes" )
1759 else:
1760 consistentMastership = False
1761 utilities.assert_equals(
1762 expect=True,
1763 actual=consistentMastership,
1764 onpass="Switch roles are consistent across all ONOS nodes",
1765 onfail="ONOS nodes have different views of switch roles" )
1766
1767 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001768 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001769 main.log.warn(
1770 "ONOS" + str( i + 1 ) + " roles: ",
1771 json.dumps(
1772 json.loads( ONOSMastership[ i ] ),
1773 sort_keys=True,
1774 indent=4,
1775 separators=( ',', ': ' ) ) )
1776 elif rolesResults and not consistentMastership:
1777 mastershipCheck = main.TRUE
1778
1779 description2 = "Compare switch roles from before failure"
1780 main.step( description2 )
1781 try:
1782 currentJson = json.loads( ONOSMastership[0] )
1783 oldJson = json.loads( mastershipState )
1784 except ( ValueError, TypeError ):
1785 main.log.exception( "Something is wrong with parsing " +
1786 "ONOSMastership[0] or mastershipState" )
1787 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1788 main.log.error( "mastershipState" + repr( mastershipState ) )
1789 main.cleanup()
1790 main.exit()
1791 mastershipCheck = main.TRUE
1792 for i in range( 1, 29 ):
1793 switchDPID = str(
1794 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1795 current = [ switch[ 'master' ] for switch in currentJson
1796 if switchDPID in switch[ 'id' ] ]
1797 old = [ switch[ 'master' ] for switch in oldJson
1798 if switchDPID in switch[ 'id' ] ]
1799 if current == old:
1800 mastershipCheck = mastershipCheck and main.TRUE
1801 else:
1802 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1803 mastershipCheck = main.FALSE
1804 utilities.assert_equals(
1805 expect=main.TRUE,
1806 actual=mastershipCheck,
1807 onpass="Mastership of Switches was not changed",
1808 onfail="Mastership of some switches changed" )
1809 mastershipCheck = mastershipCheck and consistentMastership
1810
1811 main.step( "Get the intents and compare across all nodes" )
1812 ONOSIntents = []
1813 intentCheck = main.FALSE
1814 consistentIntents = True
1815 intentsResults = True
1816 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001817 for i in range( main.numCtrls ):
1818 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001819 name="intents-" + str( i ),
1820 args=[],
1821 kwargs={ 'jsonFormat': True } )
1822 threads.append( t )
1823 t.start()
1824
1825 for t in threads:
1826 t.join()
1827 ONOSIntents.append( t.result )
1828
Jon Halle1a3b752015-07-22 13:02:46 -07001829 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001830 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1831 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1832 " intents" )
1833 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1834 repr( ONOSIntents[ i ] ) )
1835 intentsResults = False
1836 utilities.assert_equals(
1837 expect=True,
1838 actual=intentsResults,
1839 onpass="No error in reading intents output",
1840 onfail="Error in reading intents from ONOS" )
1841
1842 main.step( "Check for consistency in Intents from each controller" )
1843 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1844 main.log.info( "Intents are consistent across all ONOS " +
1845 "nodes" )
1846 else:
1847 consistentIntents = False
1848
1849 # Try to make it easy to figure out what is happening
1850 #
1851 # Intent ONOS1 ONOS2 ...
1852 # 0x01 INSTALLED INSTALLING
1853 # ... ... ...
1854 # ... ... ...
1855 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001856 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001857 title += " " * 10 + "ONOS" + str( n + 1 )
1858 main.log.warn( title )
1859 # get all intent keys in the cluster
1860 keys = []
1861 for nodeStr in ONOSIntents:
1862 node = json.loads( nodeStr )
1863 for intent in node:
1864 keys.append( intent.get( 'id' ) )
1865 keys = set( keys )
1866 for key in keys:
1867 row = "%-13s" % key
1868 for nodeStr in ONOSIntents:
1869 node = json.loads( nodeStr )
1870 for intent in node:
1871 if intent.get( 'id' ) == key:
1872 row += "%-15s" % intent.get( 'state' )
1873 main.log.warn( row )
1874 # End table view
1875
1876 utilities.assert_equals(
1877 expect=True,
1878 actual=consistentIntents,
1879 onpass="Intents are consistent across all ONOS nodes",
1880 onfail="ONOS nodes have different views of intents" )
1881 intentStates = []
1882 for node in ONOSIntents: # Iter through ONOS nodes
1883 nodeStates = []
1884 # Iter through intents of a node
1885 try:
1886 for intent in json.loads( node ):
1887 nodeStates.append( intent[ 'state' ] )
1888 except ( ValueError, TypeError ):
1889 main.log.exception( "Error in parsing intents" )
1890 main.log.error( repr( node ) )
1891 intentStates.append( nodeStates )
1892 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1893 main.log.info( dict( out ) )
1894
1895 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001896 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001897 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1898 main.log.warn( json.dumps(
1899 json.loads( ONOSIntents[ i ] ),
1900 sort_keys=True,
1901 indent=4,
1902 separators=( ',', ': ' ) ) )
1903 elif intentsResults and consistentIntents:
1904 intentCheck = main.TRUE
1905
1906 # NOTE: Store has no durability, so intents are lost across system
1907 # restarts
1908 main.step( "Compare current intents with intents before the failure" )
1909 # NOTE: this requires case 5 to pass for intentState to be set.
1910 # maybe we should stop the test if that fails?
1911 sameIntents = main.FALSE
1912 if intentState and intentState == ONOSIntents[ 0 ]:
1913 sameIntents = main.TRUE
1914 main.log.info( "Intents are consistent with before failure" )
1915 # TODO: possibly the states have changed? we may need to figure out
1916 # what the acceptable states are
1917 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1918 sameIntents = main.TRUE
1919 try:
1920 before = json.loads( intentState )
1921 after = json.loads( ONOSIntents[ 0 ] )
1922 for intent in before:
1923 if intent not in after:
1924 sameIntents = main.FALSE
1925 main.log.debug( "Intent is not currently in ONOS " +
1926 "(at least in the same form):" )
1927 main.log.debug( json.dumps( intent ) )
1928 except ( ValueError, TypeError ):
1929 main.log.exception( "Exception printing intents" )
1930 main.log.debug( repr( ONOSIntents[0] ) )
1931 main.log.debug( repr( intentState ) )
1932 if sameIntents == main.FALSE:
1933 try:
1934 main.log.debug( "ONOS intents before: " )
1935 main.log.debug( json.dumps( json.loads( intentState ),
1936 sort_keys=True, indent=4,
1937 separators=( ',', ': ' ) ) )
1938 main.log.debug( "Current ONOS intents: " )
1939 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1940 sort_keys=True, indent=4,
1941 separators=( ',', ': ' ) ) )
1942 except ( ValueError, TypeError ):
1943 main.log.exception( "Exception printing intents" )
1944 main.log.debug( repr( ONOSIntents[0] ) )
1945 main.log.debug( repr( intentState ) )
1946 utilities.assert_equals(
1947 expect=main.TRUE,
1948 actual=sameIntents,
1949 onpass="Intents are consistent with before failure",
1950 onfail="The Intents changed during failure" )
1951 intentCheck = intentCheck and sameIntents
1952
1953 main.step( "Get the OF Table entries and compare to before " +
1954 "component failure" )
1955 FlowTables = main.TRUE
1956 flows2 = []
1957 for i in range( 28 ):
1958 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall9043c902015-07-30 14:23:44 -07001959 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001960 flows2.append( tmpFlows )
Jon Hall9043c902015-07-30 14:23:44 -07001961 tempResult = main.Mininet1.flowComp(
Jon Hall5cf14d52015-07-16 12:15:19 -07001962 flow1=flows[ i ],
1963 flow2=tmpFlows )
1964 FlowTables = FlowTables and tempResult
1965 if FlowTables == main.FALSE:
1966 main.log.info( "Differences in flow table for switch: s" +
1967 str( i + 1 ) )
1968 utilities.assert_equals(
1969 expect=main.TRUE,
1970 actual=FlowTables,
1971 onpass="No changes were found in the flow tables",
1972 onfail="Changes were found in the flow tables" )
1973
1974 main.Mininet2.pingLongKill()
1975 '''
1976 main.step( "Check the continuous pings to ensure that no packets " +
1977 "were dropped during component failure" )
1978 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1979 main.params[ 'TESTONIP' ] )
1980 LossInPings = main.FALSE
1981 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1982 for i in range( 8, 18 ):
1983 main.log.info(
1984 "Checking for a loss in pings along flow from s" +
1985 str( i ) )
1986 LossInPings = main.Mininet2.checkForLoss(
1987 "/tmp/ping.h" +
1988 str( i ) ) or LossInPings
1989 if LossInPings == main.TRUE:
1990 main.log.info( "Loss in ping detected" )
1991 elif LossInPings == main.ERROR:
1992 main.log.info( "There are multiple mininet process running" )
1993 elif LossInPings == main.FALSE:
1994 main.log.info( "No Loss in the pings" )
1995 main.log.info( "No loss of dataplane connectivity" )
1996 utilities.assert_equals(
1997 expect=main.FALSE,
1998 actual=LossInPings,
1999 onpass="No Loss of connectivity",
2000 onfail="Loss of dataplane connectivity detected" )
2001 '''
2002
2003 main.step( "Leadership Election is still functional" )
2004 # Test of LeadershipElection
2005 # NOTE: this only works for the sanity test. In case of failures,
2006 # leader will likely change
Jon Halle1a3b752015-07-22 13:02:46 -07002007 leader = main.nodes[ 0 ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002008 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002009 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002010 leaderN = cli.electionTestLeader()
2011 # verify leader is ONOS1
2012 if leaderN == leader:
2013 # all is well
2014 # NOTE: In failure scenario, this could be a new node, maybe
2015 # check != ONOS1
2016 pass
2017 elif leaderN == main.FALSE:
2018 # error in response
2019 main.log.error( "Something is wrong with " +
2020 "electionTestLeader function, check the" +
2021 " error logs" )
2022 leaderResult = main.FALSE
2023 elif leader != leaderN:
2024 leaderResult = main.FALSE
2025 main.log.error( cli.name + " sees " + str( leaderN ) +
2026 " as the leader of the election app. " +
2027 "Leader should be " + str( leader ) )
2028 utilities.assert_equals(
2029 expect=main.TRUE,
2030 actual=leaderResult,
2031 onpass="Leadership election passed",
2032 onfail="Something went wrong with Leadership election" )
2033
2034 def CASE8( self, main ):
2035 """
2036 Compare topo
2037 """
2038 import json
2039 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002040 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002041 assert main, "main not defined"
2042 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002043 assert main.CLIs, "main.CLIs not defined"
2044 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002045
2046 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002047 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002048 " and ONOS"
2049
2050 main.step( "Comparing ONOS topology to MN" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002051 topoResult = main.FALSE
2052 elapsed = 0
2053 count = 0
2054 main.step( "Collecting topology information from ONOS" )
2055 startTime = time.time()
2056 # Give time for Gossip to work
2057 while topoResult == main.FALSE and elapsed < 60:
Jon Hall96091e62015-09-21 17:34:17 -07002058 devicesResults = main.TRUE
2059 linksResults = main.TRUE
2060 hostsResults = main.TRUE
2061 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002062 count += 1
2063 cliStart = time.time()
2064 devices = []
2065 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002066 for i in range( main.numCtrls ):
2067 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002068 name="devices-" + str( i ),
2069 args=[ ] )
2070 threads.append( t )
2071 t.start()
2072
2073 for t in threads:
2074 t.join()
2075 devices.append( t.result )
2076 hosts = []
2077 ipResult = main.TRUE
2078 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002079 for i in range( main.numCtrls ):
2080 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07002081 name="hosts-" + str( i ),
2082 args=[ ] )
2083 threads.append( t )
2084 t.start()
2085
2086 for t in threads:
2087 t.join()
2088 try:
2089 hosts.append( json.loads( t.result ) )
2090 except ( ValueError, TypeError ):
2091 main.log.exception( "Error parsing hosts results" )
2092 main.log.error( repr( t.result ) )
2093 for controller in range( 0, len( hosts ) ):
2094 controllerStr = str( controller + 1 )
2095 for host in hosts[ controller ]:
2096 if host is None or host.get( 'ipAddresses', [] ) == []:
2097 main.log.error(
2098 "DEBUG:Error with host ipAddresses on controller" +
2099 controllerStr + ": " + str( host ) )
2100 ipResult = main.FALSE
2101 ports = []
2102 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002103 for i in range( main.numCtrls ):
2104 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002105 name="ports-" + str( i ),
2106 args=[ ] )
2107 threads.append( t )
2108 t.start()
2109
2110 for t in threads:
2111 t.join()
2112 ports.append( t.result )
2113 links = []
2114 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002115 for i in range( main.numCtrls ):
2116 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002117 name="links-" + str( i ),
2118 args=[ ] )
2119 threads.append( t )
2120 t.start()
2121
2122 for t in threads:
2123 t.join()
2124 links.append( t.result )
2125 clusters = []
2126 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002127 for i in range( main.numCtrls ):
2128 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002129 name="clusters-" + str( i ),
2130 args=[ ] )
2131 threads.append( t )
2132 t.start()
2133
2134 for t in threads:
2135 t.join()
2136 clusters.append( t.result )
2137
2138 elapsed = time.time() - startTime
2139 cliTime = time.time() - cliStart
2140 print "Elapsed time: " + str( elapsed )
2141 print "CLI time: " + str( cliTime )
2142
2143 mnSwitches = main.Mininet1.getSwitches()
2144 mnLinks = main.Mininet1.getLinks()
2145 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002146 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002147 controllerStr = str( controller + 1 )
2148 if devices[ controller ] and ports[ controller ] and\
2149 "Error" not in devices[ controller ] and\
2150 "Error" not in ports[ controller ]:
2151
2152 currentDevicesResult = main.Mininet1.compareSwitches(
2153 mnSwitches,
2154 json.loads( devices[ controller ] ),
2155 json.loads( ports[ controller ] ) )
2156 else:
2157 currentDevicesResult = main.FALSE
2158 utilities.assert_equals( expect=main.TRUE,
2159 actual=currentDevicesResult,
2160 onpass="ONOS" + controllerStr +
2161 " Switches view is correct",
2162 onfail="ONOS" + controllerStr +
2163 " Switches view is incorrect" )
2164
2165 if links[ controller ] and "Error" not in links[ controller ]:
2166 currentLinksResult = main.Mininet1.compareLinks(
2167 mnSwitches, mnLinks,
2168 json.loads( links[ controller ] ) )
2169 else:
2170 currentLinksResult = main.FALSE
2171 utilities.assert_equals( expect=main.TRUE,
2172 actual=currentLinksResult,
2173 onpass="ONOS" + controllerStr +
2174 " links view is correct",
2175 onfail="ONOS" + controllerStr +
2176 " links view is incorrect" )
2177
2178 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2179 currentHostsResult = main.Mininet1.compareHosts(
2180 mnHosts,
2181 hosts[ controller ] )
2182 else:
2183 currentHostsResult = main.FALSE
2184 utilities.assert_equals( expect=main.TRUE,
2185 actual=currentHostsResult,
2186 onpass="ONOS" + controllerStr +
2187 " hosts exist in Mininet",
2188 onfail="ONOS" + controllerStr +
2189 " hosts don't match Mininet" )
2190 # CHECKING HOST ATTACHMENT POINTS
2191 hostAttachment = True
2192 zeroHosts = False
2193 # FIXME: topo-HA/obelisk specific mappings:
2194 # key is mac and value is dpid
2195 mappings = {}
2196 for i in range( 1, 29 ): # hosts 1 through 28
2197 # set up correct variables:
2198 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2199 if i == 1:
2200 deviceId = "1000".zfill(16)
2201 elif i == 2:
2202 deviceId = "2000".zfill(16)
2203 elif i == 3:
2204 deviceId = "3000".zfill(16)
2205 elif i == 4:
2206 deviceId = "3004".zfill(16)
2207 elif i == 5:
2208 deviceId = "5000".zfill(16)
2209 elif i == 6:
2210 deviceId = "6000".zfill(16)
2211 elif i == 7:
2212 deviceId = "6007".zfill(16)
2213 elif i >= 8 and i <= 17:
2214 dpid = '3' + str( i ).zfill( 3 )
2215 deviceId = dpid.zfill(16)
2216 elif i >= 18 and i <= 27:
2217 dpid = '6' + str( i ).zfill( 3 )
2218 deviceId = dpid.zfill(16)
2219 elif i == 28:
2220 deviceId = "2800".zfill(16)
2221 mappings[ macId ] = deviceId
2222 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2223 if hosts[ controller ] == []:
2224 main.log.warn( "There are no hosts discovered" )
2225 zeroHosts = True
2226 else:
2227 for host in hosts[ controller ]:
2228 mac = None
2229 location = None
2230 device = None
2231 port = None
2232 try:
2233 mac = host.get( 'mac' )
2234 assert mac, "mac field could not be found for this host object"
2235
2236 location = host.get( 'location' )
2237 assert location, "location field could not be found for this host object"
2238
2239 # Trim the protocol identifier off deviceId
2240 device = str( location.get( 'elementId' ) ).split(':')[1]
2241 assert device, "elementId field could not be found for this host location object"
2242
2243 port = location.get( 'port' )
2244 assert port, "port field could not be found for this host location object"
2245
2246 # Now check if this matches where they should be
2247 if mac and device and port:
2248 if str( port ) != "1":
2249 main.log.error( "The attachment port is incorrect for " +
2250 "host " + str( mac ) +
2251 ". Expected: 1 Actual: " + str( port) )
2252 hostAttachment = False
2253 if device != mappings[ str( mac ) ]:
2254 main.log.error( "The attachment device is incorrect for " +
2255 "host " + str( mac ) +
2256 ". Expected: " + mappings[ str( mac ) ] +
2257 " Actual: " + device )
2258 hostAttachment = False
2259 else:
2260 hostAttachment = False
2261 except AssertionError:
2262 main.log.exception( "Json object not as expected" )
2263 main.log.error( repr( host ) )
2264 hostAttachment = False
2265 else:
2266 main.log.error( "No hosts json output or \"Error\"" +
2267 " in output. hosts = " +
2268 repr( hosts[ controller ] ) )
2269 if zeroHosts is False:
2270 hostAttachment = True
2271
2272 # END CHECKING HOST ATTACHMENT POINTS
2273 devicesResults = devicesResults and currentDevicesResult
2274 linksResults = linksResults and currentLinksResult
2275 hostsResults = hostsResults and currentHostsResult
2276 hostAttachmentResults = hostAttachmentResults and\
2277 hostAttachment
2278 topoResult = ( devicesResults and linksResults
2279 and hostsResults and ipResult and
2280 hostAttachmentResults )
2281
2282 # Compare json objects for hosts and dataplane clusters
2283
2284 # hosts
2285 main.step( "Hosts view is consistent across all ONOS nodes" )
2286 consistentHostsResult = main.TRUE
2287 for controller in range( len( hosts ) ):
2288 controllerStr = str( controller + 1 )
2289 if "Error" not in hosts[ controller ]:
2290 if hosts[ controller ] == hosts[ 0 ]:
2291 continue
2292 else: # hosts not consistent
2293 main.log.error( "hosts from ONOS" + controllerStr +
2294 " is inconsistent with ONOS1" )
2295 main.log.warn( repr( hosts[ controller ] ) )
2296 consistentHostsResult = main.FALSE
2297
2298 else:
2299 main.log.error( "Error in getting ONOS hosts from ONOS" +
2300 controllerStr )
2301 consistentHostsResult = main.FALSE
2302 main.log.warn( "ONOS" + controllerStr +
2303 " hosts response: " +
2304 repr( hosts[ controller ] ) )
2305 utilities.assert_equals(
2306 expect=main.TRUE,
2307 actual=consistentHostsResult,
2308 onpass="Hosts view is consistent across all ONOS nodes",
2309 onfail="ONOS nodes have different views of hosts" )
2310
2311 main.step( "Hosts information is correct" )
2312 hostsResults = hostsResults and ipResult
2313 utilities.assert_equals(
2314 expect=main.TRUE,
2315 actual=hostsResults,
2316 onpass="Host information is correct",
2317 onfail="Host information is incorrect" )
2318
2319 main.step( "Host attachment points to the network" )
2320 utilities.assert_equals(
2321 expect=True,
2322 actual=hostAttachmentResults,
2323 onpass="Hosts are correctly attached to the network",
2324 onfail="ONOS did not correctly attach hosts to the network" )
2325
2326 # Strongly connected clusters of devices
2327 main.step( "Clusters view is consistent across all ONOS nodes" )
2328 consistentClustersResult = main.TRUE
2329 for controller in range( len( clusters ) ):
2330 controllerStr = str( controller + 1 )
2331 if "Error" not in clusters[ controller ]:
2332 if clusters[ controller ] == clusters[ 0 ]:
2333 continue
2334 else: # clusters not consistent
2335 main.log.error( "clusters from ONOS" +
2336 controllerStr +
2337 " is inconsistent with ONOS1" )
2338 consistentClustersResult = main.FALSE
2339
2340 else:
2341 main.log.error( "Error in getting dataplane clusters " +
2342 "from ONOS" + controllerStr )
2343 consistentClustersResult = main.FALSE
2344 main.log.warn( "ONOS" + controllerStr +
2345 " clusters response: " +
2346 repr( clusters[ controller ] ) )
2347 utilities.assert_equals(
2348 expect=main.TRUE,
2349 actual=consistentClustersResult,
2350 onpass="Clusters view is consistent across all ONOS nodes",
2351 onfail="ONOS nodes have different views of clusters" )
2352
2353 main.step( "There is only one SCC" )
2354 # there should always only be one cluster
2355 try:
2356 numClusters = len( json.loads( clusters[ 0 ] ) )
2357 except ( ValueError, TypeError ):
2358 main.log.exception( "Error parsing clusters[0]: " +
2359 repr( clusters[0] ) )
2360 clusterResults = main.FALSE
2361 if numClusters == 1:
2362 clusterResults = main.TRUE
2363 utilities.assert_equals(
2364 expect=1,
2365 actual=numClusters,
2366 onpass="ONOS shows 1 SCC",
2367 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2368
2369 topoResult = ( devicesResults and linksResults
2370 and hostsResults and consistentHostsResult
2371 and consistentClustersResult and clusterResults
2372 and ipResult and hostAttachmentResults )
2373
2374 topoResult = topoResult and int( count <= 2 )
2375 note = "note it takes about " + str( int( cliTime ) ) + \
2376 " seconds for the test to make all the cli calls to fetch " +\
2377 "the topology from each ONOS instance"
2378 main.log.info(
2379 "Very crass estimate for topology discovery/convergence( " +
2380 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2381 str( count ) + " tries" )
2382
2383 main.step( "Device information is correct" )
2384 utilities.assert_equals(
2385 expect=main.TRUE,
2386 actual=devicesResults,
2387 onpass="Device information is correct",
2388 onfail="Device information is incorrect" )
2389
2390 main.step( "Links are correct" )
2391 utilities.assert_equals(
2392 expect=main.TRUE,
2393 actual=linksResults,
2394 onpass="Link are correct",
2395 onfail="Links are incorrect" )
2396
2397 main.step( "Hosts are correct" )
2398 utilities.assert_equals(
2399 expect=main.TRUE,
2400 actual=hostsResults,
2401 onpass="Hosts are correct",
2402 onfail="Hosts are incorrect" )
2403
2404 # FIXME: move this to an ONOS state case
2405 main.step( "Checking ONOS nodes" )
2406 nodesOutput = []
2407 nodeResults = main.TRUE
2408 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002409 for i in range( main.numCtrls ):
2410 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002411 name="nodes-" + str( i ),
2412 args=[ ] )
2413 threads.append( t )
2414 t.start()
2415
2416 for t in threads:
2417 t.join()
2418 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002419 ips = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002420 for i in nodesOutput:
2421 try:
2422 current = json.loads( i )
2423 for node in current:
2424 currentResult = main.FALSE
2425 if node['ip'] in ips: # node in nodes() output is in cell
2426 if node['state'] == 'ACTIVE':
2427 currentResult = main.TRUE
2428 else:
2429 main.log.error( "Error in ONOS node availability" )
2430 main.log.error(
2431 json.dumps( current,
2432 sort_keys=True,
2433 indent=4,
2434 separators=( ',', ': ' ) ) )
2435 break
2436 nodeResults = nodeResults and currentResult
2437 except ( ValueError, TypeError ):
2438 main.log.error( "Error parsing nodes output" )
2439 main.log.warn( repr( i ) )
2440 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2441 onpass="Nodes check successful",
2442 onfail="Nodes check NOT successful" )
2443
2444 def CASE9( self, main ):
2445 """
2446 Link s3-s28 down
2447 """
2448 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002449 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002450 assert main, "main not defined"
2451 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002452 assert main.CLIs, "main.CLIs not defined"
2453 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002454 # NOTE: You should probably run a topology check after this
2455
2456 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2457
2458 description = "Turn off a link to ensure that Link Discovery " +\
2459 "is working properly"
2460 main.case( description )
2461
2462 main.step( "Kill Link between s3 and s28" )
2463 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2464 main.log.info( "Waiting " + str( linkSleep ) +
2465 " seconds for link down to be discovered" )
2466 time.sleep( linkSleep )
2467 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2468 onpass="Link down successful",
2469 onfail="Failed to bring link down" )
2470 # TODO do some sort of check here
2471
2472 def CASE10( self, main ):
2473 """
2474 Link s3-s28 up
2475 """
2476 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002477 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002478 assert main, "main not defined"
2479 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002480 assert main.CLIs, "main.CLIs not defined"
2481 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002482 # NOTE: You should probably run a topology check after this
2483
2484 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2485
2486 description = "Restore a link to ensure that Link Discovery is " + \
2487 "working properly"
2488 main.case( description )
2489
2490 main.step( "Bring link between s3 and s28 back up" )
2491 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2492 main.log.info( "Waiting " + str( linkSleep ) +
2493 " seconds for link up to be discovered" )
2494 time.sleep( linkSleep )
2495 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2496 onpass="Link up successful",
2497 onfail="Failed to bring link up" )
2498 # TODO do some sort of check here
2499
2500 def CASE11( self, main ):
2501 """
2502 Switch Down
2503 """
2504 # NOTE: You should probably run a topology check after this
2505 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002506 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002507 assert main, "main not defined"
2508 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002509 assert main.CLIs, "main.CLIs not defined"
2510 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002511
2512 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2513
2514 description = "Killing a switch to ensure it is discovered correctly"
2515 main.case( description )
2516 switch = main.params[ 'kill' ][ 'switch' ]
2517 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2518
2519 # TODO: Make this switch parameterizable
2520 main.step( "Kill " + switch )
2521 main.log.info( "Deleting " + switch )
2522 main.Mininet1.delSwitch( switch )
2523 main.log.info( "Waiting " + str( switchSleep ) +
2524 " seconds for switch down to be discovered" )
2525 time.sleep( switchSleep )
2526 device = main.ONOScli1.getDevice( dpid=switchDPID )
2527 # Peek at the deleted switch
2528 main.log.warn( str( device ) )
2529 result = main.FALSE
2530 if device and device[ 'available' ] is False:
2531 result = main.TRUE
2532 utilities.assert_equals( expect=main.TRUE, actual=result,
2533 onpass="Kill switch successful",
2534 onfail="Failed to kill switch?" )
2535
2536 def CASE12( self, main ):
2537 """
2538 Switch Up
2539 """
2540 # NOTE: You should probably run a topology check after this
2541 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002542 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002543 assert main, "main not defined"
2544 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002545 assert main.CLIs, "main.CLIs not defined"
2546 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002547 assert ONOS1Port, "ONOS1Port not defined"
2548 assert ONOS2Port, "ONOS2Port not defined"
2549 assert ONOS3Port, "ONOS3Port not defined"
2550 assert ONOS4Port, "ONOS4Port not defined"
2551 assert ONOS5Port, "ONOS5Port not defined"
2552 assert ONOS6Port, "ONOS6Port not defined"
2553 assert ONOS7Port, "ONOS7Port not defined"
2554
2555 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2556 switch = main.params[ 'kill' ][ 'switch' ]
2557 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2558 links = main.params[ 'kill' ][ 'links' ].split()
2559 description = "Adding a switch to ensure it is discovered correctly"
2560 main.case( description )
2561
2562 main.step( "Add back " + switch )
2563 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2564 for peer in links:
2565 main.Mininet1.addLink( switch, peer )
2566 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002567 for i in range( main.numCtrls ):
2568 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002569 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2570 main.log.info( "Waiting " + str( switchSleep ) +
2571 " seconds for switch up to be discovered" )
2572 time.sleep( switchSleep )
2573 device = main.ONOScli1.getDevice( dpid=switchDPID )
2574 # Peek at the deleted switch
2575 main.log.warn( str( device ) )
2576 result = main.FALSE
2577 if device and device[ 'available' ]:
2578 result = main.TRUE
2579 utilities.assert_equals( expect=main.TRUE, actual=result,
2580 onpass="add switch successful",
2581 onfail="Failed to add switch?" )
2582
2583 def CASE13( self, main ):
2584 """
2585 Clean up
2586 """
2587 import os
2588 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002589 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002590 assert main, "main not defined"
2591 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002592 assert main.CLIs, "main.CLIs not defined"
2593 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002594
2595 # printing colors to terminal
2596 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2597 'blue': '\033[94m', 'green': '\033[92m',
2598 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2599 main.case( "Test Cleanup" )
2600 main.step( "Killing tcpdumps" )
2601 main.Mininet2.stopTcpdump()
2602
2603 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002604 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002605 main.step( "Copying MN pcap and ONOS log files to test station" )
2606 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2607 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002608 # NOTE: MN Pcap file is being saved to logdir.
2609 # We scp this file as MN and TestON aren't necessarily the same vm
2610
2611 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002612 # TODO: Load these from params
2613 # NOTE: must end in /
2614 logFolder = "/opt/onos/log/"
2615 logFiles = [ "karaf.log", "karaf.log.1" ]
2616 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002617 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002618 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002619 dstName = main.logdir + "/" + node.name + "-" + f
2620 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2621 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002622 # std*.log's
2623 # NOTE: must end in /
2624 logFolder = "/opt/onos/var/"
2625 logFiles = [ "stderr.log", "stdout.log" ]
2626 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002627 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002628 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002629 dstName = main.logdir + "/" + node.name + "-" + f
2630 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2631 logFolder + f, dstName )
2632 else:
2633 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002634
2635 main.step( "Stopping Mininet" )
2636 mnResult = main.Mininet1.stopNet()
2637 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2638 onpass="Mininet stopped",
2639 onfail="MN cleanup NOT successful" )
2640
2641 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002642 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002643 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2644 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002645
2646 try:
2647 timerLog = open( main.logdir + "/Timers.csv", 'w')
2648 # Overwrite with empty line and close
2649 labels = "Gossip Intents"
2650 data = str( gossipTime )
2651 timerLog.write( labels + "\n" + data )
2652 timerLog.close()
2653 except NameError, e:
2654 main.log.exception(e)
2655
2656 def CASE14( self, main ):
2657 """
2658 start election app on all onos nodes
2659 """
Jon Halle1a3b752015-07-22 13:02:46 -07002660 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002661 assert main, "main not defined"
2662 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002663 assert main.CLIs, "main.CLIs not defined"
2664 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002665
2666 main.case("Start Leadership Election app")
2667 main.step( "Install leadership election app" )
2668 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2669 utilities.assert_equals(
2670 expect=main.TRUE,
2671 actual=appResult,
2672 onpass="Election app installed",
2673 onfail="Something went wrong with installing Leadership election" )
2674
2675 main.step( "Run for election on each node" )
2676 leaderResult = main.TRUE
2677 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002678 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002679 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002680 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002681 leader = cli.electionTestLeader()
2682 if leader is None or leader == main.FALSE:
2683 main.log.error( cli.name + ": Leader for the election app " +
2684 "should be an ONOS node, instead got '" +
2685 str( leader ) + "'" )
2686 leaderResult = main.FALSE
2687 leaders.append( leader )
2688 utilities.assert_equals(
2689 expect=main.TRUE,
2690 actual=leaderResult,
2691 onpass="Successfully ran for leadership",
2692 onfail="Failed to run for leadership" )
2693
2694 main.step( "Check that each node shows the same leader" )
2695 sameLeader = main.TRUE
2696 if len( set( leaders ) ) != 1:
2697 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002698 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002699 str( leaders ) )
2700 utilities.assert_equals(
2701 expect=main.TRUE,
2702 actual=sameLeader,
2703 onpass="Leadership is consistent for the election topic",
2704 onfail="Nodes have different leaders" )
2705
2706 def CASE15( self, main ):
2707 """
2708 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002709 15.1 Run election on each node
2710 15.2 Check that each node has the same leaders and candidates
2711 15.3 Find current leader and withdraw
2712 15.4 Check that a new node was elected leader
2713 15.5 Check that that new leader was the candidate of old leader
2714 15.6 Run for election on old leader
2715 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2716 15.8 Make sure that the old leader was added to the candidate list
2717
2718 old and new variable prefixes refer to data from before vs after
2719 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002720 """
2721 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002722 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002723 assert main, "main not defined"
2724 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002725 assert main.CLIs, "main.CLIs not defined"
2726 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002727
acsmars3a72bde2015-09-02 14:16:22 -07002728 description = "Check that Leadership Election App is still functional"
Jon Hall5cf14d52015-07-16 12:15:19 -07002729 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07002730 # NOTE: Need to re-run since being a canidate is not persistant
2731 # TODO: add check for "Command not found:" in the driver, this
2732 # means the election test app isn't loaded
Jon Hall5cf14d52015-07-16 12:15:19 -07002733
acsmars71adceb2015-08-31 15:09:26 -07002734 oldLeaders = [] # leaders by node before withdrawl from candidates
2735 newLeaders = [] # leaders by node after withdrawl from candidates
2736 oldAllCandidates = [] # list of lists of each nodes' candidates before
2737 newAllCandidates = [] # list of lists of each nodes' candidates after
2738 oldCandidates = [] # list of candidates from node 0 before withdrawl
2739 newCandidates = [] # list of candidates from node 0 after withdrawl
2740 oldLeader = '' # the old leader from oldLeaders, None if not same
2741 newLeader = '' # the new leaders fron newLoeaders, None if not same
2742 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2743 expectNoLeader = False # True when there is only one leader
2744 if main.numCtrls == 1:
2745 expectNoLeader = True
2746
2747 main.step( "Run for election on each node" )
2748 electionResult = main.TRUE
2749
2750 for cli in main.CLIs: # run test election on each node
2751 if cli.electionTestRun() == main.FALSE:
2752 electionResult = main.FALSE
2753
Jon Hall5cf14d52015-07-16 12:15:19 -07002754 utilities.assert_equals(
2755 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002756 actual=electionResult,
2757 onpass="All nodes successfully ran for leadership",
2758 onfail="At least one node failed to run for leadership" )
2759
acsmars3a72bde2015-09-02 14:16:22 -07002760 if electionResult == main.FALSE:
2761 main.log.error(
2762 "Skipping Test Case because Election Test isn't loaded" )
2763 main.skipCase()
2764
acsmars71adceb2015-08-31 15:09:26 -07002765 main.step( "Check that each node shows the same leader and candidates" )
2766 sameResult = main.TRUE
2767 failMessage = "Nodes have different leaders"
2768 for cli in main.CLIs:
2769 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2770 oldAllCandidates.append( node )
2771 oldLeaders.append( node[ 0 ] )
2772 oldCandidates = oldAllCandidates[ 0 ]
2773
2774 # Check that each node has the same leader. Defines oldLeader
2775 if len( set( oldLeaders ) ) != 1:
2776 sameResult = main.FALSE
2777 main.log.error( "More than one leader present:" + str( oldLeaders ) )
2778 oldLeader = None
2779 else:
2780 oldLeader = oldLeaders[ 0 ]
2781
2782 # Check that each node's candidate list is the same
acsmars29233db2015-11-04 11:15:00 -08002783 candidateDiscrepancy = False # Boolean of candidate mismatches
acsmars71adceb2015-08-31 15:09:26 -07002784 for candidates in oldAllCandidates:
2785 if set( candidates ) != set( oldCandidates ):
2786 sameResult = main.FALSE
acsmars29233db2015-11-04 11:15:00 -08002787 candidateDiscrepancy = True
2788
2789 if candidateDiscrepancy:
2790 failMessage += " and candidates"
acsmars71adceb2015-08-31 15:09:26 -07002791
2792 utilities.assert_equals(
2793 expect=main.TRUE,
2794 actual=sameResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002795 onpass="Leadership is consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002796 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002797
2798 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002799 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002800 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002801 if oldLeader is None:
2802 main.log.error( "Leadership isn't consistent." )
2803 withdrawResult = main.FALSE
2804 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002805 for i in range( len( main.CLIs ) ):
acsmars71adceb2015-08-31 15:09:26 -07002806 if oldLeader == main.nodes[ i ].ip_address:
2807 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002808 break
2809 else: # FOR/ELSE statement
2810 main.log.error( "Leader election, could not find current leader" )
2811 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002812 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002813 utilities.assert_equals(
2814 expect=main.TRUE,
2815 actual=withdrawResult,
2816 onpass="Node was withdrawn from election",
2817 onfail="Node was not withdrawn from election" )
2818
acsmars71adceb2015-08-31 15:09:26 -07002819 main.step( "Check that a new node was elected leader" )
2820
Jon Hall5cf14d52015-07-16 12:15:19 -07002821 # FIXME: use threads
acsmars71adceb2015-08-31 15:09:26 -07002822 newLeaderResult = main.TRUE
2823 failMessage = "Nodes have different leaders"
2824
2825 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002826 for cli in main.CLIs:
acsmars71adceb2015-08-31 15:09:26 -07002827 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2828 # elections might no have finished yet
2829 if node[ 0 ] == 'none' and not expectNoLeader:
2830 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2831 "sure elections are complete." )
2832 time.sleep(5)
2833 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2834 # election still isn't done or there is a problem
2835 if node[ 0 ] == 'none':
2836 main.log.error( "No leader was elected on at least 1 node" )
2837 newLeaderResult = main.FALSE
2838 newAllCandidates.append( node )
2839 newLeaders.append( node[ 0 ] )
2840 newCandidates = newAllCandidates[ 0 ]
2841
2842 # Check that each node has the same leader. Defines newLeader
2843 if len( set( newLeaders ) ) != 1:
2844 newLeaderResult = main.FALSE
2845 main.log.error( "Nodes have different leaders: " +
2846 str( newLeaders ) )
2847 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002848 else:
acsmars71adceb2015-08-31 15:09:26 -07002849 newLeader = newLeaders[ 0 ]
2850
2851 # Check that each node's candidate list is the same
2852 for candidates in newAllCandidates:
2853 if set( candidates ) != set( newCandidates ):
2854 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002855 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002856
2857 # Check that the new leader is not the older leader, which was withdrawn
2858 if newLeader == oldLeader:
2859 newLeaderResult = main.FALSE
2860 main.log.error( "All nodes still see old leader: " + oldLeader +
2861 " as the current leader" )
2862
Jon Hall5cf14d52015-07-16 12:15:19 -07002863 utilities.assert_equals(
2864 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002865 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002866 onpass="Leadership election passed",
2867 onfail="Something went wrong with Leadership election" )
2868
acsmars71adceb2015-08-31 15:09:26 -07002869 main.step( "Check that that new leader was the candidate of old leader")
2870 # candidates[ 2 ] should be come the top candidate after withdrawl
2871 correctCandidateResult = main.TRUE
2872 if expectNoLeader:
2873 if newLeader == 'none':
2874 main.log.info( "No leader expected. None found. Pass" )
2875 correctCandidateResult = main.TRUE
2876 else:
2877 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2878 correctCandidateResult = main.FALSE
2879 elif newLeader != oldCandidates[ 2 ]:
2880 correctCandidateResult = main.FALSE
2881 main.log.error( "Candidate " + newLeader + " was elected. " +
2882 oldCandidates[ 2 ] + " should have had priority." )
2883
2884 utilities.assert_equals(
2885 expect=main.TRUE,
2886 actual=correctCandidateResult,
2887 onpass="Correct Candidate Elected",
2888 onfail="Incorrect Candidate Elected" )
2889
Jon Hall5cf14d52015-07-16 12:15:19 -07002890 main.step( "Run for election on old leader( just so everyone " +
2891 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002892 if oldLeaderCLI is not None:
2893 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002894 else:
acsmars71adceb2015-08-31 15:09:26 -07002895 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002896 runResult = main.FALSE
2897 utilities.assert_equals(
2898 expect=main.TRUE,
2899 actual=runResult,
2900 onpass="App re-ran for election",
2901 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002902 main.step(
2903 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002904 # verify leader didn't just change
acsmars71adceb2015-08-31 15:09:26 -07002905 positionResult = main.TRUE
2906 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
2907
2908 # Reset and reuse the new candidate and leaders lists
2909 newAllCandidates = []
2910 newCandidates = []
2911 newLeaders = []
2912 for cli in main.CLIs:
2913 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2914 if oldLeader not in node: # election might no have finished yet
2915 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2916 "be sure elections are complete" )
2917 time.sleep(5)
2918 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2919 if oldLeader not in node: # election still isn't done, errors
2920 main.log.error(
2921 "Old leader was not elected on at least one node" )
2922 positionResult = main.FALSE
2923 newAllCandidates.append( node )
2924 newLeaders.append( node[ 0 ] )
2925 newCandidates = newAllCandidates[ 0 ]
2926
2927 # Check that each node has the same leader. Defines newLeader
2928 if len( set( newLeaders ) ) != 1:
2929 positionResult = main.FALSE
2930 main.log.error( "Nodes have different leaders: " +
2931 str( newLeaders ) )
2932 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002933 else:
acsmars71adceb2015-08-31 15:09:26 -07002934 newLeader = newLeaders[ 0 ]
2935
2936 # Check that each node's candidate list is the same
2937 for candidates in newAllCandidates:
2938 if set( candidates ) != set( newCandidates ):
2939 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002940 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002941
2942 # Check that the re-elected node is last on the candidate List
2943 if oldLeader != newCandidates[ -1 ]:
2944 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
2945 str( newCandidates ) )
2946 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002947
2948 utilities.assert_equals(
2949 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002950 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002951 onpass="Old leader successfully re-ran for election",
2952 onfail="Something went wrong with Leadership election after " +
2953 "the old leader re-ran for election" )
2954
2955 def CASE16( self, main ):
2956 """
2957 Install Distributed Primitives app
2958 """
2959 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002960 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002961 assert main, "main not defined"
2962 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002963 assert main.CLIs, "main.CLIs not defined"
2964 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002965
2966 # Variables for the distributed primitives tests
2967 global pCounterName
2968 global iCounterName
2969 global pCounterValue
2970 global iCounterValue
2971 global onosSet
2972 global onosSetName
2973 pCounterName = "TestON-Partitions"
2974 iCounterName = "TestON-inMemory"
2975 pCounterValue = 0
2976 iCounterValue = 0
2977 onosSet = set([])
2978 onosSetName = "TestON-set"
2979
2980 description = "Install Primitives app"
2981 main.case( description )
2982 main.step( "Install Primitives app" )
2983 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002984 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002985 utilities.assert_equals( expect=main.TRUE,
2986 actual=appResults,
2987 onpass="Primitives app activated",
2988 onfail="Primitives app not activated" )
2989 time.sleep( 5 ) # To allow all nodes to activate
2990
2991 def CASE17( self, main ):
2992 """
2993 Check for basic functionality with distributed primitives
2994 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002995 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002996 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002997 assert main, "main not defined"
2998 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002999 assert main.CLIs, "main.CLIs not defined"
3000 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003001 assert pCounterName, "pCounterName not defined"
3002 assert iCounterName, "iCounterName not defined"
3003 assert onosSetName, "onosSetName not defined"
3004 # NOTE: assert fails if value is 0/None/Empty/False
3005 try:
3006 pCounterValue
3007 except NameError:
3008 main.log.error( "pCounterValue not defined, setting to 0" )
3009 pCounterValue = 0
3010 try:
3011 iCounterValue
3012 except NameError:
3013 main.log.error( "iCounterValue not defined, setting to 0" )
3014 iCounterValue = 0
3015 try:
3016 onosSet
3017 except NameError:
3018 main.log.error( "onosSet not defined, setting to empty Set" )
3019 onosSet = set([])
3020 # Variables for the distributed primitives tests. These are local only
3021 addValue = "a"
3022 addAllValue = "a b c d e f"
3023 retainValue = "c d e f"
3024
3025 description = "Check for basic functionality with distributed " +\
3026 "primitives"
3027 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003028 main.caseExplanation = "Test the methods of the distributed " +\
3029 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003030 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003031 # Partitioned counters
3032 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003033 pCounters = []
3034 threads = []
3035 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003036 for i in range( main.numCtrls ):
3037 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3038 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003039 args=[ pCounterName ] )
3040 pCounterValue += 1
3041 addedPValues.append( pCounterValue )
3042 threads.append( t )
3043 t.start()
3044
3045 for t in threads:
3046 t.join()
3047 pCounters.append( t.result )
3048 # Check that counter incremented numController times
3049 pCounterResults = True
3050 for i in addedPValues:
3051 tmpResult = i in pCounters
3052 pCounterResults = pCounterResults and tmpResult
3053 if not tmpResult:
3054 main.log.error( str( i ) + " is not in partitioned "
3055 "counter incremented results" )
3056 utilities.assert_equals( expect=True,
3057 actual=pCounterResults,
3058 onpass="Default counter incremented",
3059 onfail="Error incrementing default" +
3060 " counter" )
3061
Jon Halle1a3b752015-07-22 13:02:46 -07003062 main.step( "Get then Increment a default counter on each node" )
3063 pCounters = []
3064 threads = []
3065 addedPValues = []
3066 for i in range( main.numCtrls ):
3067 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3068 name="counterGetAndAdd-" + str( i ),
3069 args=[ pCounterName ] )
3070 addedPValues.append( pCounterValue )
3071 pCounterValue += 1
3072 threads.append( t )
3073 t.start()
3074
3075 for t in threads:
3076 t.join()
3077 pCounters.append( t.result )
3078 # Check that counter incremented numController times
3079 pCounterResults = True
3080 for i in addedPValues:
3081 tmpResult = i in pCounters
3082 pCounterResults = pCounterResults and tmpResult
3083 if not tmpResult:
3084 main.log.error( str( i ) + " is not in partitioned "
3085 "counter incremented results" )
3086 utilities.assert_equals( expect=True,
3087 actual=pCounterResults,
3088 onpass="Default counter incremented",
3089 onfail="Error incrementing default" +
3090 " counter" )
3091
3092 main.step( "Counters we added have the correct values" )
3093 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3094 utilities.assert_equals( expect=main.TRUE,
3095 actual=incrementCheck,
3096 onpass="Added counters are correct",
3097 onfail="Added counters are incorrect" )
3098
3099 main.step( "Add -8 to then get a default counter on each node" )
3100 pCounters = []
3101 threads = []
3102 addedPValues = []
3103 for i in range( main.numCtrls ):
3104 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3105 name="counterIncrement-" + str( i ),
3106 args=[ pCounterName ],
3107 kwargs={ "delta": -8 } )
3108 pCounterValue += -8
3109 addedPValues.append( pCounterValue )
3110 threads.append( t )
3111 t.start()
3112
3113 for t in threads:
3114 t.join()
3115 pCounters.append( t.result )
3116 # Check that counter incremented numController times
3117 pCounterResults = True
3118 for i in addedPValues:
3119 tmpResult = i in pCounters
3120 pCounterResults = pCounterResults and tmpResult
3121 if not tmpResult:
3122 main.log.error( str( i ) + " is not in partitioned "
3123 "counter incremented results" )
3124 utilities.assert_equals( expect=True,
3125 actual=pCounterResults,
3126 onpass="Default counter incremented",
3127 onfail="Error incrementing default" +
3128 " counter" )
3129
3130 main.step( "Add 5 to then get a default counter on each node" )
3131 pCounters = []
3132 threads = []
3133 addedPValues = []
3134 for i in range( main.numCtrls ):
3135 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3136 name="counterIncrement-" + str( i ),
3137 args=[ pCounterName ],
3138 kwargs={ "delta": 5 } )
3139 pCounterValue += 5
3140 addedPValues.append( pCounterValue )
3141 threads.append( t )
3142 t.start()
3143
3144 for t in threads:
3145 t.join()
3146 pCounters.append( t.result )
3147 # Check that counter incremented numController times
3148 pCounterResults = True
3149 for i in addedPValues:
3150 tmpResult = i in pCounters
3151 pCounterResults = pCounterResults and tmpResult
3152 if not tmpResult:
3153 main.log.error( str( i ) + " is not in partitioned "
3154 "counter incremented results" )
3155 utilities.assert_equals( expect=True,
3156 actual=pCounterResults,
3157 onpass="Default counter incremented",
3158 onfail="Error incrementing default" +
3159 " counter" )
3160
3161 main.step( "Get then add 5 to a default counter on each node" )
3162 pCounters = []
3163 threads = []
3164 addedPValues = []
3165 for i in range( main.numCtrls ):
3166 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3167 name="counterIncrement-" + str( i ),
3168 args=[ pCounterName ],
3169 kwargs={ "delta": 5 } )
3170 addedPValues.append( pCounterValue )
3171 pCounterValue += 5
3172 threads.append( t )
3173 t.start()
3174
3175 for t in threads:
3176 t.join()
3177 pCounters.append( t.result )
3178 # Check that counter incremented numController times
3179 pCounterResults = True
3180 for i in addedPValues:
3181 tmpResult = i in pCounters
3182 pCounterResults = pCounterResults and tmpResult
3183 if not tmpResult:
3184 main.log.error( str( i ) + " is not in partitioned "
3185 "counter incremented results" )
3186 utilities.assert_equals( expect=True,
3187 actual=pCounterResults,
3188 onpass="Default counter incremented",
3189 onfail="Error incrementing default" +
3190 " counter" )
3191
3192 main.step( "Counters we added have the correct values" )
3193 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3194 utilities.assert_equals( expect=main.TRUE,
3195 actual=incrementCheck,
3196 onpass="Added counters are correct",
3197 onfail="Added counters are incorrect" )
3198
3199 # In-Memory counters
3200 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003201 iCounters = []
3202 addedIValues = []
3203 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003204 for i in range( main.numCtrls ):
3205 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003206 name="icounterIncrement-" + str( i ),
3207 args=[ iCounterName ],
3208 kwargs={ "inMemory": True } )
3209 iCounterValue += 1
3210 addedIValues.append( iCounterValue )
3211 threads.append( t )
3212 t.start()
3213
3214 for t in threads:
3215 t.join()
3216 iCounters.append( t.result )
3217 # Check that counter incremented numController times
3218 iCounterResults = True
3219 for i in addedIValues:
3220 tmpResult = i in iCounters
3221 iCounterResults = iCounterResults and tmpResult
3222 if not tmpResult:
3223 main.log.error( str( i ) + " is not in the in-memory "
3224 "counter incremented results" )
3225 utilities.assert_equals( expect=True,
3226 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003227 onpass="In-memory counter incremented",
3228 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003229 " counter" )
3230
Jon Halle1a3b752015-07-22 13:02:46 -07003231 main.step( "Get then Increment a in-memory counter on each node" )
3232 iCounters = []
3233 threads = []
3234 addedIValues = []
3235 for i in range( main.numCtrls ):
3236 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3237 name="counterGetAndAdd-" + str( i ),
3238 args=[ iCounterName ],
3239 kwargs={ "inMemory": True } )
3240 addedIValues.append( iCounterValue )
3241 iCounterValue += 1
3242 threads.append( t )
3243 t.start()
3244
3245 for t in threads:
3246 t.join()
3247 iCounters.append( t.result )
3248 # Check that counter incremented numController times
3249 iCounterResults = True
3250 for i in addedIValues:
3251 tmpResult = i in iCounters
3252 iCounterResults = iCounterResults and tmpResult
3253 if not tmpResult:
3254 main.log.error( str( i ) + " is not in in-memory "
3255 "counter incremented results" )
3256 utilities.assert_equals( expect=True,
3257 actual=iCounterResults,
3258 onpass="In-memory counter incremented",
3259 onfail="Error incrementing in-memory" +
3260 " counter" )
3261
3262 main.step( "Counters we added have the correct values" )
3263 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3264 utilities.assert_equals( expect=main.TRUE,
3265 actual=incrementCheck,
3266 onpass="Added counters are correct",
3267 onfail="Added counters are incorrect" )
3268
3269 main.step( "Add -8 to then get a in-memory counter on each node" )
3270 iCounters = []
3271 threads = []
3272 addedIValues = []
3273 for i in range( main.numCtrls ):
3274 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3275 name="counterIncrement-" + str( i ),
3276 args=[ iCounterName ],
3277 kwargs={ "delta": -8, "inMemory": True } )
3278 iCounterValue += -8
3279 addedIValues.append( iCounterValue )
3280 threads.append( t )
3281 t.start()
3282
3283 for t in threads:
3284 t.join()
3285 iCounters.append( t.result )
3286 # Check that counter incremented numController times
3287 iCounterResults = True
3288 for i in addedIValues:
3289 tmpResult = i in iCounters
3290 iCounterResults = iCounterResults and tmpResult
3291 if not tmpResult:
3292 main.log.error( str( i ) + " is not in in-memory "
3293 "counter incremented results" )
3294 utilities.assert_equals( expect=True,
3295 actual=pCounterResults,
3296 onpass="In-memory counter incremented",
3297 onfail="Error incrementing in-memory" +
3298 " counter" )
3299
3300 main.step( "Add 5 to then get a in-memory counter on each node" )
3301 iCounters = []
3302 threads = []
3303 addedIValues = []
3304 for i in range( main.numCtrls ):
3305 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3306 name="counterIncrement-" + str( i ),
3307 args=[ iCounterName ],
3308 kwargs={ "delta": 5, "inMemory": True } )
3309 iCounterValue += 5
3310 addedIValues.append( iCounterValue )
3311 threads.append( t )
3312 t.start()
3313
3314 for t in threads:
3315 t.join()
3316 iCounters.append( t.result )
3317 # Check that counter incremented numController times
3318 iCounterResults = True
3319 for i in addedIValues:
3320 tmpResult = i in iCounters
3321 iCounterResults = iCounterResults and tmpResult
3322 if not tmpResult:
3323 main.log.error( str( i ) + " is not in in-memory "
3324 "counter incremented results" )
3325 utilities.assert_equals( expect=True,
3326 actual=pCounterResults,
3327 onpass="In-memory counter incremented",
3328 onfail="Error incrementing in-memory" +
3329 " counter" )
3330
3331 main.step( "Get then add 5 to a in-memory counter on each node" )
3332 iCounters = []
3333 threads = []
3334 addedIValues = []
3335 for i in range( main.numCtrls ):
3336 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3337 name="counterIncrement-" + str( i ),
3338 args=[ iCounterName ],
3339 kwargs={ "delta": 5, "inMemory": True } )
3340 addedIValues.append( iCounterValue )
3341 iCounterValue += 5
3342 threads.append( t )
3343 t.start()
3344
3345 for t in threads:
3346 t.join()
3347 iCounters.append( t.result )
3348 # Check that counter incremented numController times
3349 iCounterResults = True
3350 for i in addedIValues:
3351 tmpResult = i in iCounters
3352 iCounterResults = iCounterResults and tmpResult
3353 if not tmpResult:
3354 main.log.error( str( i ) + " is not in in-memory "
3355 "counter incremented results" )
3356 utilities.assert_equals( expect=True,
3357 actual=iCounterResults,
3358 onpass="In-memory counter incremented",
3359 onfail="Error incrementing in-memory" +
3360 " counter" )
3361
3362 main.step( "Counters we added have the correct values" )
3363 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3364 utilities.assert_equals( expect=main.TRUE,
3365 actual=incrementCheck,
3366 onpass="Added counters are correct",
3367 onfail="Added counters are incorrect" )
3368
Jon Hall5cf14d52015-07-16 12:15:19 -07003369 main.step( "Check counters are consistant across nodes" )
Jon Hall57b50432015-10-22 10:20:10 -07003370 onosCounters, consistentCounterResults = main.Counters.consistentCheck()
Jon Hall5cf14d52015-07-16 12:15:19 -07003371 utilities.assert_equals( expect=main.TRUE,
3372 actual=consistentCounterResults,
3373 onpass="ONOS counters are consistent " +
3374 "across nodes",
3375 onfail="ONOS Counters are inconsistent " +
3376 "across nodes" )
3377
3378 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003379 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3380 incrementCheck = incrementCheck and \
3381 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003382 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003383 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003384 onpass="Added counters are correct",
3385 onfail="Added counters are incorrect" )
3386 # DISTRIBUTED SETS
3387 main.step( "Distributed Set get" )
3388 size = len( onosSet )
3389 getResponses = []
3390 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003391 for i in range( main.numCtrls ):
3392 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003393 name="setTestGet-" + str( i ),
3394 args=[ onosSetName ] )
3395 threads.append( t )
3396 t.start()
3397 for t in threads:
3398 t.join()
3399 getResponses.append( t.result )
3400
3401 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003402 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003403 if isinstance( getResponses[ i ], list):
3404 current = set( getResponses[ i ] )
3405 if len( current ) == len( getResponses[ i ] ):
3406 # no repeats
3407 if onosSet != current:
3408 main.log.error( "ONOS" + str( i + 1 ) +
3409 " has incorrect view" +
3410 " of set " + onosSetName + ":\n" +
3411 str( getResponses[ i ] ) )
3412 main.log.debug( "Expected: " + str( onosSet ) )
3413 main.log.debug( "Actual: " + str( current ) )
3414 getResults = main.FALSE
3415 else:
3416 # error, set is not a set
3417 main.log.error( "ONOS" + str( i + 1 ) +
3418 " has repeat elements in" +
3419 " set " + onosSetName + ":\n" +
3420 str( getResponses[ i ] ) )
3421 getResults = main.FALSE
3422 elif getResponses[ i ] == main.ERROR:
3423 getResults = main.FALSE
3424 utilities.assert_equals( expect=main.TRUE,
3425 actual=getResults,
3426 onpass="Set elements are correct",
3427 onfail="Set elements are incorrect" )
3428
3429 main.step( "Distributed Set size" )
3430 sizeResponses = []
3431 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003432 for i in range( main.numCtrls ):
3433 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003434 name="setTestSize-" + str( i ),
3435 args=[ onosSetName ] )
3436 threads.append( t )
3437 t.start()
3438 for t in threads:
3439 t.join()
3440 sizeResponses.append( t.result )
3441
3442 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003443 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003444 if size != sizeResponses[ i ]:
3445 sizeResults = main.FALSE
3446 main.log.error( "ONOS" + str( i + 1 ) +
3447 " expected a size of " + str( size ) +
3448 " for set " + onosSetName +
3449 " but got " + str( sizeResponses[ i ] ) )
3450 utilities.assert_equals( expect=main.TRUE,
3451 actual=sizeResults,
3452 onpass="Set sizes are correct",
3453 onfail="Set sizes are incorrect" )
3454
3455 main.step( "Distributed Set add()" )
3456 onosSet.add( addValue )
3457 addResponses = []
3458 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003459 for i in range( main.numCtrls ):
3460 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003461 name="setTestAdd-" + str( i ),
3462 args=[ onosSetName, addValue ] )
3463 threads.append( t )
3464 t.start()
3465 for t in threads:
3466 t.join()
3467 addResponses.append( t.result )
3468
3469 # main.TRUE = successfully changed the set
3470 # main.FALSE = action resulted in no change in set
3471 # main.ERROR - Some error in executing the function
3472 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003473 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003474 if addResponses[ i ] == main.TRUE:
3475 # All is well
3476 pass
3477 elif addResponses[ i ] == main.FALSE:
3478 # Already in set, probably fine
3479 pass
3480 elif addResponses[ i ] == main.ERROR:
3481 # Error in execution
3482 addResults = main.FALSE
3483 else:
3484 # unexpected result
3485 addResults = main.FALSE
3486 if addResults != main.TRUE:
3487 main.log.error( "Error executing set add" )
3488
3489 # Check if set is still correct
3490 size = len( onosSet )
3491 getResponses = []
3492 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003493 for i in range( main.numCtrls ):
3494 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003495 name="setTestGet-" + str( i ),
3496 args=[ onosSetName ] )
3497 threads.append( t )
3498 t.start()
3499 for t in threads:
3500 t.join()
3501 getResponses.append( t.result )
3502 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003503 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003504 if isinstance( getResponses[ i ], list):
3505 current = set( getResponses[ i ] )
3506 if len( current ) == len( getResponses[ i ] ):
3507 # no repeats
3508 if onosSet != current:
3509 main.log.error( "ONOS" + str( i + 1 ) +
3510 " has incorrect view" +
3511 " of set " + onosSetName + ":\n" +
3512 str( getResponses[ i ] ) )
3513 main.log.debug( "Expected: " + str( onosSet ) )
3514 main.log.debug( "Actual: " + str( current ) )
3515 getResults = main.FALSE
3516 else:
3517 # error, set is not a set
3518 main.log.error( "ONOS" + str( i + 1 ) +
3519 " has repeat elements in" +
3520 " set " + onosSetName + ":\n" +
3521 str( getResponses[ i ] ) )
3522 getResults = main.FALSE
3523 elif getResponses[ i ] == main.ERROR:
3524 getResults = main.FALSE
3525 sizeResponses = []
3526 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003527 for i in range( main.numCtrls ):
3528 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003529 name="setTestSize-" + str( i ),
3530 args=[ onosSetName ] )
3531 threads.append( t )
3532 t.start()
3533 for t in threads:
3534 t.join()
3535 sizeResponses.append( t.result )
3536 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003537 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003538 if size != sizeResponses[ i ]:
3539 sizeResults = main.FALSE
3540 main.log.error( "ONOS" + str( i + 1 ) +
3541 " expected a size of " + str( size ) +
3542 " for set " + onosSetName +
3543 " but got " + str( sizeResponses[ i ] ) )
3544 addResults = addResults and getResults and sizeResults
3545 utilities.assert_equals( expect=main.TRUE,
3546 actual=addResults,
3547 onpass="Set add correct",
3548 onfail="Set add was incorrect" )
3549
3550 main.step( "Distributed Set addAll()" )
3551 onosSet.update( addAllValue.split() )
3552 addResponses = []
3553 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003554 for i in range( main.numCtrls ):
3555 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003556 name="setTestAddAll-" + str( i ),
3557 args=[ onosSetName, addAllValue ] )
3558 threads.append( t )
3559 t.start()
3560 for t in threads:
3561 t.join()
3562 addResponses.append( t.result )
3563
3564 # main.TRUE = successfully changed the set
3565 # main.FALSE = action resulted in no change in set
3566 # main.ERROR - Some error in executing the function
3567 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003568 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003569 if addResponses[ i ] == main.TRUE:
3570 # All is well
3571 pass
3572 elif addResponses[ i ] == main.FALSE:
3573 # Already in set, probably fine
3574 pass
3575 elif addResponses[ i ] == main.ERROR:
3576 # Error in execution
3577 addAllResults = main.FALSE
3578 else:
3579 # unexpected result
3580 addAllResults = main.FALSE
3581 if addAllResults != main.TRUE:
3582 main.log.error( "Error executing set addAll" )
3583
3584 # Check if set is still correct
3585 size = len( onosSet )
3586 getResponses = []
3587 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003588 for i in range( main.numCtrls ):
3589 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003590 name="setTestGet-" + str( i ),
3591 args=[ onosSetName ] )
3592 threads.append( t )
3593 t.start()
3594 for t in threads:
3595 t.join()
3596 getResponses.append( t.result )
3597 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003598 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003599 if isinstance( getResponses[ i ], list):
3600 current = set( getResponses[ i ] )
3601 if len( current ) == len( getResponses[ i ] ):
3602 # no repeats
3603 if onosSet != current:
3604 main.log.error( "ONOS" + str( i + 1 ) +
3605 " has incorrect view" +
3606 " of set " + onosSetName + ":\n" +
3607 str( getResponses[ i ] ) )
3608 main.log.debug( "Expected: " + str( onosSet ) )
3609 main.log.debug( "Actual: " + str( current ) )
3610 getResults = main.FALSE
3611 else:
3612 # error, set is not a set
3613 main.log.error( "ONOS" + str( i + 1 ) +
3614 " has repeat elements in" +
3615 " set " + onosSetName + ":\n" +
3616 str( getResponses[ i ] ) )
3617 getResults = main.FALSE
3618 elif getResponses[ i ] == main.ERROR:
3619 getResults = main.FALSE
3620 sizeResponses = []
3621 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003622 for i in range( main.numCtrls ):
3623 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003624 name="setTestSize-" + str( i ),
3625 args=[ onosSetName ] )
3626 threads.append( t )
3627 t.start()
3628 for t in threads:
3629 t.join()
3630 sizeResponses.append( t.result )
3631 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003632 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003633 if size != sizeResponses[ i ]:
3634 sizeResults = main.FALSE
3635 main.log.error( "ONOS" + str( i + 1 ) +
3636 " expected a size of " + str( size ) +
3637 " for set " + onosSetName +
3638 " but got " + str( sizeResponses[ i ] ) )
3639 addAllResults = addAllResults and getResults and sizeResults
3640 utilities.assert_equals( expect=main.TRUE,
3641 actual=addAllResults,
3642 onpass="Set addAll correct",
3643 onfail="Set addAll was incorrect" )
3644
3645 main.step( "Distributed Set contains()" )
3646 containsResponses = []
3647 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003648 for i in range( main.numCtrls ):
3649 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003650 name="setContains-" + str( i ),
3651 args=[ onosSetName ],
3652 kwargs={ "values": addValue } )
3653 threads.append( t )
3654 t.start()
3655 for t in threads:
3656 t.join()
3657 # NOTE: This is the tuple
3658 containsResponses.append( t.result )
3659
3660 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003661 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003662 if containsResponses[ i ] == main.ERROR:
3663 containsResults = main.FALSE
3664 else:
3665 containsResults = containsResults and\
3666 containsResponses[ i ][ 1 ]
3667 utilities.assert_equals( expect=main.TRUE,
3668 actual=containsResults,
3669 onpass="Set contains is functional",
3670 onfail="Set contains failed" )
3671
3672 main.step( "Distributed Set containsAll()" )
3673 containsAllResponses = []
3674 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003675 for i in range( main.numCtrls ):
3676 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003677 name="setContainsAll-" + str( i ),
3678 args=[ onosSetName ],
3679 kwargs={ "values": addAllValue } )
3680 threads.append( t )
3681 t.start()
3682 for t in threads:
3683 t.join()
3684 # NOTE: This is the tuple
3685 containsAllResponses.append( t.result )
3686
3687 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003688 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003689 if containsResponses[ i ] == main.ERROR:
3690 containsResults = main.FALSE
3691 else:
3692 containsResults = containsResults and\
3693 containsResponses[ i ][ 1 ]
3694 utilities.assert_equals( expect=main.TRUE,
3695 actual=containsAllResults,
3696 onpass="Set containsAll is functional",
3697 onfail="Set containsAll failed" )
3698
3699 main.step( "Distributed Set remove()" )
3700 onosSet.remove( addValue )
3701 removeResponses = []
3702 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003703 for i in range( main.numCtrls ):
3704 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003705 name="setTestRemove-" + str( i ),
3706 args=[ onosSetName, addValue ] )
3707 threads.append( t )
3708 t.start()
3709 for t in threads:
3710 t.join()
3711 removeResponses.append( t.result )
3712
3713 # main.TRUE = successfully changed the set
3714 # main.FALSE = action resulted in no change in set
3715 # main.ERROR - Some error in executing the function
3716 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003717 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003718 if removeResponses[ i ] == main.TRUE:
3719 # All is well
3720 pass
3721 elif removeResponses[ i ] == main.FALSE:
3722 # not in set, probably fine
3723 pass
3724 elif removeResponses[ i ] == main.ERROR:
3725 # Error in execution
3726 removeResults = main.FALSE
3727 else:
3728 # unexpected result
3729 removeResults = main.FALSE
3730 if removeResults != main.TRUE:
3731 main.log.error( "Error executing set remove" )
3732
3733 # Check if set is still correct
3734 size = len( onosSet )
3735 getResponses = []
3736 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003737 for i in range( main.numCtrls ):
3738 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003739 name="setTestGet-" + str( i ),
3740 args=[ onosSetName ] )
3741 threads.append( t )
3742 t.start()
3743 for t in threads:
3744 t.join()
3745 getResponses.append( t.result )
3746 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003747 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003748 if isinstance( getResponses[ i ], list):
3749 current = set( getResponses[ i ] )
3750 if len( current ) == len( getResponses[ i ] ):
3751 # no repeats
3752 if onosSet != current:
3753 main.log.error( "ONOS" + str( i + 1 ) +
3754 " has incorrect view" +
3755 " of set " + onosSetName + ":\n" +
3756 str( getResponses[ i ] ) )
3757 main.log.debug( "Expected: " + str( onosSet ) )
3758 main.log.debug( "Actual: " + str( current ) )
3759 getResults = main.FALSE
3760 else:
3761 # error, set is not a set
3762 main.log.error( "ONOS" + str( i + 1 ) +
3763 " has repeat elements in" +
3764 " set " + onosSetName + ":\n" +
3765 str( getResponses[ i ] ) )
3766 getResults = main.FALSE
3767 elif getResponses[ i ] == main.ERROR:
3768 getResults = main.FALSE
3769 sizeResponses = []
3770 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003771 for i in range( main.numCtrls ):
3772 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003773 name="setTestSize-" + str( i ),
3774 args=[ onosSetName ] )
3775 threads.append( t )
3776 t.start()
3777 for t in threads:
3778 t.join()
3779 sizeResponses.append( t.result )
3780 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003781 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003782 if size != sizeResponses[ i ]:
3783 sizeResults = main.FALSE
3784 main.log.error( "ONOS" + str( i + 1 ) +
3785 " expected a size of " + str( size ) +
3786 " for set " + onosSetName +
3787 " but got " + str( sizeResponses[ i ] ) )
3788 removeResults = removeResults and getResults and sizeResults
3789 utilities.assert_equals( expect=main.TRUE,
3790 actual=removeResults,
3791 onpass="Set remove correct",
3792 onfail="Set remove was incorrect" )
3793
3794 main.step( "Distributed Set removeAll()" )
3795 onosSet.difference_update( addAllValue.split() )
3796 removeAllResponses = []
3797 threads = []
3798 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003799 for i in range( main.numCtrls ):
3800 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003801 name="setTestRemoveAll-" + str( i ),
3802 args=[ onosSetName, addAllValue ] )
3803 threads.append( t )
3804 t.start()
3805 for t in threads:
3806 t.join()
3807 removeAllResponses.append( t.result )
3808 except Exception, e:
3809 main.log.exception(e)
3810
3811 # main.TRUE = successfully changed the set
3812 # main.FALSE = action resulted in no change in set
3813 # main.ERROR - Some error in executing the function
3814 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003815 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003816 if removeAllResponses[ i ] == main.TRUE:
3817 # All is well
3818 pass
3819 elif removeAllResponses[ i ] == main.FALSE:
3820 # not in set, probably fine
3821 pass
3822 elif removeAllResponses[ i ] == main.ERROR:
3823 # Error in execution
3824 removeAllResults = main.FALSE
3825 else:
3826 # unexpected result
3827 removeAllResults = main.FALSE
3828 if removeAllResults != main.TRUE:
3829 main.log.error( "Error executing set removeAll" )
3830
3831 # Check if set is still correct
3832 size = len( onosSet )
3833 getResponses = []
3834 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003835 for i in range( main.numCtrls ):
3836 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003837 name="setTestGet-" + str( i ),
3838 args=[ onosSetName ] )
3839 threads.append( t )
3840 t.start()
3841 for t in threads:
3842 t.join()
3843 getResponses.append( t.result )
3844 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003845 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003846 if isinstance( getResponses[ i ], list):
3847 current = set( getResponses[ i ] )
3848 if len( current ) == len( getResponses[ i ] ):
3849 # no repeats
3850 if onosSet != current:
3851 main.log.error( "ONOS" + str( i + 1 ) +
3852 " has incorrect view" +
3853 " of set " + onosSetName + ":\n" +
3854 str( getResponses[ i ] ) )
3855 main.log.debug( "Expected: " + str( onosSet ) )
3856 main.log.debug( "Actual: " + str( current ) )
3857 getResults = main.FALSE
3858 else:
3859 # error, set is not a set
3860 main.log.error( "ONOS" + str( i + 1 ) +
3861 " has repeat elements in" +
3862 " set " + onosSetName + ":\n" +
3863 str( getResponses[ i ] ) )
3864 getResults = main.FALSE
3865 elif getResponses[ i ] == main.ERROR:
3866 getResults = main.FALSE
3867 sizeResponses = []
3868 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003869 for i in range( main.numCtrls ):
3870 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003871 name="setTestSize-" + str( i ),
3872 args=[ onosSetName ] )
3873 threads.append( t )
3874 t.start()
3875 for t in threads:
3876 t.join()
3877 sizeResponses.append( t.result )
3878 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003879 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003880 if size != sizeResponses[ i ]:
3881 sizeResults = main.FALSE
3882 main.log.error( "ONOS" + str( i + 1 ) +
3883 " expected a size of " + str( size ) +
3884 " for set " + onosSetName +
3885 " but got " + str( sizeResponses[ i ] ) )
3886 removeAllResults = removeAllResults and getResults and sizeResults
3887 utilities.assert_equals( expect=main.TRUE,
3888 actual=removeAllResults,
3889 onpass="Set removeAll correct",
3890 onfail="Set removeAll was incorrect" )
3891
3892 main.step( "Distributed Set addAll()" )
3893 onosSet.update( addAllValue.split() )
3894 addResponses = []
3895 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003896 for i in range( main.numCtrls ):
3897 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003898 name="setTestAddAll-" + str( i ),
3899 args=[ onosSetName, addAllValue ] )
3900 threads.append( t )
3901 t.start()
3902 for t in threads:
3903 t.join()
3904 addResponses.append( t.result )
3905
3906 # main.TRUE = successfully changed the set
3907 # main.FALSE = action resulted in no change in set
3908 # main.ERROR - Some error in executing the function
3909 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003910 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003911 if addResponses[ i ] == main.TRUE:
3912 # All is well
3913 pass
3914 elif addResponses[ i ] == main.FALSE:
3915 # Already in set, probably fine
3916 pass
3917 elif addResponses[ i ] == main.ERROR:
3918 # Error in execution
3919 addAllResults = main.FALSE
3920 else:
3921 # unexpected result
3922 addAllResults = main.FALSE
3923 if addAllResults != main.TRUE:
3924 main.log.error( "Error executing set addAll" )
3925
3926 # Check if set is still correct
3927 size = len( onosSet )
3928 getResponses = []
3929 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003930 for i in range( main.numCtrls ):
3931 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003932 name="setTestGet-" + str( i ),
3933 args=[ onosSetName ] )
3934 threads.append( t )
3935 t.start()
3936 for t in threads:
3937 t.join()
3938 getResponses.append( t.result )
3939 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003940 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003941 if isinstance( getResponses[ i ], list):
3942 current = set( getResponses[ i ] )
3943 if len( current ) == len( getResponses[ i ] ):
3944 # no repeats
3945 if onosSet != current:
3946 main.log.error( "ONOS" + str( i + 1 ) +
3947 " has incorrect view" +
3948 " of set " + onosSetName + ":\n" +
3949 str( getResponses[ i ] ) )
3950 main.log.debug( "Expected: " + str( onosSet ) )
3951 main.log.debug( "Actual: " + str( current ) )
3952 getResults = main.FALSE
3953 else:
3954 # error, set is not a set
3955 main.log.error( "ONOS" + str( i + 1 ) +
3956 " has repeat elements in" +
3957 " set " + onosSetName + ":\n" +
3958 str( getResponses[ i ] ) )
3959 getResults = main.FALSE
3960 elif getResponses[ i ] == main.ERROR:
3961 getResults = main.FALSE
3962 sizeResponses = []
3963 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003964 for i in range( main.numCtrls ):
3965 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003966 name="setTestSize-" + str( i ),
3967 args=[ onosSetName ] )
3968 threads.append( t )
3969 t.start()
3970 for t in threads:
3971 t.join()
3972 sizeResponses.append( t.result )
3973 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003974 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003975 if size != sizeResponses[ i ]:
3976 sizeResults = main.FALSE
3977 main.log.error( "ONOS" + str( i + 1 ) +
3978 " expected a size of " + str( size ) +
3979 " for set " + onosSetName +
3980 " but got " + str( sizeResponses[ i ] ) )
3981 addAllResults = addAllResults and getResults and sizeResults
3982 utilities.assert_equals( expect=main.TRUE,
3983 actual=addAllResults,
3984 onpass="Set addAll correct",
3985 onfail="Set addAll was incorrect" )
3986
3987 main.step( "Distributed Set clear()" )
3988 onosSet.clear()
3989 clearResponses = []
3990 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003991 for i in range( main.numCtrls ):
3992 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003993 name="setTestClear-" + str( i ),
3994 args=[ onosSetName, " "], # Values doesn't matter
3995 kwargs={ "clear": True } )
3996 threads.append( t )
3997 t.start()
3998 for t in threads:
3999 t.join()
4000 clearResponses.append( t.result )
4001
4002 # main.TRUE = successfully changed the set
4003 # main.FALSE = action resulted in no change in set
4004 # main.ERROR - Some error in executing the function
4005 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004006 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004007 if clearResponses[ i ] == main.TRUE:
4008 # All is well
4009 pass
4010 elif clearResponses[ i ] == main.FALSE:
4011 # Nothing set, probably fine
4012 pass
4013 elif clearResponses[ i ] == main.ERROR:
4014 # Error in execution
4015 clearResults = main.FALSE
4016 else:
4017 # unexpected result
4018 clearResults = main.FALSE
4019 if clearResults != main.TRUE:
4020 main.log.error( "Error executing set clear" )
4021
4022 # Check if set is still correct
4023 size = len( onosSet )
4024 getResponses = []
4025 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004026 for i in range( main.numCtrls ):
4027 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004028 name="setTestGet-" + str( i ),
4029 args=[ onosSetName ] )
4030 threads.append( t )
4031 t.start()
4032 for t in threads:
4033 t.join()
4034 getResponses.append( t.result )
4035 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004036 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004037 if isinstance( getResponses[ i ], list):
4038 current = set( getResponses[ i ] )
4039 if len( current ) == len( getResponses[ i ] ):
4040 # no repeats
4041 if onosSet != current:
4042 main.log.error( "ONOS" + str( i + 1 ) +
4043 " has incorrect view" +
4044 " of set " + onosSetName + ":\n" +
4045 str( getResponses[ i ] ) )
4046 main.log.debug( "Expected: " + str( onosSet ) )
4047 main.log.debug( "Actual: " + str( current ) )
4048 getResults = main.FALSE
4049 else:
4050 # error, set is not a set
4051 main.log.error( "ONOS" + str( i + 1 ) +
4052 " has repeat elements in" +
4053 " set " + onosSetName + ":\n" +
4054 str( getResponses[ i ] ) )
4055 getResults = main.FALSE
4056 elif getResponses[ i ] == main.ERROR:
4057 getResults = main.FALSE
4058 sizeResponses = []
4059 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004060 for i in range( main.numCtrls ):
4061 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004062 name="setTestSize-" + str( i ),
4063 args=[ onosSetName ] )
4064 threads.append( t )
4065 t.start()
4066 for t in threads:
4067 t.join()
4068 sizeResponses.append( t.result )
4069 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004070 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004071 if size != sizeResponses[ i ]:
4072 sizeResults = main.FALSE
4073 main.log.error( "ONOS" + str( i + 1 ) +
4074 " expected a size of " + str( size ) +
4075 " for set " + onosSetName +
4076 " but got " + str( sizeResponses[ i ] ) )
4077 clearResults = clearResults and getResults and sizeResults
4078 utilities.assert_equals( expect=main.TRUE,
4079 actual=clearResults,
4080 onpass="Set clear correct",
4081 onfail="Set clear was incorrect" )
4082
4083 main.step( "Distributed Set addAll()" )
4084 onosSet.update( addAllValue.split() )
4085 addResponses = []
4086 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004087 for i in range( main.numCtrls ):
4088 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004089 name="setTestAddAll-" + str( i ),
4090 args=[ onosSetName, addAllValue ] )
4091 threads.append( t )
4092 t.start()
4093 for t in threads:
4094 t.join()
4095 addResponses.append( t.result )
4096
4097 # main.TRUE = successfully changed the set
4098 # main.FALSE = action resulted in no change in set
4099 # main.ERROR - Some error in executing the function
4100 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004101 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004102 if addResponses[ i ] == main.TRUE:
4103 # All is well
4104 pass
4105 elif addResponses[ i ] == main.FALSE:
4106 # Already in set, probably fine
4107 pass
4108 elif addResponses[ i ] == main.ERROR:
4109 # Error in execution
4110 addAllResults = main.FALSE
4111 else:
4112 # unexpected result
4113 addAllResults = main.FALSE
4114 if addAllResults != main.TRUE:
4115 main.log.error( "Error executing set addAll" )
4116
4117 # Check if set is still correct
4118 size = len( onosSet )
4119 getResponses = []
4120 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004121 for i in range( main.numCtrls ):
4122 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004123 name="setTestGet-" + str( i ),
4124 args=[ onosSetName ] )
4125 threads.append( t )
4126 t.start()
4127 for t in threads:
4128 t.join()
4129 getResponses.append( t.result )
4130 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004131 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004132 if isinstance( getResponses[ i ], list):
4133 current = set( getResponses[ i ] )
4134 if len( current ) == len( getResponses[ i ] ):
4135 # no repeats
4136 if onosSet != current:
4137 main.log.error( "ONOS" + str( i + 1 ) +
4138 " has incorrect view" +
4139 " of set " + onosSetName + ":\n" +
4140 str( getResponses[ i ] ) )
4141 main.log.debug( "Expected: " + str( onosSet ) )
4142 main.log.debug( "Actual: " + str( current ) )
4143 getResults = main.FALSE
4144 else:
4145 # error, set is not a set
4146 main.log.error( "ONOS" + str( i + 1 ) +
4147 " has repeat elements in" +
4148 " set " + onosSetName + ":\n" +
4149 str( getResponses[ i ] ) )
4150 getResults = main.FALSE
4151 elif getResponses[ i ] == main.ERROR:
4152 getResults = main.FALSE
4153 sizeResponses = []
4154 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004155 for i in range( main.numCtrls ):
4156 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004157 name="setTestSize-" + str( i ),
4158 args=[ onosSetName ] )
4159 threads.append( t )
4160 t.start()
4161 for t in threads:
4162 t.join()
4163 sizeResponses.append( t.result )
4164 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004165 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004166 if size != sizeResponses[ i ]:
4167 sizeResults = main.FALSE
4168 main.log.error( "ONOS" + str( i + 1 ) +
4169 " expected a size of " + str( size ) +
4170 " for set " + onosSetName +
4171 " but got " + str( sizeResponses[ i ] ) )
4172 addAllResults = addAllResults and getResults and sizeResults
4173 utilities.assert_equals( expect=main.TRUE,
4174 actual=addAllResults,
4175 onpass="Set addAll correct",
4176 onfail="Set addAll was incorrect" )
4177
4178 main.step( "Distributed Set retain()" )
4179 onosSet.intersection_update( retainValue.split() )
4180 retainResponses = []
4181 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004182 for i in range( main.numCtrls ):
4183 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004184 name="setTestRetain-" + str( i ),
4185 args=[ onosSetName, retainValue ],
4186 kwargs={ "retain": True } )
4187 threads.append( t )
4188 t.start()
4189 for t in threads:
4190 t.join()
4191 retainResponses.append( t.result )
4192
4193 # main.TRUE = successfully changed the set
4194 # main.FALSE = action resulted in no change in set
4195 # main.ERROR - Some error in executing the function
4196 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004197 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004198 if retainResponses[ i ] == main.TRUE:
4199 # All is well
4200 pass
4201 elif retainResponses[ i ] == main.FALSE:
4202 # Already in set, probably fine
4203 pass
4204 elif retainResponses[ i ] == main.ERROR:
4205 # Error in execution
4206 retainResults = main.FALSE
4207 else:
4208 # unexpected result
4209 retainResults = main.FALSE
4210 if retainResults != main.TRUE:
4211 main.log.error( "Error executing set retain" )
4212
4213 # Check if set is still correct
4214 size = len( onosSet )
4215 getResponses = []
4216 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004217 for i in range( main.numCtrls ):
4218 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004219 name="setTestGet-" + str( i ),
4220 args=[ onosSetName ] )
4221 threads.append( t )
4222 t.start()
4223 for t in threads:
4224 t.join()
4225 getResponses.append( t.result )
4226 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004227 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004228 if isinstance( getResponses[ i ], list):
4229 current = set( getResponses[ i ] )
4230 if len( current ) == len( getResponses[ i ] ):
4231 # no repeats
4232 if onosSet != current:
4233 main.log.error( "ONOS" + str( i + 1 ) +
4234 " has incorrect view" +
4235 " of set " + onosSetName + ":\n" +
4236 str( getResponses[ i ] ) )
4237 main.log.debug( "Expected: " + str( onosSet ) )
4238 main.log.debug( "Actual: " + str( current ) )
4239 getResults = main.FALSE
4240 else:
4241 # error, set is not a set
4242 main.log.error( "ONOS" + str( i + 1 ) +
4243 " has repeat elements in" +
4244 " set " + onosSetName + ":\n" +
4245 str( getResponses[ i ] ) )
4246 getResults = main.FALSE
4247 elif getResponses[ i ] == main.ERROR:
4248 getResults = main.FALSE
4249 sizeResponses = []
4250 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004251 for i in range( main.numCtrls ):
4252 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004253 name="setTestSize-" + str( i ),
4254 args=[ onosSetName ] )
4255 threads.append( t )
4256 t.start()
4257 for t in threads:
4258 t.join()
4259 sizeResponses.append( t.result )
4260 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004261 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004262 if size != sizeResponses[ i ]:
4263 sizeResults = main.FALSE
4264 main.log.error( "ONOS" + str( i + 1 ) +
4265 " expected a size of " +
4266 str( size ) + " for set " + onosSetName +
4267 " but got " + str( sizeResponses[ i ] ) )
4268 retainResults = retainResults and getResults and sizeResults
4269 utilities.assert_equals( expect=main.TRUE,
4270 actual=retainResults,
4271 onpass="Set retain correct",
4272 onfail="Set retain was incorrect" )
4273
Jon Hall2a5002c2015-08-21 16:49:11 -07004274 # Transactional maps
4275 main.step( "Partitioned Transactional maps put" )
4276 tMapValue = "Testing"
4277 numKeys = 100
4278 putResult = True
4279 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4280 if len( putResponses ) == 100:
4281 for i in putResponses:
4282 if putResponses[ i ][ 'value' ] != tMapValue:
4283 putResult = False
4284 else:
4285 putResult = False
4286 if not putResult:
4287 main.log.debug( "Put response values: " + str( putResponses ) )
4288 utilities.assert_equals( expect=True,
4289 actual=putResult,
4290 onpass="Partitioned Transactional Map put successful",
4291 onfail="Partitioned Transactional Map put values are incorrect" )
4292
4293 main.step( "Partitioned Transactional maps get" )
4294 getCheck = True
4295 for n in range( 1, numKeys + 1 ):
4296 getResponses = []
4297 threads = []
4298 valueCheck = True
4299 for i in range( main.numCtrls ):
4300 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4301 name="TMap-get-" + str( i ),
4302 args=[ "Key" + str ( n ) ] )
4303 threads.append( t )
4304 t.start()
4305 for t in threads:
4306 t.join()
4307 getResponses.append( t.result )
4308 for node in getResponses:
4309 if node != tMapValue:
4310 valueCheck = False
4311 if not valueCheck:
4312 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4313 main.log.warn( getResponses )
4314 getCheck = getCheck and valueCheck
4315 utilities.assert_equals( expect=True,
4316 actual=getCheck,
4317 onpass="Partitioned Transactional Map get values were correct",
4318 onfail="Partitioned Transactional Map values incorrect" )
4319
4320 main.step( "In-memory Transactional maps put" )
4321 tMapValue = "Testing"
4322 numKeys = 100
4323 putResult = True
4324 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4325 if len( putResponses ) == 100:
4326 for i in putResponses:
4327 if putResponses[ i ][ 'value' ] != tMapValue:
4328 putResult = False
4329 else:
4330 putResult = False
4331 if not putResult:
4332 main.log.debug( "Put response values: " + str( putResponses ) )
4333 utilities.assert_equals( expect=True,
4334 actual=putResult,
4335 onpass="In-Memory Transactional Map put successful",
4336 onfail="In-Memory Transactional Map put values are incorrect" )
4337
4338 main.step( "In-Memory Transactional maps get" )
4339 getCheck = True
4340 for n in range( 1, numKeys + 1 ):
4341 getResponses = []
4342 threads = []
4343 valueCheck = True
4344 for i in range( main.numCtrls ):
4345 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4346 name="TMap-get-" + str( i ),
4347 args=[ "Key" + str ( n ) ],
4348 kwargs={ "inMemory": True } )
4349 threads.append( t )
4350 t.start()
4351 for t in threads:
4352 t.join()
4353 getResponses.append( t.result )
4354 for node in getResponses:
4355 if node != tMapValue:
4356 valueCheck = False
4357 if not valueCheck:
4358 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4359 main.log.warn( getResponses )
4360 getCheck = getCheck and valueCheck
4361 utilities.assert_equals( expect=True,
4362 actual=getCheck,
4363 onpass="In-Memory Transactional Map get values were correct",
4364 onfail="In-Memory Transactional Map values incorrect" )