blob: 1a5bbfe1f7f71c21a193be484d1271ff51d6350c [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
28class HAsanity:
29
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080052 import time
Jon Hall5cf14d52015-07-16 12:15:19 -070053 main.log.info( "ONOS HA Sanity test - initialization" )
54 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070055 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "installing ONOS, starting Mininet and ONOS" +\
57 "cli sessions."
58 # TODO: save all the timers and output them for plotting
59
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070071 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070072 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070073 global ONOS1Port
74 global ONOS2Port
75 global ONOS3Port
76 global ONOS4Port
77 global ONOS5Port
78 global ONOS6Port
79 global ONOS7Port
80
81 # FIXME: just get controller port from params?
82 # TODO: do we really need all these?
83 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
84 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
85 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
86 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
87 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
88 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
89 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
90
Jon Halle1a3b752015-07-22 13:02:46 -070091 try:
92 fileName = "Counters"
93 path = main.params[ 'imports' ][ 'path' ]
94 main.Counters = imp.load_source( fileName,
95 path + fileName + ".py" )
96 except Exception as e:
97 main.log.exception( e )
98 main.cleanup()
99 main.exit()
100
101 main.CLIs = []
102 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700103 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700104 for i in range( 1, main.numCtrls + 1 ):
105 try:
106 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
107 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
108 ipList.append( main.nodes[ -1 ].ip_address )
109 except AttributeError:
110 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700111
112 main.step( "Create cell file" )
113 cellAppString = main.params[ 'ENV' ][ 'appString' ]
114 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
115 main.Mininet1.ip_address,
116 cellAppString, ipList )
117 main.step( "Applying cell variable to environment" )
118 cellResult = main.ONOSbench.setCell( cellName )
119 verifyResult = main.ONOSbench.verifyCell()
120
121 # FIXME:this is short term fix
122 main.log.info( "Removing raft logs" )
123 main.ONOSbench.onosRemoveRaftLogs()
124
125 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700126 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700127 main.ONOSbench.onosUninstall( node.ip_address )
128
129 # Make sure ONOS is DEAD
130 main.log.info( "Killing any ONOS processes" )
131 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700132 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700133 killed = main.ONOSbench.onosKill( node.ip_address )
134 killResults = killResults and killed
135
136 cleanInstallResult = main.TRUE
137 gitPullResult = main.TRUE
138
139 main.step( "Starting Mininet" )
140 # scp topo file to mininet
141 # TODO: move to params?
142 topoName = "obelisk.py"
143 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700144 main.ONOSbench.scp( main.Mininet1,
145 filePath + topoName,
146 main.Mininet1.home,
147 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700148 mnResult = main.Mininet1.startNet( )
149 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
150 onpass="Mininet Started",
151 onfail="Error starting Mininet" )
152
153 main.step( "Git checkout and pull " + gitBranch )
154 if PULLCODE:
155 main.ONOSbench.gitCheckout( gitBranch )
156 gitPullResult = main.ONOSbench.gitPull()
157 # values of 1 or 3 are good
158 utilities.assert_lesser( expect=0, actual=gitPullResult,
159 onpass="Git pull successful",
160 onfail="Git pull failed" )
161 main.ONOSbench.getVersion( report=True )
162
163 main.step( "Using mvn clean install" )
164 cleanInstallResult = main.TRUE
165 if PULLCODE and gitPullResult == main.TRUE:
166 cleanInstallResult = main.ONOSbench.cleanInstall()
167 else:
168 main.log.warn( "Did not pull new code so skipping mvn " +
169 "clean install" )
170 utilities.assert_equals( expect=main.TRUE,
171 actual=cleanInstallResult,
172 onpass="MCI successful",
173 onfail="MCI failed" )
174 # GRAPHS
175 # NOTE: important params here:
176 # job = name of Jenkins job
177 # Plot Name = Plot-HA, only can be used if multiple plots
178 # index = The number of the graph under plot name
179 job = "HAsanity"
180 plotName = "Plot-HA"
181 graphs = '<ac:structured-macro ac:name="html">\n'
182 graphs += '<ac:plain-text-body><![CDATA[\n'
183 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
184 '/plot/' + plotName + '/getPlot?index=0' +\
185 '&width=500&height=300"' +\
186 'noborder="0" width="500" height="300" scrolling="yes" ' +\
187 'seamless="seamless"></iframe>\n'
188 graphs += ']]></ac:plain-text-body>\n'
189 graphs += '</ac:structured-macro>\n'
190 main.log.wiki(graphs)
191
192 main.step( "Creating ONOS package" )
193 packageResult = main.ONOSbench.onosPackage()
194 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
195 onpass="ONOS package successful",
196 onfail="ONOS package failed" )
197
198 main.step( "Installing ONOS package" )
199 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700200 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700201 tmpResult = main.ONOSbench.onosInstall( options="-f",
202 node=node.ip_address )
203 onosInstallResult = onosInstallResult and tmpResult
204 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
205 onpass="ONOS install successful",
206 onfail="ONOS install failed" )
207
208 main.step( "Checking if ONOS is up yet" )
209 for i in range( 2 ):
210 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700211 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700212 started = main.ONOSbench.isup( node.ip_address )
213 if not started:
214 main.log.error( node.name + " didn't start!" )
215 main.ONOSbench.onosStop( node.ip_address )
216 main.ONOSbench.onosStart( node.ip_address )
217 onosIsupResult = onosIsupResult and started
218 if onosIsupResult == main.TRUE:
219 break
220 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
221 onpass="ONOS startup successful",
222 onfail="ONOS startup failed" )
223
224 main.log.step( "Starting ONOS CLI sessions" )
225 cliResults = main.TRUE
226 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700227 for i in range( main.numCtrls ):
228 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700229 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700230 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700231 threads.append( t )
232 t.start()
233
234 for t in threads:
235 t.join()
236 cliResults = cliResults and t.result
237 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
238 onpass="ONOS cli startup successful",
239 onfail="ONOS cli startup failed" )
240
241 if main.params[ 'tcpdump' ].lower() == "true":
242 main.step( "Start Packet Capture MN" )
243 main.Mininet2.startTcpdump(
244 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
245 + "-MN.pcap",
246 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
247 port=main.params[ 'MNtcpdump' ][ 'port' ] )
248
249 main.step( "App Ids check" )
Jon Hallf3d16e72015-12-16 17:45:08 -0800250 time.sleep(60)
Jon Hall5cf14d52015-07-16 12:15:19 -0700251 appCheck = main.TRUE
252 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700253 for i in range( main.numCtrls ):
254 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700255 name="appToIDCheck-" + str( i ),
256 args=[] )
257 threads.append( t )
258 t.start()
259
260 for t in threads:
261 t.join()
262 appCheck = appCheck and t.result
263 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700264 main.log.warn( main.CLIs[0].apps() )
265 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700266 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
267 onpass="App Ids seem to be correct",
268 onfail="Something is wrong with app Ids" )
269
270 if cliResults == main.FALSE:
271 main.log.error( "Failed to start ONOS, stopping test" )
272 main.cleanup()
273 main.exit()
274
275 def CASE2( self, main ):
276 """
277 Assign devices to controllers
278 """
279 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700280 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700281 assert main, "main not defined"
282 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700283 assert main.CLIs, "main.CLIs not defined"
284 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700285 assert ONOS1Port, "ONOS1Port not defined"
286 assert ONOS2Port, "ONOS2Port not defined"
287 assert ONOS3Port, "ONOS3Port not defined"
288 assert ONOS4Port, "ONOS4Port not defined"
289 assert ONOS5Port, "ONOS5Port not defined"
290 assert ONOS6Port, "ONOS6Port not defined"
291 assert ONOS7Port, "ONOS7Port not defined"
292
293 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700294 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700295 "and check that an ONOS node becomes the " +\
296 "master of the device."
297 main.step( "Assign switches to controllers" )
298
299 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700300 for i in range( main.numCtrls ):
301 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700302 swList = []
303 for i in range( 1, 29 ):
304 swList.append( "s" + str( i ) )
305 main.Mininet1.assignSwController( sw=swList, ip=ipList )
306
307 mastershipCheck = main.TRUE
308 for i in range( 1, 29 ):
309 response = main.Mininet1.getSwController( "s" + str( i ) )
310 try:
311 main.log.info( str( response ) )
312 except Exception:
313 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700314 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700315 if re.search( "tcp:" + node.ip_address, response ):
316 mastershipCheck = mastershipCheck and main.TRUE
317 else:
318 main.log.error( "Error, node " + node.ip_address + " is " +
319 "not in the list of controllers s" +
320 str( i ) + " is connecting to." )
321 mastershipCheck = main.FALSE
322 utilities.assert_equals(
323 expect=main.TRUE,
324 actual=mastershipCheck,
325 onpass="Switch mastership assigned correctly",
326 onfail="Switches not assigned correctly to controllers" )
327
328 def CASE21( self, main ):
329 """
330 Assign mastership to controllers
331 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700332 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700333 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700334 assert main, "main not defined"
335 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700336 assert main.CLIs, "main.CLIs not defined"
337 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700338 assert ONOS1Port, "ONOS1Port not defined"
339 assert ONOS2Port, "ONOS2Port not defined"
340 assert ONOS3Port, "ONOS3Port not defined"
341 assert ONOS4Port, "ONOS4Port not defined"
342 assert ONOS5Port, "ONOS5Port not defined"
343 assert ONOS6Port, "ONOS6Port not defined"
344 assert ONOS7Port, "ONOS7Port not defined"
345
346 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700347 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700348 "device. Then manually assign" +\
349 " mastership to specific ONOS nodes using" +\
350 " 'device-role'"
351 main.step( "Assign mastership of switches to specific controllers" )
352 # Manually assign mastership to the controller we want
353 roleCall = main.TRUE
354
355 ipList = [ ]
356 deviceList = []
357 try:
358 # Assign mastership to specific controllers. This assignment was
359 # determined for a 7 node cluser, but will work with any sized
360 # cluster
361 for i in range( 1, 29 ): # switches 1 through 28
362 # set up correct variables:
363 if i == 1:
364 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700365 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700366 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
367 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700368 c = 1 % main.numCtrls
369 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700370 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
371 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700372 c = 1 % main.numCtrls
373 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700374 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
375 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700376 c = 3 % main.numCtrls
377 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700378 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
379 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700380 c = 2 % main.numCtrls
381 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700382 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
383 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700384 c = 2 % main.numCtrls
385 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700386 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
387 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700388 c = 5 % main.numCtrls
389 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700390 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
391 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700392 c = 4 % main.numCtrls
393 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700394 dpid = '3' + str( i ).zfill( 3 )
395 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
396 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700397 c = 6 % main.numCtrls
398 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700399 dpid = '6' + str( i ).zfill( 3 )
400 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
401 elif i == 28:
402 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700403 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700404 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
405 else:
406 main.log.error( "You didn't write an else statement for " +
407 "switch s" + str( i ) )
408 roleCall = main.FALSE
409 # Assign switch
410 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
411 # TODO: make this controller dynamic
412 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
413 ip )
414 ipList.append( ip )
415 deviceList.append( deviceId )
416 except ( AttributeError, AssertionError ):
417 main.log.exception( "Something is wrong with ONOS device view" )
418 main.log.info( main.ONOScli1.devices() )
419 utilities.assert_equals(
420 expect=main.TRUE,
421 actual=roleCall,
422 onpass="Re-assigned switch mastership to designated controller",
423 onfail="Something wrong with deviceRole calls" )
424
425 main.step( "Check mastership was correctly assigned" )
426 roleCheck = main.TRUE
427 # NOTE: This is due to the fact that device mastership change is not
428 # atomic and is actually a multi step process
429 time.sleep( 5 )
430 for i in range( len( ipList ) ):
431 ip = ipList[i]
432 deviceId = deviceList[i]
433 # Check assignment
434 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
435 if ip in master:
436 roleCheck = roleCheck and main.TRUE
437 else:
438 roleCheck = roleCheck and main.FALSE
439 main.log.error( "Error, controller " + ip + " is not" +
440 " master " + "of device " +
441 str( deviceId ) + ". Master is " +
442 repr( master ) + "." )
443 utilities.assert_equals(
444 expect=main.TRUE,
445 actual=roleCheck,
446 onpass="Switches were successfully reassigned to designated " +
447 "controller",
448 onfail="Switches were not successfully reassigned" )
449
450 def CASE3( self, main ):
451 """
452 Assign intents
453 """
454 import time
455 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700456 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700457 assert main, "main not defined"
458 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700459 assert main.CLIs, "main.CLIs not defined"
460 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700461 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700462 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 "assign predetermined host-to-host intents." +\
464 " After installation, check that the intent" +\
465 " is distributed to all nodes and the state" +\
466 " is INSTALLED"
467
468 # install onos-app-fwd
469 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700470 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700471 utilities.assert_equals( expect=main.TRUE, actual=installResults,
472 onpass="Install fwd successful",
473 onfail="Install fwd failed" )
474
475 main.step( "Check app ids" )
476 appCheck = main.TRUE
477 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700478 for i in range( main.numCtrls ):
479 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700480 name="appToIDCheck-" + str( i ),
481 args=[] )
482 threads.append( t )
483 t.start()
484
485 for t in threads:
486 t.join()
487 appCheck = appCheck and t.result
488 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700489 main.log.warn( main.CLIs[0].apps() )
490 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700491 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
492 onpass="App Ids seem to be correct",
493 onfail="Something is wrong with app Ids" )
494
495 main.step( "Discovering Hosts( Via pingall for now )" )
496 # FIXME: Once we have a host discovery mechanism, use that instead
497 # REACTIVE FWD test
498 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700499 passMsg = "Reactive Pingall test passed"
500 time1 = time.time()
501 pingResult = main.Mininet1.pingall()
502 time2 = time.time()
503 if not pingResult:
504 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700505 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700506 passMsg += " on the second try"
507 utilities.assert_equals(
508 expect=main.TRUE,
509 actual=pingResult,
510 onpass= passMsg,
511 onfail="Reactive Pingall failed, " +
512 "one or more ping pairs failed" )
513 main.log.info( "Time for pingall: %2f seconds" %
514 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700515 # timeout for fwd flows
516 time.sleep( 11 )
517 # uninstall onos-app-fwd
518 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700519 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700520 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
521 onpass="Uninstall fwd successful",
522 onfail="Uninstall fwd failed" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700523
524 main.step( "Check app ids" )
525 threads = []
526 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700527 for i in range( main.numCtrls ):
528 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700529 name="appToIDCheck-" + str( i ),
530 args=[] )
531 threads.append( t )
532 t.start()
533
534 for t in threads:
535 t.join()
536 appCheck2 = appCheck2 and t.result
537 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700538 main.log.warn( main.CLIs[0].apps() )
539 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700540 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
541 onpass="App Ids seem to be correct",
542 onfail="Something is wrong with app Ids" )
543
544 main.step( "Add host intents via cli" )
545 intentIds = []
546 # TODO: move the host numbers to params
547 # Maybe look at all the paths we ping?
548 intentAddResult = True
549 hostResult = main.TRUE
550 for i in range( 8, 18 ):
551 main.log.info( "Adding host intent between h" + str( i ) +
552 " and h" + str( i + 10 ) )
553 host1 = "00:00:00:00:00:" + \
554 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
555 host2 = "00:00:00:00:00:" + \
556 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
557 # NOTE: getHost can return None
558 host1Dict = main.ONOScli1.getHost( host1 )
559 host2Dict = main.ONOScli1.getHost( host2 )
560 host1Id = None
561 host2Id = None
562 if host1Dict and host2Dict:
563 host1Id = host1Dict.get( 'id', None )
564 host2Id = host2Dict.get( 'id', None )
565 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700566 nodeNum = ( i % main.numCtrls )
567 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700568 if tmpId:
569 main.log.info( "Added intent with id: " + tmpId )
570 intentIds.append( tmpId )
571 else:
572 main.log.error( "addHostIntent returned: " +
573 repr( tmpId ) )
574 else:
575 main.log.error( "Error, getHost() failed for h" + str( i ) +
576 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700577 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700578 main.log.warn( "Hosts output: " )
579 try:
580 main.log.warn( json.dumps( json.loads( hosts ),
581 sort_keys=True,
582 indent=4,
583 separators=( ',', ': ' ) ) )
584 except ( ValueError, TypeError ):
585 main.log.warn( repr( hosts ) )
586 hostResult = main.FALSE
587 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
588 onpass="Found a host id for each host",
589 onfail="Error looking up host ids" )
590
591 intentStart = time.time()
592 onosIds = main.ONOScli1.getAllIntentsId()
593 main.log.info( "Submitted intents: " + str( intentIds ) )
594 main.log.info( "Intents in ONOS: " + str( onosIds ) )
595 for intent in intentIds:
596 if intent in onosIds:
597 pass # intent submitted is in onos
598 else:
599 intentAddResult = False
600 if intentAddResult:
601 intentStop = time.time()
602 else:
603 intentStop = None
604 # Print the intent states
605 intents = main.ONOScli1.intents()
606 intentStates = []
607 installedCheck = True
608 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
609 count = 0
610 try:
611 for intent in json.loads( intents ):
612 state = intent.get( 'state', None )
613 if "INSTALLED" not in state:
614 installedCheck = False
615 intentId = intent.get( 'id', None )
616 intentStates.append( ( intentId, state ) )
617 except ( ValueError, TypeError ):
618 main.log.exception( "Error parsing intents" )
619 # add submitted intents not in the store
620 tmplist = [ i for i, s in intentStates ]
621 missingIntents = False
622 for i in intentIds:
623 if i not in tmplist:
624 intentStates.append( ( i, " - " ) )
625 missingIntents = True
626 intentStates.sort()
627 for i, s in intentStates:
628 count += 1
629 main.log.info( "%-6s%-15s%-15s" %
630 ( str( count ), str( i ), str( s ) ) )
631 leaders = main.ONOScli1.leaders()
632 try:
633 missing = False
634 if leaders:
635 parsedLeaders = json.loads( leaders )
636 main.log.warn( json.dumps( parsedLeaders,
637 sort_keys=True,
638 indent=4,
639 separators=( ',', ': ' ) ) )
640 # check for all intent partitions
641 topics = []
642 for i in range( 14 ):
643 topics.append( "intent-partition-" + str( i ) )
644 main.log.debug( topics )
645 ONOStopics = [ j['topic'] for j in parsedLeaders ]
646 for topic in topics:
647 if topic not in ONOStopics:
648 main.log.error( "Error: " + topic +
649 " not in leaders" )
650 missing = True
651 else:
652 main.log.error( "leaders() returned None" )
653 except ( ValueError, TypeError ):
654 main.log.exception( "Error parsing leaders" )
655 main.log.error( repr( leaders ) )
656 # Check all nodes
657 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700658 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700659 response = node.leaders( jsonFormat=False)
660 main.log.warn( str( node.name ) + " leaders output: \n" +
661 str( response ) )
662
663 partitions = main.ONOScli1.partitions()
664 try:
665 if partitions :
666 parsedPartitions = json.loads( partitions )
667 main.log.warn( json.dumps( parsedPartitions,
668 sort_keys=True,
669 indent=4,
670 separators=( ',', ': ' ) ) )
671 # TODO check for a leader in all paritions
672 # TODO check for consistency among nodes
673 else:
674 main.log.error( "partitions() returned None" )
675 except ( ValueError, TypeError ):
676 main.log.exception( "Error parsing partitions" )
677 main.log.error( repr( partitions ) )
678 pendingMap = main.ONOScli1.pendingMap()
679 try:
680 if pendingMap :
681 parsedPending = json.loads( pendingMap )
682 main.log.warn( json.dumps( parsedPending,
683 sort_keys=True,
684 indent=4,
685 separators=( ',', ': ' ) ) )
686 # TODO check something here?
687 else:
688 main.log.error( "pendingMap() returned None" )
689 except ( ValueError, TypeError ):
690 main.log.exception( "Error parsing pending map" )
691 main.log.error( repr( pendingMap ) )
692
693 intentAddResult = bool( intentAddResult and not missingIntents and
694 installedCheck )
695 if not intentAddResult:
696 main.log.error( "Error in pushing host intents to ONOS" )
697
698 main.step( "Intent Anti-Entropy dispersion" )
699 for i in range(100):
700 correct = True
701 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700702 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700703 onosIds = []
704 ids = cli.getAllIntentsId()
705 onosIds.append( ids )
706 main.log.debug( "Intents in " + cli.name + ": " +
707 str( sorted( onosIds ) ) )
708 if sorted( ids ) != sorted( intentIds ):
709 main.log.warn( "Set of intent IDs doesn't match" )
710 correct = False
711 break
712 else:
713 intents = json.loads( cli.intents() )
714 for intent in intents:
715 if intent[ 'state' ] != "INSTALLED":
716 main.log.warn( "Intent " + intent[ 'id' ] +
717 " is " + intent[ 'state' ] )
718 correct = False
719 break
720 if correct:
721 break
722 else:
723 time.sleep(1)
724 if not intentStop:
725 intentStop = time.time()
726 global gossipTime
727 gossipTime = intentStop - intentStart
728 main.log.info( "It took about " + str( gossipTime ) +
729 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700730 gossipPeriod = int( main.params['timers']['gossip'] )
731 maxGossipTime = gossipPeriod * len( main.nodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700732 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700733 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700734 onpass="ECM anti-entropy for intents worked within " +
735 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700736 onfail="Intent ECM anti-entropy took too long. " +
737 "Expected time:{}, Actual time:{}".format( maxGossipTime,
738 gossipTime ) )
739 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700740 intentAddResult = True
741
742 if not intentAddResult or "key" in pendingMap:
743 import time
744 installedCheck = True
745 main.log.info( "Sleeping 60 seconds to see if intents are found" )
746 time.sleep( 60 )
747 onosIds = main.ONOScli1.getAllIntentsId()
748 main.log.info( "Submitted intents: " + str( intentIds ) )
749 main.log.info( "Intents in ONOS: " + str( onosIds ) )
750 # Print the intent states
751 intents = main.ONOScli1.intents()
752 intentStates = []
753 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
754 count = 0
755 try:
756 for intent in json.loads( intents ):
757 # Iter through intents of a node
758 state = intent.get( 'state', None )
759 if "INSTALLED" not in state:
760 installedCheck = False
761 intentId = intent.get( 'id', None )
762 intentStates.append( ( intentId, state ) )
763 except ( ValueError, TypeError ):
764 main.log.exception( "Error parsing intents" )
765 # add submitted intents not in the store
766 tmplist = [ i for i, s in intentStates ]
767 for i in intentIds:
768 if i not in tmplist:
769 intentStates.append( ( i, " - " ) )
770 intentStates.sort()
771 for i, s in intentStates:
772 count += 1
773 main.log.info( "%-6s%-15s%-15s" %
774 ( str( count ), str( i ), str( s ) ) )
775 leaders = main.ONOScli1.leaders()
776 try:
777 missing = False
778 if leaders:
779 parsedLeaders = json.loads( leaders )
780 main.log.warn( json.dumps( parsedLeaders,
781 sort_keys=True,
782 indent=4,
783 separators=( ',', ': ' ) ) )
784 # check for all intent partitions
785 # check for election
786 topics = []
787 for i in range( 14 ):
788 topics.append( "intent-partition-" + str( i ) )
789 # FIXME: this should only be after we start the app
790 topics.append( "org.onosproject.election" )
791 main.log.debug( topics )
792 ONOStopics = [ j['topic'] for j in parsedLeaders ]
793 for topic in topics:
794 if topic not in ONOStopics:
795 main.log.error( "Error: " + topic +
796 " not in leaders" )
797 missing = True
798 else:
799 main.log.error( "leaders() returned None" )
800 except ( ValueError, TypeError ):
801 main.log.exception( "Error parsing leaders" )
802 main.log.error( repr( leaders ) )
803 # Check all nodes
804 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700805 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700806 response = node.leaders( jsonFormat=False)
807 main.log.warn( str( node.name ) + " leaders output: \n" +
808 str( response ) )
809
810 partitions = main.ONOScli1.partitions()
811 try:
812 if partitions :
813 parsedPartitions = json.loads( partitions )
814 main.log.warn( json.dumps( parsedPartitions,
815 sort_keys=True,
816 indent=4,
817 separators=( ',', ': ' ) ) )
818 # TODO check for a leader in all paritions
819 # TODO check for consistency among nodes
820 else:
821 main.log.error( "partitions() returned None" )
822 except ( ValueError, TypeError ):
823 main.log.exception( "Error parsing partitions" )
824 main.log.error( repr( partitions ) )
825 pendingMap = main.ONOScli1.pendingMap()
826 try:
827 if pendingMap :
828 parsedPending = json.loads( pendingMap )
829 main.log.warn( json.dumps( parsedPending,
830 sort_keys=True,
831 indent=4,
832 separators=( ',', ': ' ) ) )
833 # TODO check something here?
834 else:
835 main.log.error( "pendingMap() returned None" )
836 except ( ValueError, TypeError ):
837 main.log.exception( "Error parsing pending map" )
838 main.log.error( repr( pendingMap ) )
839
840 def CASE4( self, main ):
841 """
842 Ping across added host intents
843 """
844 import json
845 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700846 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700847 assert main, "main not defined"
848 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700849 assert main.CLIs, "main.CLIs not defined"
850 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700851 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700852 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700853 "functionality and check the state of " +\
854 "the intent"
855 main.step( "Ping across added host intents" )
856 PingResult = main.TRUE
857 for i in range( 8, 18 ):
858 ping = main.Mininet1.pingHost( src="h" + str( i ),
859 target="h" + str( i + 10 ) )
860 PingResult = PingResult and ping
861 if ping == main.FALSE:
862 main.log.warn( "Ping failed between h" + str( i ) +
863 " and h" + str( i + 10 ) )
864 elif ping == main.TRUE:
865 main.log.info( "Ping test passed!" )
866 # Don't set PingResult or you'd override failures
867 if PingResult == main.FALSE:
868 main.log.error(
869 "Intents have not been installed correctly, pings failed." )
870 # TODO: pretty print
871 main.log.warn( "ONOS1 intents: " )
872 try:
873 tmpIntents = main.ONOScli1.intents()
874 main.log.warn( json.dumps( json.loads( tmpIntents ),
875 sort_keys=True,
876 indent=4,
877 separators=( ',', ': ' ) ) )
878 except ( ValueError, TypeError ):
879 main.log.warn( repr( tmpIntents ) )
880 utilities.assert_equals(
881 expect=main.TRUE,
882 actual=PingResult,
883 onpass="Intents have been installed correctly and pings work",
884 onfail="Intents have not been installed correctly, pings failed." )
885
886 main.step( "Check Intent state" )
887 installedCheck = False
888 loopCount = 0
889 while not installedCheck and loopCount < 40:
890 installedCheck = True
891 # Print the intent states
892 intents = main.ONOScli1.intents()
893 intentStates = []
894 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
895 count = 0
896 # Iter through intents of a node
897 try:
898 for intent in json.loads( intents ):
899 state = intent.get( 'state', None )
900 if "INSTALLED" not in state:
901 installedCheck = False
902 intentId = intent.get( 'id', None )
903 intentStates.append( ( intentId, state ) )
904 except ( ValueError, TypeError ):
905 main.log.exception( "Error parsing intents." )
906 # Print states
907 intentStates.sort()
908 for i, s in intentStates:
909 count += 1
910 main.log.info( "%-6s%-15s%-15s" %
911 ( str( count ), str( i ), str( s ) ) )
912 if not installedCheck:
913 time.sleep( 1 )
914 loopCount += 1
915 utilities.assert_equals( expect=True, actual=installedCheck,
916 onpass="Intents are all INSTALLED",
917 onfail="Intents are not all in " +
918 "INSTALLED state" )
919
920 main.step( "Check leadership of topics" )
921 leaders = main.ONOScli1.leaders()
922 topicCheck = main.TRUE
923 try:
924 if leaders:
925 parsedLeaders = json.loads( leaders )
926 main.log.warn( json.dumps( parsedLeaders,
927 sort_keys=True,
928 indent=4,
929 separators=( ',', ': ' ) ) )
930 # check for all intent partitions
931 # check for election
932 # TODO: Look at Devices as topics now that it uses this system
933 topics = []
934 for i in range( 14 ):
935 topics.append( "intent-partition-" + str( i ) )
936 # FIXME: this should only be after we start the app
937 # FIXME: topics.append( "org.onosproject.election" )
938 # Print leaders output
939 main.log.debug( topics )
940 ONOStopics = [ j['topic'] for j in parsedLeaders ]
941 for topic in topics:
942 if topic not in ONOStopics:
943 main.log.error( "Error: " + topic +
944 " not in leaders" )
945 topicCheck = main.FALSE
946 else:
947 main.log.error( "leaders() returned None" )
948 topicCheck = main.FALSE
949 except ( ValueError, TypeError ):
950 topicCheck = main.FALSE
951 main.log.exception( "Error parsing leaders" )
952 main.log.error( repr( leaders ) )
953 # TODO: Check for a leader of these topics
954 # Check all nodes
955 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700956 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700957 response = node.leaders( jsonFormat=False)
958 main.log.warn( str( node.name ) + " leaders output: \n" +
959 str( response ) )
960
961 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
962 onpass="intent Partitions is in leaders",
963 onfail="Some topics were lost " )
964 # Print partitions
965 partitions = main.ONOScli1.partitions()
966 try:
967 if partitions :
968 parsedPartitions = json.loads( partitions )
969 main.log.warn( json.dumps( parsedPartitions,
970 sort_keys=True,
971 indent=4,
972 separators=( ',', ': ' ) ) )
973 # TODO check for a leader in all paritions
974 # TODO check for consistency among nodes
975 else:
976 main.log.error( "partitions() returned None" )
977 except ( ValueError, TypeError ):
978 main.log.exception( "Error parsing partitions" )
979 main.log.error( repr( partitions ) )
980 # Print Pending Map
981 pendingMap = main.ONOScli1.pendingMap()
982 try:
983 if pendingMap :
984 parsedPending = json.loads( pendingMap )
985 main.log.warn( json.dumps( parsedPending,
986 sort_keys=True,
987 indent=4,
988 separators=( ',', ': ' ) ) )
989 # TODO check something here?
990 else:
991 main.log.error( "pendingMap() returned None" )
992 except ( ValueError, TypeError ):
993 main.log.exception( "Error parsing pending map" )
994 main.log.error( repr( pendingMap ) )
995
996 if not installedCheck:
997 main.log.info( "Waiting 60 seconds to see if the state of " +
998 "intents change" )
999 time.sleep( 60 )
1000 # Print the intent states
1001 intents = main.ONOScli1.intents()
1002 intentStates = []
1003 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1004 count = 0
1005 # Iter through intents of a node
1006 try:
1007 for intent in json.loads( intents ):
1008 state = intent.get( 'state', None )
1009 if "INSTALLED" not in state:
1010 installedCheck = False
1011 intentId = intent.get( 'id', None )
1012 intentStates.append( ( intentId, state ) )
1013 except ( ValueError, TypeError ):
1014 main.log.exception( "Error parsing intents." )
1015 intentStates.sort()
1016 for i, s in intentStates:
1017 count += 1
1018 main.log.info( "%-6s%-15s%-15s" %
1019 ( str( count ), str( i ), str( s ) ) )
1020 leaders = main.ONOScli1.leaders()
1021 try:
1022 missing = False
1023 if leaders:
1024 parsedLeaders = json.loads( leaders )
1025 main.log.warn( json.dumps( parsedLeaders,
1026 sort_keys=True,
1027 indent=4,
1028 separators=( ',', ': ' ) ) )
1029 # check for all intent partitions
1030 # check for election
1031 topics = []
1032 for i in range( 14 ):
1033 topics.append( "intent-partition-" + str( i ) )
1034 # FIXME: this should only be after we start the app
1035 topics.append( "org.onosproject.election" )
1036 main.log.debug( topics )
1037 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1038 for topic in topics:
1039 if topic not in ONOStopics:
1040 main.log.error( "Error: " + topic +
1041 " not in leaders" )
1042 missing = True
1043 else:
1044 main.log.error( "leaders() returned None" )
1045 except ( ValueError, TypeError ):
1046 main.log.exception( "Error parsing leaders" )
1047 main.log.error( repr( leaders ) )
1048 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001049 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001050 response = node.leaders( jsonFormat=False)
1051 main.log.warn( str( node.name ) + " leaders output: \n" +
1052 str( response ) )
1053
1054 partitions = main.ONOScli1.partitions()
1055 try:
1056 if partitions :
1057 parsedPartitions = json.loads( partitions )
1058 main.log.warn( json.dumps( parsedPartitions,
1059 sort_keys=True,
1060 indent=4,
1061 separators=( ',', ': ' ) ) )
1062 # TODO check for a leader in all paritions
1063 # TODO check for consistency among nodes
1064 else:
1065 main.log.error( "partitions() returned None" )
1066 except ( ValueError, TypeError ):
1067 main.log.exception( "Error parsing partitions" )
1068 main.log.error( repr( partitions ) )
1069 pendingMap = main.ONOScli1.pendingMap()
1070 try:
1071 if pendingMap :
1072 parsedPending = json.loads( pendingMap )
1073 main.log.warn( json.dumps( parsedPending,
1074 sort_keys=True,
1075 indent=4,
1076 separators=( ',', ': ' ) ) )
1077 # TODO check something here?
1078 else:
1079 main.log.error( "pendingMap() returned None" )
1080 except ( ValueError, TypeError ):
1081 main.log.exception( "Error parsing pending map" )
1082 main.log.error( repr( pendingMap ) )
1083 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001084 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001085 main.step( "Wait a minute then ping again" )
1086 # the wait is above
1087 PingResult = main.TRUE
1088 for i in range( 8, 18 ):
1089 ping = main.Mininet1.pingHost( src="h" + str( i ),
1090 target="h" + str( i + 10 ) )
1091 PingResult = PingResult and ping
1092 if ping == main.FALSE:
1093 main.log.warn( "Ping failed between h" + str( i ) +
1094 " and h" + str( i + 10 ) )
1095 elif ping == main.TRUE:
1096 main.log.info( "Ping test passed!" )
1097 # Don't set PingResult or you'd override failures
1098 if PingResult == main.FALSE:
1099 main.log.error(
1100 "Intents have not been installed correctly, pings failed." )
1101 # TODO: pretty print
1102 main.log.warn( "ONOS1 intents: " )
1103 try:
1104 tmpIntents = main.ONOScli1.intents()
1105 main.log.warn( json.dumps( json.loads( tmpIntents ),
1106 sort_keys=True,
1107 indent=4,
1108 separators=( ',', ': ' ) ) )
1109 except ( ValueError, TypeError ):
1110 main.log.warn( repr( tmpIntents ) )
1111 utilities.assert_equals(
1112 expect=main.TRUE,
1113 actual=PingResult,
1114 onpass="Intents have been installed correctly and pings work",
1115 onfail="Intents have not been installed correctly, pings failed." )
1116
1117 def CASE5( self, main ):
1118 """
1119 Reading state of ONOS
1120 """
1121 import json
1122 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001123 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001124 assert main, "main not defined"
1125 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001126 assert main.CLIs, "main.CLIs not defined"
1127 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001128
1129 main.case( "Setting up and gathering data for current state" )
1130 # The general idea for this test case is to pull the state of
1131 # ( intents,flows, topology,... ) from each ONOS node
1132 # We can then compare them with each other and also with past states
1133
1134 main.step( "Check that each switch has a master" )
1135 global mastershipState
1136 mastershipState = '[]'
1137
1138 # Assert that each device has a master
1139 rolesNotNull = main.TRUE
1140 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001141 for i in range( main.numCtrls ):
1142 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001143 name="rolesNotNull-" + str( i ),
1144 args=[] )
1145 threads.append( t )
1146 t.start()
1147
1148 for t in threads:
1149 t.join()
1150 rolesNotNull = rolesNotNull and t.result
1151 utilities.assert_equals(
1152 expect=main.TRUE,
1153 actual=rolesNotNull,
1154 onpass="Each device has a master",
1155 onfail="Some devices don't have a master assigned" )
1156
1157 main.step( "Get the Mastership of each switch from each controller" )
1158 ONOSMastership = []
1159 mastershipCheck = main.FALSE
1160 consistentMastership = True
1161 rolesResults = True
1162 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001163 for i in range( main.numCtrls ):
1164 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001165 name="roles-" + str( i ),
1166 args=[] )
1167 threads.append( t )
1168 t.start()
1169
1170 for t in threads:
1171 t.join()
1172 ONOSMastership.append( t.result )
1173
Jon Halle1a3b752015-07-22 13:02:46 -07001174 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001175 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1176 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1177 " roles" )
1178 main.log.warn(
1179 "ONOS" + str( i + 1 ) + " mastership response: " +
1180 repr( ONOSMastership[i] ) )
1181 rolesResults = False
1182 utilities.assert_equals(
1183 expect=True,
1184 actual=rolesResults,
1185 onpass="No error in reading roles output",
1186 onfail="Error in reading roles from ONOS" )
1187
1188 main.step( "Check for consistency in roles from each controller" )
1189 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1190 main.log.info(
1191 "Switch roles are consistent across all ONOS nodes" )
1192 else:
1193 consistentMastership = False
1194 utilities.assert_equals(
1195 expect=True,
1196 actual=consistentMastership,
1197 onpass="Switch roles are consistent across all ONOS nodes",
1198 onfail="ONOS nodes have different views of switch roles" )
1199
1200 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001201 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001202 try:
1203 main.log.warn(
1204 "ONOS" + str( i + 1 ) + " roles: ",
1205 json.dumps(
1206 json.loads( ONOSMastership[ i ] ),
1207 sort_keys=True,
1208 indent=4,
1209 separators=( ',', ': ' ) ) )
1210 except ( ValueError, TypeError ):
1211 main.log.warn( repr( ONOSMastership[ i ] ) )
1212 elif rolesResults and consistentMastership:
1213 mastershipCheck = main.TRUE
1214 mastershipState = ONOSMastership[ 0 ]
1215
1216 main.step( "Get the intents from each controller" )
1217 global intentState
1218 intentState = []
1219 ONOSIntents = []
1220 intentCheck = main.FALSE
1221 consistentIntents = True
1222 intentsResults = True
1223 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001224 for i in range( main.numCtrls ):
1225 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001226 name="intents-" + str( i ),
1227 args=[],
1228 kwargs={ 'jsonFormat': True } )
1229 threads.append( t )
1230 t.start()
1231
1232 for t in threads:
1233 t.join()
1234 ONOSIntents.append( t.result )
1235
Jon Halle1a3b752015-07-22 13:02:46 -07001236 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001237 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1238 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1239 " intents" )
1240 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1241 repr( ONOSIntents[ i ] ) )
1242 intentsResults = False
1243 utilities.assert_equals(
1244 expect=True,
1245 actual=intentsResults,
1246 onpass="No error in reading intents output",
1247 onfail="Error in reading intents from ONOS" )
1248
1249 main.step( "Check for consistency in Intents from each controller" )
1250 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1251 main.log.info( "Intents are consistent across all ONOS " +
1252 "nodes" )
1253 else:
1254 consistentIntents = False
1255 main.log.error( "Intents not consistent" )
1256 utilities.assert_equals(
1257 expect=True,
1258 actual=consistentIntents,
1259 onpass="Intents are consistent across all ONOS nodes",
1260 onfail="ONOS nodes have different views of intents" )
1261
1262 if intentsResults:
1263 # Try to make it easy to figure out what is happening
1264 #
1265 # Intent ONOS1 ONOS2 ...
1266 # 0x01 INSTALLED INSTALLING
1267 # ... ... ...
1268 # ... ... ...
1269 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001270 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001271 title += " " * 10 + "ONOS" + str( n + 1 )
1272 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001273 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001274 keys = []
1275 try:
1276 # Get the set of all intent keys
1277 for nodeStr in ONOSIntents:
1278 node = json.loads( nodeStr )
1279 for intent in node:
1280 keys.append( intent.get( 'id' ) )
1281 keys = set( keys )
1282 # For each intent key, print the state on each node
1283 for key in keys:
1284 row = "%-13s" % key
1285 for nodeStr in ONOSIntents:
1286 node = json.loads( nodeStr )
1287 for intent in node:
1288 if intent.get( 'id', "Error" ) == key:
1289 row += "%-15s" % intent.get( 'state' )
1290 main.log.warn( row )
1291 # End of intent state table
1292 except ValueError as e:
1293 main.log.exception( e )
1294 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1295
1296 if intentsResults and not consistentIntents:
1297 # print the json objects
1298 n = len(ONOSIntents)
1299 main.log.debug( "ONOS" + str( n ) + " intents: " )
1300 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1301 sort_keys=True,
1302 indent=4,
1303 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001304 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001305 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1306 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1307 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1308 sort_keys=True,
1309 indent=4,
1310 separators=( ',', ': ' ) ) )
1311 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001312 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001313 str( n ) + " intents" )
1314 elif intentsResults and consistentIntents:
1315 intentCheck = main.TRUE
1316 intentState = ONOSIntents[ 0 ]
1317
1318 main.step( "Get the flows from each controller" )
1319 global flowState
1320 flowState = []
1321 ONOSFlows = []
1322 ONOSFlowsJson = []
1323 flowCheck = main.FALSE
1324 consistentFlows = True
1325 flowsResults = True
1326 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001327 for i in range( main.numCtrls ):
1328 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001329 name="flows-" + str( i ),
1330 args=[],
1331 kwargs={ 'jsonFormat': True } )
1332 threads.append( t )
1333 t.start()
1334
1335 # NOTE: Flows command can take some time to run
1336 time.sleep(30)
1337 for t in threads:
1338 t.join()
1339 result = t.result
1340 ONOSFlows.append( result )
1341
Jon Halle1a3b752015-07-22 13:02:46 -07001342 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001343 num = str( i + 1 )
1344 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1345 main.log.error( "Error in getting ONOS" + num + " flows" )
1346 main.log.warn( "ONOS" + num + " flows response: " +
1347 repr( ONOSFlows[ i ] ) )
1348 flowsResults = False
1349 ONOSFlowsJson.append( None )
1350 else:
1351 try:
1352 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1353 except ( ValueError, TypeError ):
1354 # FIXME: change this to log.error?
1355 main.log.exception( "Error in parsing ONOS" + num +
1356 " response as json." )
1357 main.log.error( repr( ONOSFlows[ i ] ) )
1358 ONOSFlowsJson.append( None )
1359 flowsResults = False
1360 utilities.assert_equals(
1361 expect=True,
1362 actual=flowsResults,
1363 onpass="No error in reading flows output",
1364 onfail="Error in reading flows from ONOS" )
1365
1366 main.step( "Check for consistency in Flows from each controller" )
1367 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1368 if all( tmp ):
1369 main.log.info( "Flow count is consistent across all ONOS nodes" )
1370 else:
1371 consistentFlows = False
1372 utilities.assert_equals(
1373 expect=True,
1374 actual=consistentFlows,
1375 onpass="The flow count is consistent across all ONOS nodes",
1376 onfail="ONOS nodes have different flow counts" )
1377
1378 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001379 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001380 try:
1381 main.log.warn(
1382 "ONOS" + str( i + 1 ) + " flows: " +
1383 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1384 indent=4, separators=( ',', ': ' ) ) )
1385 except ( ValueError, TypeError ):
1386 main.log.warn(
1387 "ONOS" + str( i + 1 ) + " flows: " +
1388 repr( ONOSFlows[ i ] ) )
1389 elif flowsResults and consistentFlows:
1390 flowCheck = main.TRUE
1391 flowState = ONOSFlows[ 0 ]
1392
1393 main.step( "Get the OF Table entries" )
1394 global flows
1395 flows = []
1396 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001397 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001398 if flowCheck == main.FALSE:
1399 for table in flows:
1400 main.log.warn( table )
GlennRC68467eb2015-11-16 18:01:01 -08001401
Jon Hall5cf14d52015-07-16 12:15:19 -07001402 # TODO: Compare switch flow tables with ONOS flow tables
1403
1404 main.step( "Start continuous pings" )
1405 main.Mininet2.pingLong(
1406 src=main.params[ 'PING' ][ 'source1' ],
1407 target=main.params[ 'PING' ][ 'target1' ],
1408 pingTime=500 )
1409 main.Mininet2.pingLong(
1410 src=main.params[ 'PING' ][ 'source2' ],
1411 target=main.params[ 'PING' ][ 'target2' ],
1412 pingTime=500 )
1413 main.Mininet2.pingLong(
1414 src=main.params[ 'PING' ][ 'source3' ],
1415 target=main.params[ 'PING' ][ 'target3' ],
1416 pingTime=500 )
1417 main.Mininet2.pingLong(
1418 src=main.params[ 'PING' ][ 'source4' ],
1419 target=main.params[ 'PING' ][ 'target4' ],
1420 pingTime=500 )
1421 main.Mininet2.pingLong(
1422 src=main.params[ 'PING' ][ 'source5' ],
1423 target=main.params[ 'PING' ][ 'target5' ],
1424 pingTime=500 )
1425 main.Mininet2.pingLong(
1426 src=main.params[ 'PING' ][ 'source6' ],
1427 target=main.params[ 'PING' ][ 'target6' ],
1428 pingTime=500 )
1429 main.Mininet2.pingLong(
1430 src=main.params[ 'PING' ][ 'source7' ],
1431 target=main.params[ 'PING' ][ 'target7' ],
1432 pingTime=500 )
1433 main.Mininet2.pingLong(
1434 src=main.params[ 'PING' ][ 'source8' ],
1435 target=main.params[ 'PING' ][ 'target8' ],
1436 pingTime=500 )
1437 main.Mininet2.pingLong(
1438 src=main.params[ 'PING' ][ 'source9' ],
1439 target=main.params[ 'PING' ][ 'target9' ],
1440 pingTime=500 )
1441 main.Mininet2.pingLong(
1442 src=main.params[ 'PING' ][ 'source10' ],
1443 target=main.params[ 'PING' ][ 'target10' ],
1444 pingTime=500 )
1445
1446 main.step( "Collecting topology information from ONOS" )
1447 devices = []
1448 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001449 for i in range( main.numCtrls ):
1450 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001451 name="devices-" + str( i ),
1452 args=[ ] )
1453 threads.append( t )
1454 t.start()
1455
1456 for t in threads:
1457 t.join()
1458 devices.append( t.result )
1459 hosts = []
1460 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001461 for i in range( main.numCtrls ):
1462 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001463 name="hosts-" + str( i ),
1464 args=[ ] )
1465 threads.append( t )
1466 t.start()
1467
1468 for t in threads:
1469 t.join()
1470 try:
1471 hosts.append( json.loads( t.result ) )
1472 except ( ValueError, TypeError ):
1473 # FIXME: better handling of this, print which node
1474 # Maybe use thread name?
1475 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001476 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001477 hosts.append( None )
1478
1479 ports = []
1480 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001481 for i in range( main.numCtrls ):
1482 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001483 name="ports-" + str( i ),
1484 args=[ ] )
1485 threads.append( t )
1486 t.start()
1487
1488 for t in threads:
1489 t.join()
1490 ports.append( t.result )
1491 links = []
1492 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001493 for i in range( main.numCtrls ):
1494 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001495 name="links-" + str( i ),
1496 args=[ ] )
1497 threads.append( t )
1498 t.start()
1499
1500 for t in threads:
1501 t.join()
1502 links.append( t.result )
1503 clusters = []
1504 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001505 for i in range( main.numCtrls ):
1506 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001507 name="clusters-" + str( i ),
1508 args=[ ] )
1509 threads.append( t )
1510 t.start()
1511
1512 for t in threads:
1513 t.join()
1514 clusters.append( t.result )
1515 # Compare json objects for hosts and dataplane clusters
1516
1517 # hosts
1518 main.step( "Host view is consistent across ONOS nodes" )
1519 consistentHostsResult = main.TRUE
1520 for controller in range( len( hosts ) ):
1521 controllerStr = str( controller + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001522 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001523 if hosts[ controller ] == hosts[ 0 ]:
1524 continue
1525 else: # hosts not consistent
1526 main.log.error( "hosts from ONOS" +
1527 controllerStr +
1528 " is inconsistent with ONOS1" )
1529 main.log.warn( repr( hosts[ controller ] ) )
1530 consistentHostsResult = main.FALSE
1531
1532 else:
1533 main.log.error( "Error in getting ONOS hosts from ONOS" +
1534 controllerStr )
1535 consistentHostsResult = main.FALSE
1536 main.log.warn( "ONOS" + controllerStr +
1537 " hosts response: " +
1538 repr( hosts[ controller ] ) )
1539 utilities.assert_equals(
1540 expect=main.TRUE,
1541 actual=consistentHostsResult,
1542 onpass="Hosts view is consistent across all ONOS nodes",
1543 onfail="ONOS nodes have different views of hosts" )
1544
1545 main.step( "Each host has an IP address" )
1546 ipResult = main.TRUE
1547 for controller in range( 0, len( hosts ) ):
1548 controllerStr = str( controller + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001549 if hosts[ controller ]:
1550 for host in hosts[ controller ]:
1551 if not host.get( 'ipAddresses', [ ] ):
1552 main.log.error( "Error with host ips on controller" +
1553 controllerStr + ": " + str( host ) )
1554 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001555 utilities.assert_equals(
1556 expect=main.TRUE,
1557 actual=ipResult,
1558 onpass="The ips of the hosts aren't empty",
1559 onfail="The ip of at least one host is missing" )
1560
1561 # Strongly connected clusters of devices
1562 main.step( "Cluster view is consistent across ONOS nodes" )
1563 consistentClustersResult = main.TRUE
1564 for controller in range( len( clusters ) ):
1565 controllerStr = str( controller + 1 )
1566 if "Error" not in clusters[ controller ]:
1567 if clusters[ controller ] == clusters[ 0 ]:
1568 continue
1569 else: # clusters not consistent
1570 main.log.error( "clusters from ONOS" + controllerStr +
1571 " is inconsistent with ONOS1" )
1572 consistentClustersResult = main.FALSE
1573
1574 else:
1575 main.log.error( "Error in getting dataplane clusters " +
1576 "from ONOS" + controllerStr )
1577 consistentClustersResult = main.FALSE
1578 main.log.warn( "ONOS" + controllerStr +
1579 " clusters response: " +
1580 repr( clusters[ controller ] ) )
1581 utilities.assert_equals(
1582 expect=main.TRUE,
1583 actual=consistentClustersResult,
1584 onpass="Clusters view is consistent across all ONOS nodes",
1585 onfail="ONOS nodes have different views of clusters" )
1586 # there should always only be one cluster
1587 main.step( "Cluster view correct across ONOS nodes" )
1588 try:
1589 numClusters = len( json.loads( clusters[ 0 ] ) )
1590 except ( ValueError, TypeError ):
1591 main.log.exception( "Error parsing clusters[0]: " +
1592 repr( clusters[ 0 ] ) )
1593 clusterResults = main.FALSE
1594 if numClusters == 1:
1595 clusterResults = main.TRUE
1596 utilities.assert_equals(
1597 expect=1,
1598 actual=numClusters,
1599 onpass="ONOS shows 1 SCC",
1600 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1601
1602 main.step( "Comparing ONOS topology to MN" )
1603 devicesResults = main.TRUE
1604 linksResults = main.TRUE
1605 hostsResults = main.TRUE
1606 mnSwitches = main.Mininet1.getSwitches()
1607 mnLinks = main.Mininet1.getLinks()
1608 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001609 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001610 controllerStr = str( controller + 1 )
1611 if devices[ controller ] and ports[ controller ] and\
1612 "Error" not in devices[ controller ] and\
1613 "Error" not in ports[ controller ]:
1614
1615 currentDevicesResult = main.Mininet1.compareSwitches(
1616 mnSwitches,
1617 json.loads( devices[ controller ] ),
1618 json.loads( ports[ controller ] ) )
1619 else:
1620 currentDevicesResult = main.FALSE
1621 utilities.assert_equals( expect=main.TRUE,
1622 actual=currentDevicesResult,
1623 onpass="ONOS" + controllerStr +
1624 " Switches view is correct",
1625 onfail="ONOS" + controllerStr +
1626 " Switches view is incorrect" )
1627 if links[ controller ] and "Error" not in links[ controller ]:
1628 currentLinksResult = main.Mininet1.compareLinks(
1629 mnSwitches, mnLinks,
1630 json.loads( links[ controller ] ) )
1631 else:
1632 currentLinksResult = main.FALSE
1633 utilities.assert_equals( expect=main.TRUE,
1634 actual=currentLinksResult,
1635 onpass="ONOS" + controllerStr +
1636 " links view is correct",
1637 onfail="ONOS" + controllerStr +
1638 " links view is incorrect" )
1639
Jon Hall657cdf62015-12-17 14:40:51 -08001640 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001641 currentHostsResult = main.Mininet1.compareHosts(
1642 mnHosts,
1643 hosts[ controller ] )
1644 else:
1645 currentHostsResult = main.FALSE
1646 utilities.assert_equals( expect=main.TRUE,
1647 actual=currentHostsResult,
1648 onpass="ONOS" + controllerStr +
1649 " hosts exist in Mininet",
1650 onfail="ONOS" + controllerStr +
1651 " hosts don't match Mininet" )
1652
1653 devicesResults = devicesResults and currentDevicesResult
1654 linksResults = linksResults and currentLinksResult
1655 hostsResults = hostsResults and currentHostsResult
1656
1657 main.step( "Device information is correct" )
1658 utilities.assert_equals(
1659 expect=main.TRUE,
1660 actual=devicesResults,
1661 onpass="Device information is correct",
1662 onfail="Device information is incorrect" )
1663
1664 main.step( "Links are correct" )
1665 utilities.assert_equals(
1666 expect=main.TRUE,
1667 actual=linksResults,
1668 onpass="Link are correct",
1669 onfail="Links are incorrect" )
1670
1671 main.step( "Hosts are correct" )
1672 utilities.assert_equals(
1673 expect=main.TRUE,
1674 actual=hostsResults,
1675 onpass="Hosts are correct",
1676 onfail="Hosts are incorrect" )
1677
1678 def CASE6( self, main ):
1679 """
1680 The Failure case. Since this is the Sanity test, we do nothing.
1681 """
1682 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001683 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001684 assert main, "main not defined"
1685 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001686 assert main.CLIs, "main.CLIs not defined"
1687 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001688 main.case( "Wait 60 seconds instead of inducing a failure" )
1689 time.sleep( 60 )
1690 utilities.assert_equals(
1691 expect=main.TRUE,
1692 actual=main.TRUE,
1693 onpass="Sleeping 60 seconds",
1694 onfail="Something is terribly wrong with my math" )
1695
1696 def CASE7( self, main ):
1697 """
1698 Check state after ONOS failure
1699 """
1700 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001701 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001702 assert main, "main not defined"
1703 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001704 assert main.CLIs, "main.CLIs not defined"
1705 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001706 main.case( "Running ONOS Constant State Tests" )
1707
1708 main.step( "Check that each switch has a master" )
1709 # Assert that each device has a master
1710 rolesNotNull = main.TRUE
1711 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001712 for i in range( main.numCtrls ):
1713 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001714 name="rolesNotNull-" + str( i ),
1715 args=[ ] )
1716 threads.append( t )
1717 t.start()
1718
1719 for t in threads:
1720 t.join()
1721 rolesNotNull = rolesNotNull and t.result
1722 utilities.assert_equals(
1723 expect=main.TRUE,
1724 actual=rolesNotNull,
1725 onpass="Each device has a master",
1726 onfail="Some devices don't have a master assigned" )
1727
1728 main.step( "Read device roles from ONOS" )
1729 ONOSMastership = []
1730 mastershipCheck = main.FALSE
1731 consistentMastership = True
1732 rolesResults = True
1733 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001734 for i in range( main.numCtrls ):
1735 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001736 name="roles-" + str( i ),
1737 args=[] )
1738 threads.append( t )
1739 t.start()
1740
1741 for t in threads:
1742 t.join()
1743 ONOSMastership.append( t.result )
1744
Jon Halle1a3b752015-07-22 13:02:46 -07001745 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001746 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1747 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1748 " roles" )
1749 main.log.warn(
1750 "ONOS" + str( i + 1 ) + " mastership response: " +
1751 repr( ONOSMastership[i] ) )
1752 rolesResults = False
1753 utilities.assert_equals(
1754 expect=True,
1755 actual=rolesResults,
1756 onpass="No error in reading roles output",
1757 onfail="Error in reading roles from ONOS" )
1758
1759 main.step( "Check for consistency in roles from each controller" )
1760 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1761 main.log.info(
1762 "Switch roles are consistent across all ONOS nodes" )
1763 else:
1764 consistentMastership = False
1765 utilities.assert_equals(
1766 expect=True,
1767 actual=consistentMastership,
1768 onpass="Switch roles are consistent across all ONOS nodes",
1769 onfail="ONOS nodes have different views of switch roles" )
1770
1771 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001772 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001773 main.log.warn(
1774 "ONOS" + str( i + 1 ) + " roles: ",
1775 json.dumps(
1776 json.loads( ONOSMastership[ i ] ),
1777 sort_keys=True,
1778 indent=4,
1779 separators=( ',', ': ' ) ) )
1780 elif rolesResults and not consistentMastership:
1781 mastershipCheck = main.TRUE
1782
1783 description2 = "Compare switch roles from before failure"
1784 main.step( description2 )
1785 try:
1786 currentJson = json.loads( ONOSMastership[0] )
1787 oldJson = json.loads( mastershipState )
1788 except ( ValueError, TypeError ):
1789 main.log.exception( "Something is wrong with parsing " +
1790 "ONOSMastership[0] or mastershipState" )
1791 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1792 main.log.error( "mastershipState" + repr( mastershipState ) )
1793 main.cleanup()
1794 main.exit()
1795 mastershipCheck = main.TRUE
1796 for i in range( 1, 29 ):
1797 switchDPID = str(
1798 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1799 current = [ switch[ 'master' ] for switch in currentJson
1800 if switchDPID in switch[ 'id' ] ]
1801 old = [ switch[ 'master' ] for switch in oldJson
1802 if switchDPID in switch[ 'id' ] ]
1803 if current == old:
1804 mastershipCheck = mastershipCheck and main.TRUE
1805 else:
1806 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1807 mastershipCheck = main.FALSE
1808 utilities.assert_equals(
1809 expect=main.TRUE,
1810 actual=mastershipCheck,
1811 onpass="Mastership of Switches was not changed",
1812 onfail="Mastership of some switches changed" )
1813 mastershipCheck = mastershipCheck and consistentMastership
1814
1815 main.step( "Get the intents and compare across all nodes" )
1816 ONOSIntents = []
1817 intentCheck = main.FALSE
1818 consistentIntents = True
1819 intentsResults = True
1820 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001821 for i in range( main.numCtrls ):
1822 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001823 name="intents-" + str( i ),
1824 args=[],
1825 kwargs={ 'jsonFormat': True } )
1826 threads.append( t )
1827 t.start()
1828
1829 for t in threads:
1830 t.join()
1831 ONOSIntents.append( t.result )
1832
Jon Halle1a3b752015-07-22 13:02:46 -07001833 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001834 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1835 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1836 " intents" )
1837 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1838 repr( ONOSIntents[ i ] ) )
1839 intentsResults = False
1840 utilities.assert_equals(
1841 expect=True,
1842 actual=intentsResults,
1843 onpass="No error in reading intents output",
1844 onfail="Error in reading intents from ONOS" )
1845
1846 main.step( "Check for consistency in Intents from each controller" )
1847 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1848 main.log.info( "Intents are consistent across all ONOS " +
1849 "nodes" )
1850 else:
1851 consistentIntents = False
1852
1853 # Try to make it easy to figure out what is happening
1854 #
1855 # Intent ONOS1 ONOS2 ...
1856 # 0x01 INSTALLED INSTALLING
1857 # ... ... ...
1858 # ... ... ...
1859 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001860 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001861 title += " " * 10 + "ONOS" + str( n + 1 )
1862 main.log.warn( title )
1863 # get all intent keys in the cluster
1864 keys = []
1865 for nodeStr in ONOSIntents:
1866 node = json.loads( nodeStr )
1867 for intent in node:
1868 keys.append( intent.get( 'id' ) )
1869 keys = set( keys )
1870 for key in keys:
1871 row = "%-13s" % key
1872 for nodeStr in ONOSIntents:
1873 node = json.loads( nodeStr )
1874 for intent in node:
1875 if intent.get( 'id' ) == key:
1876 row += "%-15s" % intent.get( 'state' )
1877 main.log.warn( row )
1878 # End table view
1879
1880 utilities.assert_equals(
1881 expect=True,
1882 actual=consistentIntents,
1883 onpass="Intents are consistent across all ONOS nodes",
1884 onfail="ONOS nodes have different views of intents" )
1885 intentStates = []
1886 for node in ONOSIntents: # Iter through ONOS nodes
1887 nodeStates = []
1888 # Iter through intents of a node
1889 try:
1890 for intent in json.loads( node ):
1891 nodeStates.append( intent[ 'state' ] )
1892 except ( ValueError, TypeError ):
1893 main.log.exception( "Error in parsing intents" )
1894 main.log.error( repr( node ) )
1895 intentStates.append( nodeStates )
1896 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1897 main.log.info( dict( out ) )
1898
1899 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001900 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001901 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1902 main.log.warn( json.dumps(
1903 json.loads( ONOSIntents[ i ] ),
1904 sort_keys=True,
1905 indent=4,
1906 separators=( ',', ': ' ) ) )
1907 elif intentsResults and consistentIntents:
1908 intentCheck = main.TRUE
1909
1910 # NOTE: Store has no durability, so intents are lost across system
1911 # restarts
1912 main.step( "Compare current intents with intents before the failure" )
1913 # NOTE: this requires case 5 to pass for intentState to be set.
1914 # maybe we should stop the test if that fails?
1915 sameIntents = main.FALSE
1916 if intentState and intentState == ONOSIntents[ 0 ]:
1917 sameIntents = main.TRUE
1918 main.log.info( "Intents are consistent with before failure" )
1919 # TODO: possibly the states have changed? we may need to figure out
1920 # what the acceptable states are
1921 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1922 sameIntents = main.TRUE
1923 try:
1924 before = json.loads( intentState )
1925 after = json.loads( ONOSIntents[ 0 ] )
1926 for intent in before:
1927 if intent not in after:
1928 sameIntents = main.FALSE
1929 main.log.debug( "Intent is not currently in ONOS " +
1930 "(at least in the same form):" )
1931 main.log.debug( json.dumps( intent ) )
1932 except ( ValueError, TypeError ):
1933 main.log.exception( "Exception printing intents" )
1934 main.log.debug( repr( ONOSIntents[0] ) )
1935 main.log.debug( repr( intentState ) )
1936 if sameIntents == main.FALSE:
1937 try:
1938 main.log.debug( "ONOS intents before: " )
1939 main.log.debug( json.dumps( json.loads( intentState ),
1940 sort_keys=True, indent=4,
1941 separators=( ',', ': ' ) ) )
1942 main.log.debug( "Current ONOS intents: " )
1943 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1944 sort_keys=True, indent=4,
1945 separators=( ',', ': ' ) ) )
1946 except ( ValueError, TypeError ):
1947 main.log.exception( "Exception printing intents" )
1948 main.log.debug( repr( ONOSIntents[0] ) )
1949 main.log.debug( repr( intentState ) )
1950 utilities.assert_equals(
1951 expect=main.TRUE,
1952 actual=sameIntents,
1953 onpass="Intents are consistent with before failure",
1954 onfail="The Intents changed during failure" )
1955 intentCheck = intentCheck and sameIntents
1956
1957 main.step( "Get the OF Table entries and compare to before " +
1958 "component failure" )
1959 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07001960 for i in range( 28 ):
1961 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08001962 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
1963 FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
Jon Hall5cf14d52015-07-16 12:15:19 -07001964 if FlowTables == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08001965 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
1966
Jon Hall5cf14d52015-07-16 12:15:19 -07001967 utilities.assert_equals(
1968 expect=main.TRUE,
1969 actual=FlowTables,
1970 onpass="No changes were found in the flow tables",
1971 onfail="Changes were found in the flow tables" )
1972
1973 main.Mininet2.pingLongKill()
1974 '''
1975 main.step( "Check the continuous pings to ensure that no packets " +
1976 "were dropped during component failure" )
1977 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1978 main.params[ 'TESTONIP' ] )
1979 LossInPings = main.FALSE
1980 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1981 for i in range( 8, 18 ):
1982 main.log.info(
1983 "Checking for a loss in pings along flow from s" +
1984 str( i ) )
1985 LossInPings = main.Mininet2.checkForLoss(
1986 "/tmp/ping.h" +
1987 str( i ) ) or LossInPings
1988 if LossInPings == main.TRUE:
1989 main.log.info( "Loss in ping detected" )
1990 elif LossInPings == main.ERROR:
1991 main.log.info( "There are multiple mininet process running" )
1992 elif LossInPings == main.FALSE:
1993 main.log.info( "No Loss in the pings" )
1994 main.log.info( "No loss of dataplane connectivity" )
1995 utilities.assert_equals(
1996 expect=main.FALSE,
1997 actual=LossInPings,
1998 onpass="No Loss of connectivity",
1999 onfail="Loss of dataplane connectivity detected" )
2000 '''
2001
2002 main.step( "Leadership Election is still functional" )
2003 # Test of LeadershipElection
2004 # NOTE: this only works for the sanity test. In case of failures,
2005 # leader will likely change
Jon Halle1a3b752015-07-22 13:02:46 -07002006 leader = main.nodes[ 0 ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002007 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002008 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002009 leaderN = cli.electionTestLeader()
2010 # verify leader is ONOS1
2011 if leaderN == leader:
2012 # all is well
2013 # NOTE: In failure scenario, this could be a new node, maybe
2014 # check != ONOS1
2015 pass
2016 elif leaderN == main.FALSE:
2017 # error in response
2018 main.log.error( "Something is wrong with " +
2019 "electionTestLeader function, check the" +
2020 " error logs" )
2021 leaderResult = main.FALSE
2022 elif leader != leaderN:
2023 leaderResult = main.FALSE
2024 main.log.error( cli.name + " sees " + str( leaderN ) +
2025 " as the leader of the election app. " +
2026 "Leader should be " + str( leader ) )
2027 utilities.assert_equals(
2028 expect=main.TRUE,
2029 actual=leaderResult,
2030 onpass="Leadership election passed",
2031 onfail="Something went wrong with Leadership election" )
2032
2033 def CASE8( self, main ):
2034 """
2035 Compare topo
2036 """
2037 import json
2038 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002039 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002040 assert main, "main not defined"
2041 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002042 assert main.CLIs, "main.CLIs not defined"
2043 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002044
2045 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002046 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002047 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002048 topoResult = main.FALSE
2049 elapsed = 0
2050 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002051 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002052 startTime = time.time()
2053 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002054 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002055 devicesResults = main.TRUE
2056 linksResults = main.TRUE
2057 hostsResults = main.TRUE
2058 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002059 count += 1
2060 cliStart = time.time()
2061 devices = []
2062 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002063 for i in range( main.numCtrls ):
2064 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002065 name="devices-" + str( i ),
2066 args=[ ] )
2067 threads.append( t )
2068 t.start()
2069
2070 for t in threads:
2071 t.join()
2072 devices.append( t.result )
2073 hosts = []
2074 ipResult = main.TRUE
2075 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002076 for i in range( main.numCtrls ):
2077 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07002078 name="hosts-" + str( i ),
2079 args=[ ] )
2080 threads.append( t )
2081 t.start()
2082
2083 for t in threads:
2084 t.join()
2085 try:
2086 hosts.append( json.loads( t.result ) )
2087 except ( ValueError, TypeError ):
2088 main.log.exception( "Error parsing hosts results" )
2089 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002090 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002091 for controller in range( 0, len( hosts ) ):
2092 controllerStr = str( controller + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002093 if hosts[ controller ]:
2094 for host in hosts[ controller ]:
2095 if host is None or host.get( 'ipAddresses', [] ) == []:
2096 main.log.error(
2097 "Error with host ipAddresses on controller" +
2098 controllerStr + ": " + str( host ) )
2099 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002100 ports = []
2101 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002102 for i in range( main.numCtrls ):
2103 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002104 name="ports-" + str( i ),
2105 args=[ ] )
2106 threads.append( t )
2107 t.start()
2108
2109 for t in threads:
2110 t.join()
2111 ports.append( t.result )
2112 links = []
2113 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002114 for i in range( main.numCtrls ):
2115 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002116 name="links-" + str( i ),
2117 args=[ ] )
2118 threads.append( t )
2119 t.start()
2120
2121 for t in threads:
2122 t.join()
2123 links.append( t.result )
2124 clusters = []
2125 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002126 for i in range( main.numCtrls ):
2127 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002128 name="clusters-" + str( i ),
2129 args=[ ] )
2130 threads.append( t )
2131 t.start()
2132
2133 for t in threads:
2134 t.join()
2135 clusters.append( t.result )
2136
2137 elapsed = time.time() - startTime
2138 cliTime = time.time() - cliStart
2139 print "Elapsed time: " + str( elapsed )
2140 print "CLI time: " + str( cliTime )
2141
2142 mnSwitches = main.Mininet1.getSwitches()
2143 mnLinks = main.Mininet1.getLinks()
2144 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002145 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002146 controllerStr = str( controller + 1 )
2147 if devices[ controller ] and ports[ controller ] and\
2148 "Error" not in devices[ controller ] and\
2149 "Error" not in ports[ controller ]:
2150
2151 currentDevicesResult = main.Mininet1.compareSwitches(
2152 mnSwitches,
2153 json.loads( devices[ controller ] ),
2154 json.loads( ports[ controller ] ) )
2155 else:
2156 currentDevicesResult = main.FALSE
2157 utilities.assert_equals( expect=main.TRUE,
2158 actual=currentDevicesResult,
2159 onpass="ONOS" + controllerStr +
2160 " Switches view is correct",
2161 onfail="ONOS" + controllerStr +
2162 " Switches view is incorrect" )
2163
2164 if links[ controller ] and "Error" not in links[ controller ]:
2165 currentLinksResult = main.Mininet1.compareLinks(
2166 mnSwitches, mnLinks,
2167 json.loads( links[ controller ] ) )
2168 else:
2169 currentLinksResult = main.FALSE
2170 utilities.assert_equals( expect=main.TRUE,
2171 actual=currentLinksResult,
2172 onpass="ONOS" + controllerStr +
2173 " links view is correct",
2174 onfail="ONOS" + controllerStr +
2175 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002176 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002177 currentHostsResult = main.Mininet1.compareHosts(
2178 mnHosts,
2179 hosts[ controller ] )
2180 else:
2181 currentHostsResult = main.FALSE
2182 utilities.assert_equals( expect=main.TRUE,
2183 actual=currentHostsResult,
2184 onpass="ONOS" + controllerStr +
2185 " hosts exist in Mininet",
2186 onfail="ONOS" + controllerStr +
2187 " hosts don't match Mininet" )
2188 # CHECKING HOST ATTACHMENT POINTS
2189 hostAttachment = True
2190 zeroHosts = False
2191 # FIXME: topo-HA/obelisk specific mappings:
2192 # key is mac and value is dpid
2193 mappings = {}
2194 for i in range( 1, 29 ): # hosts 1 through 28
2195 # set up correct variables:
2196 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2197 if i == 1:
2198 deviceId = "1000".zfill(16)
2199 elif i == 2:
2200 deviceId = "2000".zfill(16)
2201 elif i == 3:
2202 deviceId = "3000".zfill(16)
2203 elif i == 4:
2204 deviceId = "3004".zfill(16)
2205 elif i == 5:
2206 deviceId = "5000".zfill(16)
2207 elif i == 6:
2208 deviceId = "6000".zfill(16)
2209 elif i == 7:
2210 deviceId = "6007".zfill(16)
2211 elif i >= 8 and i <= 17:
2212 dpid = '3' + str( i ).zfill( 3 )
2213 deviceId = dpid.zfill(16)
2214 elif i >= 18 and i <= 27:
2215 dpid = '6' + str( i ).zfill( 3 )
2216 deviceId = dpid.zfill(16)
2217 elif i == 28:
2218 deviceId = "2800".zfill(16)
2219 mappings[ macId ] = deviceId
Jon Hall657cdf62015-12-17 14:40:51 -08002220 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002221 if hosts[ controller ] == []:
2222 main.log.warn( "There are no hosts discovered" )
2223 zeroHosts = True
2224 else:
2225 for host in hosts[ controller ]:
2226 mac = None
2227 location = None
2228 device = None
2229 port = None
2230 try:
2231 mac = host.get( 'mac' )
2232 assert mac, "mac field could not be found for this host object"
2233
2234 location = host.get( 'location' )
2235 assert location, "location field could not be found for this host object"
2236
2237 # Trim the protocol identifier off deviceId
2238 device = str( location.get( 'elementId' ) ).split(':')[1]
2239 assert device, "elementId field could not be found for this host location object"
2240
2241 port = location.get( 'port' )
2242 assert port, "port field could not be found for this host location object"
2243
2244 # Now check if this matches where they should be
2245 if mac and device and port:
2246 if str( port ) != "1":
2247 main.log.error( "The attachment port is incorrect for " +
2248 "host " + str( mac ) +
2249 ". Expected: 1 Actual: " + str( port) )
2250 hostAttachment = False
2251 if device != mappings[ str( mac ) ]:
2252 main.log.error( "The attachment device is incorrect for " +
2253 "host " + str( mac ) +
2254 ". Expected: " + mappings[ str( mac ) ] +
2255 " Actual: " + device )
2256 hostAttachment = False
2257 else:
2258 hostAttachment = False
2259 except AssertionError:
2260 main.log.exception( "Json object not as expected" )
2261 main.log.error( repr( host ) )
2262 hostAttachment = False
2263 else:
2264 main.log.error( "No hosts json output or \"Error\"" +
2265 " in output. hosts = " +
2266 repr( hosts[ controller ] ) )
2267 if zeroHosts is False:
2268 hostAttachment = True
2269
2270 # END CHECKING HOST ATTACHMENT POINTS
2271 devicesResults = devicesResults and currentDevicesResult
2272 linksResults = linksResults and currentLinksResult
2273 hostsResults = hostsResults and currentHostsResult
2274 hostAttachmentResults = hostAttachmentResults and\
2275 hostAttachment
2276 topoResult = ( devicesResults and linksResults
2277 and hostsResults and ipResult and
2278 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002279 utilities.assert_equals( expect=True,
2280 actual=topoResult,
2281 onpass="ONOS topology matches Mininet",
2282 onfail="ONOS topology don't match Mininet" )
2283 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002284
2285 # Compare json objects for hosts and dataplane clusters
2286
2287 # hosts
2288 main.step( "Hosts view is consistent across all ONOS nodes" )
2289 consistentHostsResult = main.TRUE
2290 for controller in range( len( hosts ) ):
2291 controllerStr = str( controller + 1 )
Jon Hall657cdf62015-12-17 14:40:51 -08002292 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002293 if hosts[ controller ] == hosts[ 0 ]:
2294 continue
2295 else: # hosts not consistent
2296 main.log.error( "hosts from ONOS" + controllerStr +
2297 " is inconsistent with ONOS1" )
2298 main.log.warn( repr( hosts[ controller ] ) )
2299 consistentHostsResult = main.FALSE
2300
2301 else:
2302 main.log.error( "Error in getting ONOS hosts from ONOS" +
2303 controllerStr )
2304 consistentHostsResult = main.FALSE
2305 main.log.warn( "ONOS" + controllerStr +
2306 " hosts response: " +
2307 repr( hosts[ controller ] ) )
2308 utilities.assert_equals(
2309 expect=main.TRUE,
2310 actual=consistentHostsResult,
2311 onpass="Hosts view is consistent across all ONOS nodes",
2312 onfail="ONOS nodes have different views of hosts" )
2313
2314 main.step( "Hosts information is correct" )
2315 hostsResults = hostsResults and ipResult
2316 utilities.assert_equals(
2317 expect=main.TRUE,
2318 actual=hostsResults,
2319 onpass="Host information is correct",
2320 onfail="Host information is incorrect" )
2321
2322 main.step( "Host attachment points to the network" )
2323 utilities.assert_equals(
2324 expect=True,
2325 actual=hostAttachmentResults,
2326 onpass="Hosts are correctly attached to the network",
2327 onfail="ONOS did not correctly attach hosts to the network" )
2328
2329 # Strongly connected clusters of devices
2330 main.step( "Clusters view is consistent across all ONOS nodes" )
2331 consistentClustersResult = main.TRUE
2332 for controller in range( len( clusters ) ):
2333 controllerStr = str( controller + 1 )
2334 if "Error" not in clusters[ controller ]:
2335 if clusters[ controller ] == clusters[ 0 ]:
2336 continue
2337 else: # clusters not consistent
2338 main.log.error( "clusters from ONOS" +
2339 controllerStr +
2340 " is inconsistent with ONOS1" )
2341 consistentClustersResult = main.FALSE
2342
2343 else:
2344 main.log.error( "Error in getting dataplane clusters " +
2345 "from ONOS" + controllerStr )
2346 consistentClustersResult = main.FALSE
2347 main.log.warn( "ONOS" + controllerStr +
2348 " clusters response: " +
2349 repr( clusters[ controller ] ) )
2350 utilities.assert_equals(
2351 expect=main.TRUE,
2352 actual=consistentClustersResult,
2353 onpass="Clusters view is consistent across all ONOS nodes",
2354 onfail="ONOS nodes have different views of clusters" )
2355
2356 main.step( "There is only one SCC" )
2357 # there should always only be one cluster
2358 try:
2359 numClusters = len( json.loads( clusters[ 0 ] ) )
2360 except ( ValueError, TypeError ):
2361 main.log.exception( "Error parsing clusters[0]: " +
2362 repr( clusters[0] ) )
2363 clusterResults = main.FALSE
2364 if numClusters == 1:
2365 clusterResults = main.TRUE
2366 utilities.assert_equals(
2367 expect=1,
2368 actual=numClusters,
2369 onpass="ONOS shows 1 SCC",
2370 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2371
2372 topoResult = ( devicesResults and linksResults
2373 and hostsResults and consistentHostsResult
2374 and consistentClustersResult and clusterResults
2375 and ipResult and hostAttachmentResults )
2376
2377 topoResult = topoResult and int( count <= 2 )
2378 note = "note it takes about " + str( int( cliTime ) ) + \
2379 " seconds for the test to make all the cli calls to fetch " +\
2380 "the topology from each ONOS instance"
2381 main.log.info(
2382 "Very crass estimate for topology discovery/convergence( " +
2383 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2384 str( count ) + " tries" )
2385
2386 main.step( "Device information is correct" )
2387 utilities.assert_equals(
2388 expect=main.TRUE,
2389 actual=devicesResults,
2390 onpass="Device information is correct",
2391 onfail="Device information is incorrect" )
2392
2393 main.step( "Links are correct" )
2394 utilities.assert_equals(
2395 expect=main.TRUE,
2396 actual=linksResults,
2397 onpass="Link are correct",
2398 onfail="Links are incorrect" )
2399
2400 main.step( "Hosts are correct" )
2401 utilities.assert_equals(
2402 expect=main.TRUE,
2403 actual=hostsResults,
2404 onpass="Hosts are correct",
2405 onfail="Hosts are incorrect" )
2406
2407 # FIXME: move this to an ONOS state case
2408 main.step( "Checking ONOS nodes" )
2409 nodesOutput = []
2410 nodeResults = main.TRUE
2411 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002412 for i in range( main.numCtrls ):
2413 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002414 name="nodes-" + str( i ),
2415 args=[ ] )
2416 threads.append( t )
2417 t.start()
2418
2419 for t in threads:
2420 t.join()
2421 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002422 ips = [ node.ip_address for node in main.nodes ]
Jon Halle9b1fa32015-12-08 15:32:21 -08002423 ips.sort()
Jon Hall5cf14d52015-07-16 12:15:19 -07002424 for i in nodesOutput:
2425 try:
2426 current = json.loads( i )
Jon Halle9b1fa32015-12-08 15:32:21 -08002427 activeIps = []
2428 currentResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002429 for node in current:
Jon Halle9b1fa32015-12-08 15:32:21 -08002430 if node['state'] == 'ACTIVE':
2431 activeIps.append( node['ip'] )
2432 activeIps.sort()
2433 if ips == activeIps:
2434 currentResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002435 except ( ValueError, TypeError ):
2436 main.log.error( "Error parsing nodes output" )
2437 main.log.warn( repr( i ) )
Jon Halle9b1fa32015-12-08 15:32:21 -08002438 currentResult = main.FALSE
2439 nodeResults = nodeResults and currentResult
Jon Hall5cf14d52015-07-16 12:15:19 -07002440 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2441 onpass="Nodes check successful",
2442 onfail="Nodes check NOT successful" )
2443
2444 def CASE9( self, main ):
2445 """
2446 Link s3-s28 down
2447 """
2448 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002449 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002450 assert main, "main not defined"
2451 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002452 assert main.CLIs, "main.CLIs not defined"
2453 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002454 # NOTE: You should probably run a topology check after this
2455
2456 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2457
2458 description = "Turn off a link to ensure that Link Discovery " +\
2459 "is working properly"
2460 main.case( description )
2461
2462 main.step( "Kill Link between s3 and s28" )
2463 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2464 main.log.info( "Waiting " + str( linkSleep ) +
2465 " seconds for link down to be discovered" )
2466 time.sleep( linkSleep )
2467 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2468 onpass="Link down successful",
2469 onfail="Failed to bring link down" )
2470 # TODO do some sort of check here
2471
2472 def CASE10( self, main ):
2473 """
2474 Link s3-s28 up
2475 """
2476 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002477 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002478 assert main, "main not defined"
2479 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002480 assert main.CLIs, "main.CLIs not defined"
2481 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002482 # NOTE: You should probably run a topology check after this
2483
2484 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2485
2486 description = "Restore a link to ensure that Link Discovery is " + \
2487 "working properly"
2488 main.case( description )
2489
2490 main.step( "Bring link between s3 and s28 back up" )
2491 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2492 main.log.info( "Waiting " + str( linkSleep ) +
2493 " seconds for link up to be discovered" )
2494 time.sleep( linkSleep )
2495 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2496 onpass="Link up successful",
2497 onfail="Failed to bring link up" )
2498 # TODO do some sort of check here
2499
2500 def CASE11( self, main ):
2501 """
2502 Switch Down
2503 """
2504 # NOTE: You should probably run a topology check after this
2505 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002506 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002507 assert main, "main not defined"
2508 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002509 assert main.CLIs, "main.CLIs not defined"
2510 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002511
2512 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2513
2514 description = "Killing a switch to ensure it is discovered correctly"
2515 main.case( description )
2516 switch = main.params[ 'kill' ][ 'switch' ]
2517 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2518
2519 # TODO: Make this switch parameterizable
2520 main.step( "Kill " + switch )
2521 main.log.info( "Deleting " + switch )
2522 main.Mininet1.delSwitch( switch )
2523 main.log.info( "Waiting " + str( switchSleep ) +
2524 " seconds for switch down to be discovered" )
2525 time.sleep( switchSleep )
2526 device = main.ONOScli1.getDevice( dpid=switchDPID )
2527 # Peek at the deleted switch
2528 main.log.warn( str( device ) )
2529 result = main.FALSE
2530 if device and device[ 'available' ] is False:
2531 result = main.TRUE
2532 utilities.assert_equals( expect=main.TRUE, actual=result,
2533 onpass="Kill switch successful",
2534 onfail="Failed to kill switch?" )
2535
2536 def CASE12( self, main ):
2537 """
2538 Switch Up
2539 """
2540 # NOTE: You should probably run a topology check after this
2541 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002542 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002543 assert main, "main not defined"
2544 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002545 assert main.CLIs, "main.CLIs not defined"
2546 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002547 assert ONOS1Port, "ONOS1Port not defined"
2548 assert ONOS2Port, "ONOS2Port not defined"
2549 assert ONOS3Port, "ONOS3Port not defined"
2550 assert ONOS4Port, "ONOS4Port not defined"
2551 assert ONOS5Port, "ONOS5Port not defined"
2552 assert ONOS6Port, "ONOS6Port not defined"
2553 assert ONOS7Port, "ONOS7Port not defined"
2554
2555 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2556 switch = main.params[ 'kill' ][ 'switch' ]
2557 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2558 links = main.params[ 'kill' ][ 'links' ].split()
2559 description = "Adding a switch to ensure it is discovered correctly"
2560 main.case( description )
2561
2562 main.step( "Add back " + switch )
2563 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2564 for peer in links:
2565 main.Mininet1.addLink( switch, peer )
2566 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002567 for i in range( main.numCtrls ):
2568 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002569 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2570 main.log.info( "Waiting " + str( switchSleep ) +
2571 " seconds for switch up to be discovered" )
2572 time.sleep( switchSleep )
2573 device = main.ONOScli1.getDevice( dpid=switchDPID )
2574 # Peek at the deleted switch
2575 main.log.warn( str( device ) )
2576 result = main.FALSE
2577 if device and device[ 'available' ]:
2578 result = main.TRUE
2579 utilities.assert_equals( expect=main.TRUE, actual=result,
2580 onpass="add switch successful",
2581 onfail="Failed to add switch?" )
2582
2583 def CASE13( self, main ):
2584 """
2585 Clean up
2586 """
2587 import os
2588 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002589 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002590 assert main, "main not defined"
2591 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002592 assert main.CLIs, "main.CLIs not defined"
2593 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002594
2595 # printing colors to terminal
2596 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2597 'blue': '\033[94m', 'green': '\033[92m',
2598 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2599 main.case( "Test Cleanup" )
2600 main.step( "Killing tcpdumps" )
2601 main.Mininet2.stopTcpdump()
2602
2603 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002604 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002605 main.step( "Copying MN pcap and ONOS log files to test station" )
2606 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2607 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002608 # NOTE: MN Pcap file is being saved to logdir.
2609 # We scp this file as MN and TestON aren't necessarily the same vm
2610
2611 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002612 # TODO: Load these from params
2613 # NOTE: must end in /
2614 logFolder = "/opt/onos/log/"
2615 logFiles = [ "karaf.log", "karaf.log.1" ]
2616 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002617 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002618 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002619 dstName = main.logdir + "/" + node.name + "-" + f
2620 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2621 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002622 # std*.log's
2623 # NOTE: must end in /
2624 logFolder = "/opt/onos/var/"
2625 logFiles = [ "stderr.log", "stdout.log" ]
2626 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002627 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002628 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002629 dstName = main.logdir + "/" + node.name + "-" + f
2630 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2631 logFolder + f, dstName )
2632 else:
2633 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002634
2635 main.step( "Stopping Mininet" )
2636 mnResult = main.Mininet1.stopNet()
2637 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2638 onpass="Mininet stopped",
2639 onfail="MN cleanup NOT successful" )
2640
2641 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002642 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002643 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2644 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002645
2646 try:
2647 timerLog = open( main.logdir + "/Timers.csv", 'w')
2648 # Overwrite with empty line and close
2649 labels = "Gossip Intents"
2650 data = str( gossipTime )
2651 timerLog.write( labels + "\n" + data )
2652 timerLog.close()
2653 except NameError, e:
2654 main.log.exception(e)
2655
2656 def CASE14( self, main ):
2657 """
2658 start election app on all onos nodes
2659 """
Jon Halle1a3b752015-07-22 13:02:46 -07002660 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002661 assert main, "main not defined"
2662 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002663 assert main.CLIs, "main.CLIs not defined"
2664 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002665
2666 main.case("Start Leadership Election app")
2667 main.step( "Install leadership election app" )
2668 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2669 utilities.assert_equals(
2670 expect=main.TRUE,
2671 actual=appResult,
2672 onpass="Election app installed",
2673 onfail="Something went wrong with installing Leadership election" )
2674
2675 main.step( "Run for election on each node" )
2676 leaderResult = main.TRUE
2677 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002678 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002679 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002680 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002681 leader = cli.electionTestLeader()
2682 if leader is None or leader == main.FALSE:
2683 main.log.error( cli.name + ": Leader for the election app " +
2684 "should be an ONOS node, instead got '" +
2685 str( leader ) + "'" )
2686 leaderResult = main.FALSE
2687 leaders.append( leader )
2688 utilities.assert_equals(
2689 expect=main.TRUE,
2690 actual=leaderResult,
2691 onpass="Successfully ran for leadership",
2692 onfail="Failed to run for leadership" )
2693
2694 main.step( "Check that each node shows the same leader" )
2695 sameLeader = main.TRUE
2696 if len( set( leaders ) ) != 1:
2697 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002698 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002699 str( leaders ) )
2700 utilities.assert_equals(
2701 expect=main.TRUE,
2702 actual=sameLeader,
2703 onpass="Leadership is consistent for the election topic",
2704 onfail="Nodes have different leaders" )
2705
2706 def CASE15( self, main ):
2707 """
2708 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002709 15.1 Run election on each node
2710 15.2 Check that each node has the same leaders and candidates
2711 15.3 Find current leader and withdraw
2712 15.4 Check that a new node was elected leader
2713 15.5 Check that that new leader was the candidate of old leader
2714 15.6 Run for election on old leader
2715 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2716 15.8 Make sure that the old leader was added to the candidate list
2717
2718 old and new variable prefixes refer to data from before vs after
2719 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002720 """
2721 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002722 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002723 assert main, "main not defined"
2724 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002725 assert main.CLIs, "main.CLIs not defined"
2726 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002727
acsmars3a72bde2015-09-02 14:16:22 -07002728 description = "Check that Leadership Election App is still functional"
Jon Hall5cf14d52015-07-16 12:15:19 -07002729 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07002730 # NOTE: Need to re-run since being a canidate is not persistant
2731 # TODO: add check for "Command not found:" in the driver, this
2732 # means the election test app isn't loaded
Jon Hall5cf14d52015-07-16 12:15:19 -07002733
acsmars71adceb2015-08-31 15:09:26 -07002734 oldLeaders = [] # leaders by node before withdrawl from candidates
2735 newLeaders = [] # leaders by node after withdrawl from candidates
2736 oldAllCandidates = [] # list of lists of each nodes' candidates before
2737 newAllCandidates = [] # list of lists of each nodes' candidates after
2738 oldCandidates = [] # list of candidates from node 0 before withdrawl
2739 newCandidates = [] # list of candidates from node 0 after withdrawl
2740 oldLeader = '' # the old leader from oldLeaders, None if not same
2741 newLeader = '' # the new leaders fron newLoeaders, None if not same
2742 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2743 expectNoLeader = False # True when there is only one leader
2744 if main.numCtrls == 1:
2745 expectNoLeader = True
2746
2747 main.step( "Run for election on each node" )
2748 electionResult = main.TRUE
2749
2750 for cli in main.CLIs: # run test election on each node
2751 if cli.electionTestRun() == main.FALSE:
2752 electionResult = main.FALSE
2753
Jon Hall5cf14d52015-07-16 12:15:19 -07002754 utilities.assert_equals(
2755 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002756 actual=electionResult,
2757 onpass="All nodes successfully ran for leadership",
2758 onfail="At least one node failed to run for leadership" )
2759
acsmars3a72bde2015-09-02 14:16:22 -07002760 if electionResult == main.FALSE:
2761 main.log.error(
2762 "Skipping Test Case because Election Test isn't loaded" )
2763 main.skipCase()
2764
acsmars71adceb2015-08-31 15:09:26 -07002765 main.step( "Check that each node shows the same leader and candidates" )
2766 sameResult = main.TRUE
2767 failMessage = "Nodes have different leaders"
2768 for cli in main.CLIs:
2769 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2770 oldAllCandidates.append( node )
2771 oldLeaders.append( node[ 0 ] )
2772 oldCandidates = oldAllCandidates[ 0 ]
2773
2774 # Check that each node has the same leader. Defines oldLeader
2775 if len( set( oldLeaders ) ) != 1:
2776 sameResult = main.FALSE
2777 main.log.error( "More than one leader present:" + str( oldLeaders ) )
2778 oldLeader = None
2779 else:
2780 oldLeader = oldLeaders[ 0 ]
2781
2782 # Check that each node's candidate list is the same
acsmars29233db2015-11-04 11:15:00 -08002783 candidateDiscrepancy = False # Boolean of candidate mismatches
acsmars71adceb2015-08-31 15:09:26 -07002784 for candidates in oldAllCandidates:
2785 if set( candidates ) != set( oldCandidates ):
2786 sameResult = main.FALSE
acsmars29233db2015-11-04 11:15:00 -08002787 candidateDiscrepancy = True
2788
2789 if candidateDiscrepancy:
2790 failMessage += " and candidates"
acsmars71adceb2015-08-31 15:09:26 -07002791
2792 utilities.assert_equals(
2793 expect=main.TRUE,
2794 actual=sameResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002795 onpass="Leadership is consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002796 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002797
2798 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002799 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002800 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002801 if oldLeader is None:
2802 main.log.error( "Leadership isn't consistent." )
2803 withdrawResult = main.FALSE
2804 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002805 for i in range( len( main.CLIs ) ):
acsmars71adceb2015-08-31 15:09:26 -07002806 if oldLeader == main.nodes[ i ].ip_address:
2807 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002808 break
2809 else: # FOR/ELSE statement
2810 main.log.error( "Leader election, could not find current leader" )
2811 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002812 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002813 utilities.assert_equals(
2814 expect=main.TRUE,
2815 actual=withdrawResult,
2816 onpass="Node was withdrawn from election",
2817 onfail="Node was not withdrawn from election" )
2818
acsmars71adceb2015-08-31 15:09:26 -07002819 main.step( "Check that a new node was elected leader" )
2820
Jon Hall5cf14d52015-07-16 12:15:19 -07002821 # FIXME: use threads
acsmars71adceb2015-08-31 15:09:26 -07002822 newLeaderResult = main.TRUE
2823 failMessage = "Nodes have different leaders"
2824
2825 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002826 for cli in main.CLIs:
acsmars71adceb2015-08-31 15:09:26 -07002827 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2828 # elections might no have finished yet
2829 if node[ 0 ] == 'none' and not expectNoLeader:
2830 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2831 "sure elections are complete." )
2832 time.sleep(5)
2833 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2834 # election still isn't done or there is a problem
2835 if node[ 0 ] == 'none':
2836 main.log.error( "No leader was elected on at least 1 node" )
2837 newLeaderResult = main.FALSE
2838 newAllCandidates.append( node )
2839 newLeaders.append( node[ 0 ] )
2840 newCandidates = newAllCandidates[ 0 ]
2841
2842 # Check that each node has the same leader. Defines newLeader
2843 if len( set( newLeaders ) ) != 1:
2844 newLeaderResult = main.FALSE
2845 main.log.error( "Nodes have different leaders: " +
2846 str( newLeaders ) )
2847 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002848 else:
acsmars71adceb2015-08-31 15:09:26 -07002849 newLeader = newLeaders[ 0 ]
2850
2851 # Check that each node's candidate list is the same
2852 for candidates in newAllCandidates:
2853 if set( candidates ) != set( newCandidates ):
2854 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002855 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002856
2857 # Check that the new leader is not the older leader, which was withdrawn
2858 if newLeader == oldLeader:
2859 newLeaderResult = main.FALSE
2860 main.log.error( "All nodes still see old leader: " + oldLeader +
2861 " as the current leader" )
2862
Jon Hall5cf14d52015-07-16 12:15:19 -07002863 utilities.assert_equals(
2864 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002865 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002866 onpass="Leadership election passed",
2867 onfail="Something went wrong with Leadership election" )
2868
acsmars71adceb2015-08-31 15:09:26 -07002869 main.step( "Check that that new leader was the candidate of old leader")
2870 # candidates[ 2 ] should be come the top candidate after withdrawl
2871 correctCandidateResult = main.TRUE
2872 if expectNoLeader:
2873 if newLeader == 'none':
2874 main.log.info( "No leader expected. None found. Pass" )
2875 correctCandidateResult = main.TRUE
2876 else:
2877 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2878 correctCandidateResult = main.FALSE
2879 elif newLeader != oldCandidates[ 2 ]:
2880 correctCandidateResult = main.FALSE
2881 main.log.error( "Candidate " + newLeader + " was elected. " +
2882 oldCandidates[ 2 ] + " should have had priority." )
2883
2884 utilities.assert_equals(
2885 expect=main.TRUE,
2886 actual=correctCandidateResult,
2887 onpass="Correct Candidate Elected",
2888 onfail="Incorrect Candidate Elected" )
2889
Jon Hall5cf14d52015-07-16 12:15:19 -07002890 main.step( "Run for election on old leader( just so everyone " +
2891 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002892 if oldLeaderCLI is not None:
2893 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002894 else:
acsmars71adceb2015-08-31 15:09:26 -07002895 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002896 runResult = main.FALSE
2897 utilities.assert_equals(
2898 expect=main.TRUE,
2899 actual=runResult,
2900 onpass="App re-ran for election",
2901 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002902 main.step(
2903 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002904 # verify leader didn't just change
acsmars71adceb2015-08-31 15:09:26 -07002905 positionResult = main.TRUE
2906 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
2907
2908 # Reset and reuse the new candidate and leaders lists
2909 newAllCandidates = []
2910 newCandidates = []
2911 newLeaders = []
2912 for cli in main.CLIs:
2913 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2914 if oldLeader not in node: # election might no have finished yet
2915 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2916 "be sure elections are complete" )
2917 time.sleep(5)
2918 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2919 if oldLeader not in node: # election still isn't done, errors
2920 main.log.error(
2921 "Old leader was not elected on at least one node" )
2922 positionResult = main.FALSE
2923 newAllCandidates.append( node )
2924 newLeaders.append( node[ 0 ] )
2925 newCandidates = newAllCandidates[ 0 ]
2926
2927 # Check that each node has the same leader. Defines newLeader
2928 if len( set( newLeaders ) ) != 1:
2929 positionResult = main.FALSE
2930 main.log.error( "Nodes have different leaders: " +
2931 str( newLeaders ) )
2932 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002933 else:
acsmars71adceb2015-08-31 15:09:26 -07002934 newLeader = newLeaders[ 0 ]
2935
2936 # Check that each node's candidate list is the same
2937 for candidates in newAllCandidates:
2938 if set( candidates ) != set( newCandidates ):
2939 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002940 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002941
2942 # Check that the re-elected node is last on the candidate List
2943 if oldLeader != newCandidates[ -1 ]:
2944 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
2945 str( newCandidates ) )
2946 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002947
2948 utilities.assert_equals(
2949 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002950 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002951 onpass="Old leader successfully re-ran for election",
2952 onfail="Something went wrong with Leadership election after " +
2953 "the old leader re-ran for election" )
2954
2955 def CASE16( self, main ):
2956 """
2957 Install Distributed Primitives app
2958 """
2959 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002960 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002961 assert main, "main not defined"
2962 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002963 assert main.CLIs, "main.CLIs not defined"
2964 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002965
2966 # Variables for the distributed primitives tests
2967 global pCounterName
2968 global iCounterName
2969 global pCounterValue
2970 global iCounterValue
2971 global onosSet
2972 global onosSetName
2973 pCounterName = "TestON-Partitions"
2974 iCounterName = "TestON-inMemory"
2975 pCounterValue = 0
2976 iCounterValue = 0
2977 onosSet = set([])
2978 onosSetName = "TestON-set"
2979
2980 description = "Install Primitives app"
2981 main.case( description )
2982 main.step( "Install Primitives app" )
2983 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002984 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002985 utilities.assert_equals( expect=main.TRUE,
2986 actual=appResults,
2987 onpass="Primitives app activated",
2988 onfail="Primitives app not activated" )
2989 time.sleep( 5 ) # To allow all nodes to activate
2990
2991 def CASE17( self, main ):
2992 """
2993 Check for basic functionality with distributed primitives
2994 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002995 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002996 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002997 assert main, "main not defined"
2998 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002999 assert main.CLIs, "main.CLIs not defined"
3000 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003001 assert pCounterName, "pCounterName not defined"
3002 assert iCounterName, "iCounterName not defined"
3003 assert onosSetName, "onosSetName not defined"
3004 # NOTE: assert fails if value is 0/None/Empty/False
3005 try:
3006 pCounterValue
3007 except NameError:
3008 main.log.error( "pCounterValue not defined, setting to 0" )
3009 pCounterValue = 0
3010 try:
3011 iCounterValue
3012 except NameError:
3013 main.log.error( "iCounterValue not defined, setting to 0" )
3014 iCounterValue = 0
3015 try:
3016 onosSet
3017 except NameError:
3018 main.log.error( "onosSet not defined, setting to empty Set" )
3019 onosSet = set([])
3020 # Variables for the distributed primitives tests. These are local only
3021 addValue = "a"
3022 addAllValue = "a b c d e f"
3023 retainValue = "c d e f"
3024
3025 description = "Check for basic functionality with distributed " +\
3026 "primitives"
3027 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003028 main.caseExplanation = "Test the methods of the distributed " +\
3029 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003030 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003031 # Partitioned counters
3032 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003033 pCounters = []
3034 threads = []
3035 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003036 for i in range( main.numCtrls ):
3037 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3038 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003039 args=[ pCounterName ] )
3040 pCounterValue += 1
3041 addedPValues.append( pCounterValue )
3042 threads.append( t )
3043 t.start()
3044
3045 for t in threads:
3046 t.join()
3047 pCounters.append( t.result )
3048 # Check that counter incremented numController times
3049 pCounterResults = True
3050 for i in addedPValues:
3051 tmpResult = i in pCounters
3052 pCounterResults = pCounterResults and tmpResult
3053 if not tmpResult:
3054 main.log.error( str( i ) + " is not in partitioned "
3055 "counter incremented results" )
3056 utilities.assert_equals( expect=True,
3057 actual=pCounterResults,
3058 onpass="Default counter incremented",
3059 onfail="Error incrementing default" +
3060 " counter" )
3061
Jon Halle1a3b752015-07-22 13:02:46 -07003062 main.step( "Get then Increment a default counter on each node" )
3063 pCounters = []
3064 threads = []
3065 addedPValues = []
3066 for i in range( main.numCtrls ):
3067 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3068 name="counterGetAndAdd-" + str( i ),
3069 args=[ pCounterName ] )
3070 addedPValues.append( pCounterValue )
3071 pCounterValue += 1
3072 threads.append( t )
3073 t.start()
3074
3075 for t in threads:
3076 t.join()
3077 pCounters.append( t.result )
3078 # Check that counter incremented numController times
3079 pCounterResults = True
3080 for i in addedPValues:
3081 tmpResult = i in pCounters
3082 pCounterResults = pCounterResults and tmpResult
3083 if not tmpResult:
3084 main.log.error( str( i ) + " is not in partitioned "
3085 "counter incremented results" )
3086 utilities.assert_equals( expect=True,
3087 actual=pCounterResults,
3088 onpass="Default counter incremented",
3089 onfail="Error incrementing default" +
3090 " counter" )
3091
3092 main.step( "Counters we added have the correct values" )
3093 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3094 utilities.assert_equals( expect=main.TRUE,
3095 actual=incrementCheck,
3096 onpass="Added counters are correct",
3097 onfail="Added counters are incorrect" )
3098
3099 main.step( "Add -8 to then get a default counter on each node" )
3100 pCounters = []
3101 threads = []
3102 addedPValues = []
3103 for i in range( main.numCtrls ):
3104 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3105 name="counterIncrement-" + str( i ),
3106 args=[ pCounterName ],
3107 kwargs={ "delta": -8 } )
3108 pCounterValue += -8
3109 addedPValues.append( pCounterValue )
3110 threads.append( t )
3111 t.start()
3112
3113 for t in threads:
3114 t.join()
3115 pCounters.append( t.result )
3116 # Check that counter incremented numController times
3117 pCounterResults = True
3118 for i in addedPValues:
3119 tmpResult = i in pCounters
3120 pCounterResults = pCounterResults and tmpResult
3121 if not tmpResult:
3122 main.log.error( str( i ) + " is not in partitioned "
3123 "counter incremented results" )
3124 utilities.assert_equals( expect=True,
3125 actual=pCounterResults,
3126 onpass="Default counter incremented",
3127 onfail="Error incrementing default" +
3128 " counter" )
3129
3130 main.step( "Add 5 to then get a default counter on each node" )
3131 pCounters = []
3132 threads = []
3133 addedPValues = []
3134 for i in range( main.numCtrls ):
3135 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3136 name="counterIncrement-" + str( i ),
3137 args=[ pCounterName ],
3138 kwargs={ "delta": 5 } )
3139 pCounterValue += 5
3140 addedPValues.append( pCounterValue )
3141 threads.append( t )
3142 t.start()
3143
3144 for t in threads:
3145 t.join()
3146 pCounters.append( t.result )
3147 # Check that counter incremented numController times
3148 pCounterResults = True
3149 for i in addedPValues:
3150 tmpResult = i in pCounters
3151 pCounterResults = pCounterResults and tmpResult
3152 if not tmpResult:
3153 main.log.error( str( i ) + " is not in partitioned "
3154 "counter incremented results" )
3155 utilities.assert_equals( expect=True,
3156 actual=pCounterResults,
3157 onpass="Default counter incremented",
3158 onfail="Error incrementing default" +
3159 " counter" )
3160
3161 main.step( "Get then add 5 to a default counter on each node" )
3162 pCounters = []
3163 threads = []
3164 addedPValues = []
3165 for i in range( main.numCtrls ):
3166 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3167 name="counterIncrement-" + str( i ),
3168 args=[ pCounterName ],
3169 kwargs={ "delta": 5 } )
3170 addedPValues.append( pCounterValue )
3171 pCounterValue += 5
3172 threads.append( t )
3173 t.start()
3174
3175 for t in threads:
3176 t.join()
3177 pCounters.append( t.result )
3178 # Check that counter incremented numController times
3179 pCounterResults = True
3180 for i in addedPValues:
3181 tmpResult = i in pCounters
3182 pCounterResults = pCounterResults and tmpResult
3183 if not tmpResult:
3184 main.log.error( str( i ) + " is not in partitioned "
3185 "counter incremented results" )
3186 utilities.assert_equals( expect=True,
3187 actual=pCounterResults,
3188 onpass="Default counter incremented",
3189 onfail="Error incrementing default" +
3190 " counter" )
3191
3192 main.step( "Counters we added have the correct values" )
3193 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3194 utilities.assert_equals( expect=main.TRUE,
3195 actual=incrementCheck,
3196 onpass="Added counters are correct",
3197 onfail="Added counters are incorrect" )
3198
3199 # In-Memory counters
3200 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003201 iCounters = []
3202 addedIValues = []
3203 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003204 for i in range( main.numCtrls ):
3205 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003206 name="icounterIncrement-" + str( i ),
3207 args=[ iCounterName ],
3208 kwargs={ "inMemory": True } )
3209 iCounterValue += 1
3210 addedIValues.append( iCounterValue )
3211 threads.append( t )
3212 t.start()
3213
3214 for t in threads:
3215 t.join()
3216 iCounters.append( t.result )
3217 # Check that counter incremented numController times
3218 iCounterResults = True
3219 for i in addedIValues:
3220 tmpResult = i in iCounters
3221 iCounterResults = iCounterResults and tmpResult
3222 if not tmpResult:
3223 main.log.error( str( i ) + " is not in the in-memory "
3224 "counter incremented results" )
3225 utilities.assert_equals( expect=True,
3226 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003227 onpass="In-memory counter incremented",
3228 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003229 " counter" )
3230
Jon Halle1a3b752015-07-22 13:02:46 -07003231 main.step( "Get then Increment a in-memory counter on each node" )
3232 iCounters = []
3233 threads = []
3234 addedIValues = []
3235 for i in range( main.numCtrls ):
3236 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3237 name="counterGetAndAdd-" + str( i ),
3238 args=[ iCounterName ],
3239 kwargs={ "inMemory": True } )
3240 addedIValues.append( iCounterValue )
3241 iCounterValue += 1
3242 threads.append( t )
3243 t.start()
3244
3245 for t in threads:
3246 t.join()
3247 iCounters.append( t.result )
3248 # Check that counter incremented numController times
3249 iCounterResults = True
3250 for i in addedIValues:
3251 tmpResult = i in iCounters
3252 iCounterResults = iCounterResults and tmpResult
3253 if not tmpResult:
3254 main.log.error( str( i ) + " is not in in-memory "
3255 "counter incremented results" )
3256 utilities.assert_equals( expect=True,
3257 actual=iCounterResults,
3258 onpass="In-memory counter incremented",
3259 onfail="Error incrementing in-memory" +
3260 " counter" )
3261
3262 main.step( "Counters we added have the correct values" )
3263 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3264 utilities.assert_equals( expect=main.TRUE,
3265 actual=incrementCheck,
3266 onpass="Added counters are correct",
3267 onfail="Added counters are incorrect" )
3268
3269 main.step( "Add -8 to then get a in-memory counter on each node" )
3270 iCounters = []
3271 threads = []
3272 addedIValues = []
3273 for i in range( main.numCtrls ):
3274 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3275 name="counterIncrement-" + str( i ),
3276 args=[ iCounterName ],
3277 kwargs={ "delta": -8, "inMemory": True } )
3278 iCounterValue += -8
3279 addedIValues.append( iCounterValue )
3280 threads.append( t )
3281 t.start()
3282
3283 for t in threads:
3284 t.join()
3285 iCounters.append( t.result )
3286 # Check that counter incremented numController times
3287 iCounterResults = True
3288 for i in addedIValues:
3289 tmpResult = i in iCounters
3290 iCounterResults = iCounterResults and tmpResult
3291 if not tmpResult:
3292 main.log.error( str( i ) + " is not in in-memory "
3293 "counter incremented results" )
3294 utilities.assert_equals( expect=True,
3295 actual=pCounterResults,
3296 onpass="In-memory counter incremented",
3297 onfail="Error incrementing in-memory" +
3298 " counter" )
3299
3300 main.step( "Add 5 to then get a in-memory counter on each node" )
3301 iCounters = []
3302 threads = []
3303 addedIValues = []
3304 for i in range( main.numCtrls ):
3305 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3306 name="counterIncrement-" + str( i ),
3307 args=[ iCounterName ],
3308 kwargs={ "delta": 5, "inMemory": True } )
3309 iCounterValue += 5
3310 addedIValues.append( iCounterValue )
3311 threads.append( t )
3312 t.start()
3313
3314 for t in threads:
3315 t.join()
3316 iCounters.append( t.result )
3317 # Check that counter incremented numController times
3318 iCounterResults = True
3319 for i in addedIValues:
3320 tmpResult = i in iCounters
3321 iCounterResults = iCounterResults and tmpResult
3322 if not tmpResult:
3323 main.log.error( str( i ) + " is not in in-memory "
3324 "counter incremented results" )
3325 utilities.assert_equals( expect=True,
3326 actual=pCounterResults,
3327 onpass="In-memory counter incremented",
3328 onfail="Error incrementing in-memory" +
3329 " counter" )
3330
3331 main.step( "Get then add 5 to a in-memory counter on each node" )
3332 iCounters = []
3333 threads = []
3334 addedIValues = []
3335 for i in range( main.numCtrls ):
3336 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3337 name="counterIncrement-" + str( i ),
3338 args=[ iCounterName ],
3339 kwargs={ "delta": 5, "inMemory": True } )
3340 addedIValues.append( iCounterValue )
3341 iCounterValue += 5
3342 threads.append( t )
3343 t.start()
3344
3345 for t in threads:
3346 t.join()
3347 iCounters.append( t.result )
3348 # Check that counter incremented numController times
3349 iCounterResults = True
3350 for i in addedIValues:
3351 tmpResult = i in iCounters
3352 iCounterResults = iCounterResults and tmpResult
3353 if not tmpResult:
3354 main.log.error( str( i ) + " is not in in-memory "
3355 "counter incremented results" )
3356 utilities.assert_equals( expect=True,
3357 actual=iCounterResults,
3358 onpass="In-memory counter incremented",
3359 onfail="Error incrementing in-memory" +
3360 " counter" )
3361
3362 main.step( "Counters we added have the correct values" )
3363 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3364 utilities.assert_equals( expect=main.TRUE,
3365 actual=incrementCheck,
3366 onpass="Added counters are correct",
3367 onfail="Added counters are incorrect" )
3368
Jon Hall5cf14d52015-07-16 12:15:19 -07003369 main.step( "Check counters are consistant across nodes" )
Jon Hall57b50432015-10-22 10:20:10 -07003370 onosCounters, consistentCounterResults = main.Counters.consistentCheck()
Jon Hall5cf14d52015-07-16 12:15:19 -07003371 utilities.assert_equals( expect=main.TRUE,
3372 actual=consistentCounterResults,
3373 onpass="ONOS counters are consistent " +
3374 "across nodes",
3375 onfail="ONOS Counters are inconsistent " +
3376 "across nodes" )
3377
3378 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003379 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3380 incrementCheck = incrementCheck and \
3381 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003382 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003383 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003384 onpass="Added counters are correct",
3385 onfail="Added counters are incorrect" )
3386 # DISTRIBUTED SETS
3387 main.step( "Distributed Set get" )
3388 size = len( onosSet )
3389 getResponses = []
3390 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003391 for i in range( main.numCtrls ):
3392 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003393 name="setTestGet-" + str( i ),
3394 args=[ onosSetName ] )
3395 threads.append( t )
3396 t.start()
3397 for t in threads:
3398 t.join()
3399 getResponses.append( t.result )
3400
3401 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003402 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003403 if isinstance( getResponses[ i ], list):
3404 current = set( getResponses[ i ] )
3405 if len( current ) == len( getResponses[ i ] ):
3406 # no repeats
3407 if onosSet != current:
3408 main.log.error( "ONOS" + str( i + 1 ) +
3409 " has incorrect view" +
3410 " of set " + onosSetName + ":\n" +
3411 str( getResponses[ i ] ) )
3412 main.log.debug( "Expected: " + str( onosSet ) )
3413 main.log.debug( "Actual: " + str( current ) )
3414 getResults = main.FALSE
3415 else:
3416 # error, set is not a set
3417 main.log.error( "ONOS" + str( i + 1 ) +
3418 " has repeat elements in" +
3419 " set " + onosSetName + ":\n" +
3420 str( getResponses[ i ] ) )
3421 getResults = main.FALSE
3422 elif getResponses[ i ] == main.ERROR:
3423 getResults = main.FALSE
3424 utilities.assert_equals( expect=main.TRUE,
3425 actual=getResults,
3426 onpass="Set elements are correct",
3427 onfail="Set elements are incorrect" )
3428
3429 main.step( "Distributed Set size" )
3430 sizeResponses = []
3431 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003432 for i in range( main.numCtrls ):
3433 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003434 name="setTestSize-" + str( i ),
3435 args=[ onosSetName ] )
3436 threads.append( t )
3437 t.start()
3438 for t in threads:
3439 t.join()
3440 sizeResponses.append( t.result )
3441
3442 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003443 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003444 if size != sizeResponses[ i ]:
3445 sizeResults = main.FALSE
3446 main.log.error( "ONOS" + str( i + 1 ) +
3447 " expected a size of " + str( size ) +
3448 " for set " + onosSetName +
3449 " but got " + str( sizeResponses[ i ] ) )
3450 utilities.assert_equals( expect=main.TRUE,
3451 actual=sizeResults,
3452 onpass="Set sizes are correct",
3453 onfail="Set sizes are incorrect" )
3454
3455 main.step( "Distributed Set add()" )
3456 onosSet.add( addValue )
3457 addResponses = []
3458 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003459 for i in range( main.numCtrls ):
3460 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003461 name="setTestAdd-" + str( i ),
3462 args=[ onosSetName, addValue ] )
3463 threads.append( t )
3464 t.start()
3465 for t in threads:
3466 t.join()
3467 addResponses.append( t.result )
3468
3469 # main.TRUE = successfully changed the set
3470 # main.FALSE = action resulted in no change in set
3471 # main.ERROR - Some error in executing the function
3472 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003473 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003474 if addResponses[ i ] == main.TRUE:
3475 # All is well
3476 pass
3477 elif addResponses[ i ] == main.FALSE:
3478 # Already in set, probably fine
3479 pass
3480 elif addResponses[ i ] == main.ERROR:
3481 # Error in execution
3482 addResults = main.FALSE
3483 else:
3484 # unexpected result
3485 addResults = main.FALSE
3486 if addResults != main.TRUE:
3487 main.log.error( "Error executing set add" )
3488
3489 # Check if set is still correct
3490 size = len( onosSet )
3491 getResponses = []
3492 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003493 for i in range( main.numCtrls ):
3494 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003495 name="setTestGet-" + str( i ),
3496 args=[ onosSetName ] )
3497 threads.append( t )
3498 t.start()
3499 for t in threads:
3500 t.join()
3501 getResponses.append( t.result )
3502 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003503 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003504 if isinstance( getResponses[ i ], list):
3505 current = set( getResponses[ i ] )
3506 if len( current ) == len( getResponses[ i ] ):
3507 # no repeats
3508 if onosSet != current:
3509 main.log.error( "ONOS" + str( i + 1 ) +
3510 " has incorrect view" +
3511 " of set " + onosSetName + ":\n" +
3512 str( getResponses[ i ] ) )
3513 main.log.debug( "Expected: " + str( onosSet ) )
3514 main.log.debug( "Actual: " + str( current ) )
3515 getResults = main.FALSE
3516 else:
3517 # error, set is not a set
3518 main.log.error( "ONOS" + str( i + 1 ) +
3519 " has repeat elements in" +
3520 " set " + onosSetName + ":\n" +
3521 str( getResponses[ i ] ) )
3522 getResults = main.FALSE
3523 elif getResponses[ i ] == main.ERROR:
3524 getResults = main.FALSE
3525 sizeResponses = []
3526 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003527 for i in range( main.numCtrls ):
3528 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003529 name="setTestSize-" + str( i ),
3530 args=[ onosSetName ] )
3531 threads.append( t )
3532 t.start()
3533 for t in threads:
3534 t.join()
3535 sizeResponses.append( t.result )
3536 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003537 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003538 if size != sizeResponses[ i ]:
3539 sizeResults = main.FALSE
3540 main.log.error( "ONOS" + str( i + 1 ) +
3541 " expected a size of " + str( size ) +
3542 " for set " + onosSetName +
3543 " but got " + str( sizeResponses[ i ] ) )
3544 addResults = addResults and getResults and sizeResults
3545 utilities.assert_equals( expect=main.TRUE,
3546 actual=addResults,
3547 onpass="Set add correct",
3548 onfail="Set add was incorrect" )
3549
3550 main.step( "Distributed Set addAll()" )
3551 onosSet.update( addAllValue.split() )
3552 addResponses = []
3553 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003554 for i in range( main.numCtrls ):
3555 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003556 name="setTestAddAll-" + str( i ),
3557 args=[ onosSetName, addAllValue ] )
3558 threads.append( t )
3559 t.start()
3560 for t in threads:
3561 t.join()
3562 addResponses.append( t.result )
3563
3564 # main.TRUE = successfully changed the set
3565 # main.FALSE = action resulted in no change in set
3566 # main.ERROR - Some error in executing the function
3567 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003568 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003569 if addResponses[ i ] == main.TRUE:
3570 # All is well
3571 pass
3572 elif addResponses[ i ] == main.FALSE:
3573 # Already in set, probably fine
3574 pass
3575 elif addResponses[ i ] == main.ERROR:
3576 # Error in execution
3577 addAllResults = main.FALSE
3578 else:
3579 # unexpected result
3580 addAllResults = main.FALSE
3581 if addAllResults != main.TRUE:
3582 main.log.error( "Error executing set addAll" )
3583
3584 # Check if set is still correct
3585 size = len( onosSet )
3586 getResponses = []
3587 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003588 for i in range( main.numCtrls ):
3589 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003590 name="setTestGet-" + str( i ),
3591 args=[ onosSetName ] )
3592 threads.append( t )
3593 t.start()
3594 for t in threads:
3595 t.join()
3596 getResponses.append( t.result )
3597 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003598 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003599 if isinstance( getResponses[ i ], list):
3600 current = set( getResponses[ i ] )
3601 if len( current ) == len( getResponses[ i ] ):
3602 # no repeats
3603 if onosSet != current:
3604 main.log.error( "ONOS" + str( i + 1 ) +
3605 " has incorrect view" +
3606 " of set " + onosSetName + ":\n" +
3607 str( getResponses[ i ] ) )
3608 main.log.debug( "Expected: " + str( onosSet ) )
3609 main.log.debug( "Actual: " + str( current ) )
3610 getResults = main.FALSE
3611 else:
3612 # error, set is not a set
3613 main.log.error( "ONOS" + str( i + 1 ) +
3614 " has repeat elements in" +
3615 " set " + onosSetName + ":\n" +
3616 str( getResponses[ i ] ) )
3617 getResults = main.FALSE
3618 elif getResponses[ i ] == main.ERROR:
3619 getResults = main.FALSE
3620 sizeResponses = []
3621 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003622 for i in range( main.numCtrls ):
3623 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003624 name="setTestSize-" + str( i ),
3625 args=[ onosSetName ] )
3626 threads.append( t )
3627 t.start()
3628 for t in threads:
3629 t.join()
3630 sizeResponses.append( t.result )
3631 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003632 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003633 if size != sizeResponses[ i ]:
3634 sizeResults = main.FALSE
3635 main.log.error( "ONOS" + str( i + 1 ) +
3636 " expected a size of " + str( size ) +
3637 " for set " + onosSetName +
3638 " but got " + str( sizeResponses[ i ] ) )
3639 addAllResults = addAllResults and getResults and sizeResults
3640 utilities.assert_equals( expect=main.TRUE,
3641 actual=addAllResults,
3642 onpass="Set addAll correct",
3643 onfail="Set addAll was incorrect" )
3644
3645 main.step( "Distributed Set contains()" )
3646 containsResponses = []
3647 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003648 for i in range( main.numCtrls ):
3649 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003650 name="setContains-" + str( i ),
3651 args=[ onosSetName ],
3652 kwargs={ "values": addValue } )
3653 threads.append( t )
3654 t.start()
3655 for t in threads:
3656 t.join()
3657 # NOTE: This is the tuple
3658 containsResponses.append( t.result )
3659
3660 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003661 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003662 if containsResponses[ i ] == main.ERROR:
3663 containsResults = main.FALSE
3664 else:
3665 containsResults = containsResults and\
3666 containsResponses[ i ][ 1 ]
3667 utilities.assert_equals( expect=main.TRUE,
3668 actual=containsResults,
3669 onpass="Set contains is functional",
3670 onfail="Set contains failed" )
3671
3672 main.step( "Distributed Set containsAll()" )
3673 containsAllResponses = []
3674 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003675 for i in range( main.numCtrls ):
3676 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003677 name="setContainsAll-" + str( i ),
3678 args=[ onosSetName ],
3679 kwargs={ "values": addAllValue } )
3680 threads.append( t )
3681 t.start()
3682 for t in threads:
3683 t.join()
3684 # NOTE: This is the tuple
3685 containsAllResponses.append( t.result )
3686
3687 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003688 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003689 if containsResponses[ i ] == main.ERROR:
3690 containsResults = main.FALSE
3691 else:
3692 containsResults = containsResults and\
3693 containsResponses[ i ][ 1 ]
3694 utilities.assert_equals( expect=main.TRUE,
3695 actual=containsAllResults,
3696 onpass="Set containsAll is functional",
3697 onfail="Set containsAll failed" )
3698
3699 main.step( "Distributed Set remove()" )
3700 onosSet.remove( addValue )
3701 removeResponses = []
3702 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003703 for i in range( main.numCtrls ):
3704 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003705 name="setTestRemove-" + str( i ),
3706 args=[ onosSetName, addValue ] )
3707 threads.append( t )
3708 t.start()
3709 for t in threads:
3710 t.join()
3711 removeResponses.append( t.result )
3712
3713 # main.TRUE = successfully changed the set
3714 # main.FALSE = action resulted in no change in set
3715 # main.ERROR - Some error in executing the function
3716 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003717 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003718 if removeResponses[ i ] == main.TRUE:
3719 # All is well
3720 pass
3721 elif removeResponses[ i ] == main.FALSE:
3722 # not in set, probably fine
3723 pass
3724 elif removeResponses[ i ] == main.ERROR:
3725 # Error in execution
3726 removeResults = main.FALSE
3727 else:
3728 # unexpected result
3729 removeResults = main.FALSE
3730 if removeResults != main.TRUE:
3731 main.log.error( "Error executing set remove" )
3732
3733 # Check if set is still correct
3734 size = len( onosSet )
3735 getResponses = []
3736 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003737 for i in range( main.numCtrls ):
3738 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003739 name="setTestGet-" + str( i ),
3740 args=[ onosSetName ] )
3741 threads.append( t )
3742 t.start()
3743 for t in threads:
3744 t.join()
3745 getResponses.append( t.result )
3746 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003747 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003748 if isinstance( getResponses[ i ], list):
3749 current = set( getResponses[ i ] )
3750 if len( current ) == len( getResponses[ i ] ):
3751 # no repeats
3752 if onosSet != current:
3753 main.log.error( "ONOS" + str( i + 1 ) +
3754 " has incorrect view" +
3755 " of set " + onosSetName + ":\n" +
3756 str( getResponses[ i ] ) )
3757 main.log.debug( "Expected: " + str( onosSet ) )
3758 main.log.debug( "Actual: " + str( current ) )
3759 getResults = main.FALSE
3760 else:
3761 # error, set is not a set
3762 main.log.error( "ONOS" + str( i + 1 ) +
3763 " has repeat elements in" +
3764 " set " + onosSetName + ":\n" +
3765 str( getResponses[ i ] ) )
3766 getResults = main.FALSE
3767 elif getResponses[ i ] == main.ERROR:
3768 getResults = main.FALSE
3769 sizeResponses = []
3770 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003771 for i in range( main.numCtrls ):
3772 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003773 name="setTestSize-" + str( i ),
3774 args=[ onosSetName ] )
3775 threads.append( t )
3776 t.start()
3777 for t in threads:
3778 t.join()
3779 sizeResponses.append( t.result )
3780 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003781 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003782 if size != sizeResponses[ i ]:
3783 sizeResults = main.FALSE
3784 main.log.error( "ONOS" + str( i + 1 ) +
3785 " expected a size of " + str( size ) +
3786 " for set " + onosSetName +
3787 " but got " + str( sizeResponses[ i ] ) )
3788 removeResults = removeResults and getResults and sizeResults
3789 utilities.assert_equals( expect=main.TRUE,
3790 actual=removeResults,
3791 onpass="Set remove correct",
3792 onfail="Set remove was incorrect" )
3793
3794 main.step( "Distributed Set removeAll()" )
3795 onosSet.difference_update( addAllValue.split() )
3796 removeAllResponses = []
3797 threads = []
3798 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003799 for i in range( main.numCtrls ):
3800 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003801 name="setTestRemoveAll-" + str( i ),
3802 args=[ onosSetName, addAllValue ] )
3803 threads.append( t )
3804 t.start()
3805 for t in threads:
3806 t.join()
3807 removeAllResponses.append( t.result )
3808 except Exception, e:
3809 main.log.exception(e)
3810
3811 # main.TRUE = successfully changed the set
3812 # main.FALSE = action resulted in no change in set
3813 # main.ERROR - Some error in executing the function
3814 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003815 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003816 if removeAllResponses[ i ] == main.TRUE:
3817 # All is well
3818 pass
3819 elif removeAllResponses[ i ] == main.FALSE:
3820 # not in set, probably fine
3821 pass
3822 elif removeAllResponses[ i ] == main.ERROR:
3823 # Error in execution
3824 removeAllResults = main.FALSE
3825 else:
3826 # unexpected result
3827 removeAllResults = main.FALSE
3828 if removeAllResults != main.TRUE:
3829 main.log.error( "Error executing set removeAll" )
3830
3831 # Check if set is still correct
3832 size = len( onosSet )
3833 getResponses = []
3834 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003835 for i in range( main.numCtrls ):
3836 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003837 name="setTestGet-" + str( i ),
3838 args=[ onosSetName ] )
3839 threads.append( t )
3840 t.start()
3841 for t in threads:
3842 t.join()
3843 getResponses.append( t.result )
3844 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003845 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003846 if isinstance( getResponses[ i ], list):
3847 current = set( getResponses[ i ] )
3848 if len( current ) == len( getResponses[ i ] ):
3849 # no repeats
3850 if onosSet != current:
3851 main.log.error( "ONOS" + str( i + 1 ) +
3852 " has incorrect view" +
3853 " of set " + onosSetName + ":\n" +
3854 str( getResponses[ i ] ) )
3855 main.log.debug( "Expected: " + str( onosSet ) )
3856 main.log.debug( "Actual: " + str( current ) )
3857 getResults = main.FALSE
3858 else:
3859 # error, set is not a set
3860 main.log.error( "ONOS" + str( i + 1 ) +
3861 " has repeat elements in" +
3862 " set " + onosSetName + ":\n" +
3863 str( getResponses[ i ] ) )
3864 getResults = main.FALSE
3865 elif getResponses[ i ] == main.ERROR:
3866 getResults = main.FALSE
3867 sizeResponses = []
3868 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003869 for i in range( main.numCtrls ):
3870 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003871 name="setTestSize-" + str( i ),
3872 args=[ onosSetName ] )
3873 threads.append( t )
3874 t.start()
3875 for t in threads:
3876 t.join()
3877 sizeResponses.append( t.result )
3878 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003879 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003880 if size != sizeResponses[ i ]:
3881 sizeResults = main.FALSE
3882 main.log.error( "ONOS" + str( i + 1 ) +
3883 " expected a size of " + str( size ) +
3884 " for set " + onosSetName +
3885 " but got " + str( sizeResponses[ i ] ) )
3886 removeAllResults = removeAllResults and getResults and sizeResults
3887 utilities.assert_equals( expect=main.TRUE,
3888 actual=removeAllResults,
3889 onpass="Set removeAll correct",
3890 onfail="Set removeAll was incorrect" )
3891
3892 main.step( "Distributed Set addAll()" )
3893 onosSet.update( addAllValue.split() )
3894 addResponses = []
3895 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003896 for i in range( main.numCtrls ):
3897 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003898 name="setTestAddAll-" + str( i ),
3899 args=[ onosSetName, addAllValue ] )
3900 threads.append( t )
3901 t.start()
3902 for t in threads:
3903 t.join()
3904 addResponses.append( t.result )
3905
3906 # main.TRUE = successfully changed the set
3907 # main.FALSE = action resulted in no change in set
3908 # main.ERROR - Some error in executing the function
3909 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003910 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003911 if addResponses[ i ] == main.TRUE:
3912 # All is well
3913 pass
3914 elif addResponses[ i ] == main.FALSE:
3915 # Already in set, probably fine
3916 pass
3917 elif addResponses[ i ] == main.ERROR:
3918 # Error in execution
3919 addAllResults = main.FALSE
3920 else:
3921 # unexpected result
3922 addAllResults = main.FALSE
3923 if addAllResults != main.TRUE:
3924 main.log.error( "Error executing set addAll" )
3925
3926 # Check if set is still correct
3927 size = len( onosSet )
3928 getResponses = []
3929 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003930 for i in range( main.numCtrls ):
3931 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003932 name="setTestGet-" + str( i ),
3933 args=[ onosSetName ] )
3934 threads.append( t )
3935 t.start()
3936 for t in threads:
3937 t.join()
3938 getResponses.append( t.result )
3939 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003940 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003941 if isinstance( getResponses[ i ], list):
3942 current = set( getResponses[ i ] )
3943 if len( current ) == len( getResponses[ i ] ):
3944 # no repeats
3945 if onosSet != current:
3946 main.log.error( "ONOS" + str( i + 1 ) +
3947 " has incorrect view" +
3948 " of set " + onosSetName + ":\n" +
3949 str( getResponses[ i ] ) )
3950 main.log.debug( "Expected: " + str( onosSet ) )
3951 main.log.debug( "Actual: " + str( current ) )
3952 getResults = main.FALSE
3953 else:
3954 # error, set is not a set
3955 main.log.error( "ONOS" + str( i + 1 ) +
3956 " has repeat elements in" +
3957 " set " + onosSetName + ":\n" +
3958 str( getResponses[ i ] ) )
3959 getResults = main.FALSE
3960 elif getResponses[ i ] == main.ERROR:
3961 getResults = main.FALSE
3962 sizeResponses = []
3963 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003964 for i in range( main.numCtrls ):
3965 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003966 name="setTestSize-" + str( i ),
3967 args=[ onosSetName ] )
3968 threads.append( t )
3969 t.start()
3970 for t in threads:
3971 t.join()
3972 sizeResponses.append( t.result )
3973 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003974 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003975 if size != sizeResponses[ i ]:
3976 sizeResults = main.FALSE
3977 main.log.error( "ONOS" + str( i + 1 ) +
3978 " expected a size of " + str( size ) +
3979 " for set " + onosSetName +
3980 " but got " + str( sizeResponses[ i ] ) )
3981 addAllResults = addAllResults and getResults and sizeResults
3982 utilities.assert_equals( expect=main.TRUE,
3983 actual=addAllResults,
3984 onpass="Set addAll correct",
3985 onfail="Set addAll was incorrect" )
3986
3987 main.step( "Distributed Set clear()" )
3988 onosSet.clear()
3989 clearResponses = []
3990 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003991 for i in range( main.numCtrls ):
3992 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003993 name="setTestClear-" + str( i ),
3994 args=[ onosSetName, " "], # Values doesn't matter
3995 kwargs={ "clear": True } )
3996 threads.append( t )
3997 t.start()
3998 for t in threads:
3999 t.join()
4000 clearResponses.append( t.result )
4001
4002 # main.TRUE = successfully changed the set
4003 # main.FALSE = action resulted in no change in set
4004 # main.ERROR - Some error in executing the function
4005 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004006 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004007 if clearResponses[ i ] == main.TRUE:
4008 # All is well
4009 pass
4010 elif clearResponses[ i ] == main.FALSE:
4011 # Nothing set, probably fine
4012 pass
4013 elif clearResponses[ i ] == main.ERROR:
4014 # Error in execution
4015 clearResults = main.FALSE
4016 else:
4017 # unexpected result
4018 clearResults = main.FALSE
4019 if clearResults != main.TRUE:
4020 main.log.error( "Error executing set clear" )
4021
4022 # Check if set is still correct
4023 size = len( onosSet )
4024 getResponses = []
4025 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004026 for i in range( main.numCtrls ):
4027 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004028 name="setTestGet-" + str( i ),
4029 args=[ onosSetName ] )
4030 threads.append( t )
4031 t.start()
4032 for t in threads:
4033 t.join()
4034 getResponses.append( t.result )
4035 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004036 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004037 if isinstance( getResponses[ i ], list):
4038 current = set( getResponses[ i ] )
4039 if len( current ) == len( getResponses[ i ] ):
4040 # no repeats
4041 if onosSet != current:
4042 main.log.error( "ONOS" + str( i + 1 ) +
4043 " has incorrect view" +
4044 " of set " + onosSetName + ":\n" +
4045 str( getResponses[ i ] ) )
4046 main.log.debug( "Expected: " + str( onosSet ) )
4047 main.log.debug( "Actual: " + str( current ) )
4048 getResults = main.FALSE
4049 else:
4050 # error, set is not a set
4051 main.log.error( "ONOS" + str( i + 1 ) +
4052 " has repeat elements in" +
4053 " set " + onosSetName + ":\n" +
4054 str( getResponses[ i ] ) )
4055 getResults = main.FALSE
4056 elif getResponses[ i ] == main.ERROR:
4057 getResults = main.FALSE
4058 sizeResponses = []
4059 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004060 for i in range( main.numCtrls ):
4061 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004062 name="setTestSize-" + str( i ),
4063 args=[ onosSetName ] )
4064 threads.append( t )
4065 t.start()
4066 for t in threads:
4067 t.join()
4068 sizeResponses.append( t.result )
4069 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004070 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004071 if size != sizeResponses[ i ]:
4072 sizeResults = main.FALSE
4073 main.log.error( "ONOS" + str( i + 1 ) +
4074 " expected a size of " + str( size ) +
4075 " for set " + onosSetName +
4076 " but got " + str( sizeResponses[ i ] ) )
4077 clearResults = clearResults and getResults and sizeResults
4078 utilities.assert_equals( expect=main.TRUE,
4079 actual=clearResults,
4080 onpass="Set clear correct",
4081 onfail="Set clear was incorrect" )
4082
4083 main.step( "Distributed Set addAll()" )
4084 onosSet.update( addAllValue.split() )
4085 addResponses = []
4086 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004087 for i in range( main.numCtrls ):
4088 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004089 name="setTestAddAll-" + str( i ),
4090 args=[ onosSetName, addAllValue ] )
4091 threads.append( t )
4092 t.start()
4093 for t in threads:
4094 t.join()
4095 addResponses.append( t.result )
4096
4097 # main.TRUE = successfully changed the set
4098 # main.FALSE = action resulted in no change in set
4099 # main.ERROR - Some error in executing the function
4100 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004101 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004102 if addResponses[ i ] == main.TRUE:
4103 # All is well
4104 pass
4105 elif addResponses[ i ] == main.FALSE:
4106 # Already in set, probably fine
4107 pass
4108 elif addResponses[ i ] == main.ERROR:
4109 # Error in execution
4110 addAllResults = main.FALSE
4111 else:
4112 # unexpected result
4113 addAllResults = main.FALSE
4114 if addAllResults != main.TRUE:
4115 main.log.error( "Error executing set addAll" )
4116
4117 # Check if set is still correct
4118 size = len( onosSet )
4119 getResponses = []
4120 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004121 for i in range( main.numCtrls ):
4122 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004123 name="setTestGet-" + str( i ),
4124 args=[ onosSetName ] )
4125 threads.append( t )
4126 t.start()
4127 for t in threads:
4128 t.join()
4129 getResponses.append( t.result )
4130 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004131 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004132 if isinstance( getResponses[ i ], list):
4133 current = set( getResponses[ i ] )
4134 if len( current ) == len( getResponses[ i ] ):
4135 # no repeats
4136 if onosSet != current:
4137 main.log.error( "ONOS" + str( i + 1 ) +
4138 " has incorrect view" +
4139 " of set " + onosSetName + ":\n" +
4140 str( getResponses[ i ] ) )
4141 main.log.debug( "Expected: " + str( onosSet ) )
4142 main.log.debug( "Actual: " + str( current ) )
4143 getResults = main.FALSE
4144 else:
4145 # error, set is not a set
4146 main.log.error( "ONOS" + str( i + 1 ) +
4147 " has repeat elements in" +
4148 " set " + onosSetName + ":\n" +
4149 str( getResponses[ i ] ) )
4150 getResults = main.FALSE
4151 elif getResponses[ i ] == main.ERROR:
4152 getResults = main.FALSE
4153 sizeResponses = []
4154 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004155 for i in range( main.numCtrls ):
4156 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004157 name="setTestSize-" + str( i ),
4158 args=[ onosSetName ] )
4159 threads.append( t )
4160 t.start()
4161 for t in threads:
4162 t.join()
4163 sizeResponses.append( t.result )
4164 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004165 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004166 if size != sizeResponses[ i ]:
4167 sizeResults = main.FALSE
4168 main.log.error( "ONOS" + str( i + 1 ) +
4169 " expected a size of " + str( size ) +
4170 " for set " + onosSetName +
4171 " but got " + str( sizeResponses[ i ] ) )
4172 addAllResults = addAllResults and getResults and sizeResults
4173 utilities.assert_equals( expect=main.TRUE,
4174 actual=addAllResults,
4175 onpass="Set addAll correct",
4176 onfail="Set addAll was incorrect" )
4177
4178 main.step( "Distributed Set retain()" )
4179 onosSet.intersection_update( retainValue.split() )
4180 retainResponses = []
4181 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004182 for i in range( main.numCtrls ):
4183 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004184 name="setTestRetain-" + str( i ),
4185 args=[ onosSetName, retainValue ],
4186 kwargs={ "retain": True } )
4187 threads.append( t )
4188 t.start()
4189 for t in threads:
4190 t.join()
4191 retainResponses.append( t.result )
4192
4193 # main.TRUE = successfully changed the set
4194 # main.FALSE = action resulted in no change in set
4195 # main.ERROR - Some error in executing the function
4196 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004197 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004198 if retainResponses[ i ] == main.TRUE:
4199 # All is well
4200 pass
4201 elif retainResponses[ i ] == main.FALSE:
4202 # Already in set, probably fine
4203 pass
4204 elif retainResponses[ i ] == main.ERROR:
4205 # Error in execution
4206 retainResults = main.FALSE
4207 else:
4208 # unexpected result
4209 retainResults = main.FALSE
4210 if retainResults != main.TRUE:
4211 main.log.error( "Error executing set retain" )
4212
4213 # Check if set is still correct
4214 size = len( onosSet )
4215 getResponses = []
4216 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004217 for i in range( main.numCtrls ):
4218 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004219 name="setTestGet-" + str( i ),
4220 args=[ onosSetName ] )
4221 threads.append( t )
4222 t.start()
4223 for t in threads:
4224 t.join()
4225 getResponses.append( t.result )
4226 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004227 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004228 if isinstance( getResponses[ i ], list):
4229 current = set( getResponses[ i ] )
4230 if len( current ) == len( getResponses[ i ] ):
4231 # no repeats
4232 if onosSet != current:
4233 main.log.error( "ONOS" + str( i + 1 ) +
4234 " has incorrect view" +
4235 " of set " + onosSetName + ":\n" +
4236 str( getResponses[ i ] ) )
4237 main.log.debug( "Expected: " + str( onosSet ) )
4238 main.log.debug( "Actual: " + str( current ) )
4239 getResults = main.FALSE
4240 else:
4241 # error, set is not a set
4242 main.log.error( "ONOS" + str( i + 1 ) +
4243 " has repeat elements in" +
4244 " set " + onosSetName + ":\n" +
4245 str( getResponses[ i ] ) )
4246 getResults = main.FALSE
4247 elif getResponses[ i ] == main.ERROR:
4248 getResults = main.FALSE
4249 sizeResponses = []
4250 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004251 for i in range( main.numCtrls ):
4252 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004253 name="setTestSize-" + str( i ),
4254 args=[ onosSetName ] )
4255 threads.append( t )
4256 t.start()
4257 for t in threads:
4258 t.join()
4259 sizeResponses.append( t.result )
4260 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004261 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004262 if size != sizeResponses[ i ]:
4263 sizeResults = main.FALSE
4264 main.log.error( "ONOS" + str( i + 1 ) +
4265 " expected a size of " +
4266 str( size ) + " for set " + onosSetName +
4267 " but got " + str( sizeResponses[ i ] ) )
4268 retainResults = retainResults and getResults and sizeResults
4269 utilities.assert_equals( expect=main.TRUE,
4270 actual=retainResults,
4271 onpass="Set retain correct",
4272 onfail="Set retain was incorrect" )
4273
Jon Hall2a5002c2015-08-21 16:49:11 -07004274 # Transactional maps
4275 main.step( "Partitioned Transactional maps put" )
4276 tMapValue = "Testing"
4277 numKeys = 100
4278 putResult = True
4279 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4280 if len( putResponses ) == 100:
4281 for i in putResponses:
4282 if putResponses[ i ][ 'value' ] != tMapValue:
4283 putResult = False
4284 else:
4285 putResult = False
4286 if not putResult:
4287 main.log.debug( "Put response values: " + str( putResponses ) )
4288 utilities.assert_equals( expect=True,
4289 actual=putResult,
4290 onpass="Partitioned Transactional Map put successful",
4291 onfail="Partitioned Transactional Map put values are incorrect" )
4292
4293 main.step( "Partitioned Transactional maps get" )
4294 getCheck = True
4295 for n in range( 1, numKeys + 1 ):
4296 getResponses = []
4297 threads = []
4298 valueCheck = True
4299 for i in range( main.numCtrls ):
4300 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4301 name="TMap-get-" + str( i ),
4302 args=[ "Key" + str ( n ) ] )
4303 threads.append( t )
4304 t.start()
4305 for t in threads:
4306 t.join()
4307 getResponses.append( t.result )
4308 for node in getResponses:
4309 if node != tMapValue:
4310 valueCheck = False
4311 if not valueCheck:
4312 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4313 main.log.warn( getResponses )
4314 getCheck = getCheck and valueCheck
4315 utilities.assert_equals( expect=True,
4316 actual=getCheck,
4317 onpass="Partitioned Transactional Map get values were correct",
4318 onfail="Partitioned Transactional Map values incorrect" )
4319
4320 main.step( "In-memory Transactional maps put" )
4321 tMapValue = "Testing"
4322 numKeys = 100
4323 putResult = True
4324 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4325 if len( putResponses ) == 100:
4326 for i in putResponses:
4327 if putResponses[ i ][ 'value' ] != tMapValue:
4328 putResult = False
4329 else:
4330 putResult = False
4331 if not putResult:
4332 main.log.debug( "Put response values: " + str( putResponses ) )
4333 utilities.assert_equals( expect=True,
4334 actual=putResult,
4335 onpass="In-Memory Transactional Map put successful",
4336 onfail="In-Memory Transactional Map put values are incorrect" )
4337
4338 main.step( "In-Memory Transactional maps get" )
4339 getCheck = True
4340 for n in range( 1, numKeys + 1 ):
4341 getResponses = []
4342 threads = []
4343 valueCheck = True
4344 for i in range( main.numCtrls ):
4345 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4346 name="TMap-get-" + str( i ),
4347 args=[ "Key" + str ( n ) ],
4348 kwargs={ "inMemory": True } )
4349 threads.append( t )
4350 t.start()
4351 for t in threads:
4352 t.join()
4353 getResponses.append( t.result )
4354 for node in getResponses:
4355 if node != tMapValue:
4356 valueCheck = False
4357 if not valueCheck:
4358 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4359 main.log.warn( getResponses )
4360 getCheck = getCheck and valueCheck
4361 utilities.assert_equals( expect=True,
4362 actual=getCheck,
4363 onpass="In-Memory Transactional Map get values were correct",
4364 onfail="In-Memory Transactional Map values incorrect" )