blob: 70609d391524d032cc6c2ae0e9558def490063b4 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
28class HAsanity:
29
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080052 import time
Jon Hall5cf14d52015-07-16 12:15:19 -070053 main.log.info( "ONOS HA Sanity test - initialization" )
54 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070055 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "installing ONOS, starting Mininet and ONOS" +\
57 "cli sessions."
58 # TODO: save all the timers and output them for plotting
59
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070071 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070072 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070073 global ONOS1Port
74 global ONOS2Port
75 global ONOS3Port
76 global ONOS4Port
77 global ONOS5Port
78 global ONOS6Port
79 global ONOS7Port
80
81 # FIXME: just get controller port from params?
82 # TODO: do we really need all these?
83 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
84 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
85 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
86 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
87 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
88 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
89 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
90
Jon Halle1a3b752015-07-22 13:02:46 -070091 try:
92 fileName = "Counters"
93 path = main.params[ 'imports' ][ 'path' ]
94 main.Counters = imp.load_source( fileName,
95 path + fileName + ".py" )
96 except Exception as e:
97 main.log.exception( e )
98 main.cleanup()
99 main.exit()
100
101 main.CLIs = []
102 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700103 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700104 for i in range( 1, main.numCtrls + 1 ):
105 try:
106 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
107 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
108 ipList.append( main.nodes[ -1 ].ip_address )
109 except AttributeError:
110 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700111
112 main.step( "Create cell file" )
113 cellAppString = main.params[ 'ENV' ][ 'appString' ]
114 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
115 main.Mininet1.ip_address,
116 cellAppString, ipList )
117 main.step( "Applying cell variable to environment" )
118 cellResult = main.ONOSbench.setCell( cellName )
119 verifyResult = main.ONOSbench.verifyCell()
120
121 # FIXME:this is short term fix
122 main.log.info( "Removing raft logs" )
123 main.ONOSbench.onosRemoveRaftLogs()
124
125 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700126 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700127 main.ONOSbench.onosUninstall( node.ip_address )
128
129 # Make sure ONOS is DEAD
130 main.log.info( "Killing any ONOS processes" )
131 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700132 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700133 killed = main.ONOSbench.onosKill( node.ip_address )
134 killResults = killResults and killed
135
136 cleanInstallResult = main.TRUE
137 gitPullResult = main.TRUE
138
139 main.step( "Starting Mininet" )
140 # scp topo file to mininet
141 # TODO: move to params?
142 topoName = "obelisk.py"
143 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700144 main.ONOSbench.scp( main.Mininet1,
145 filePath + topoName,
146 main.Mininet1.home,
147 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700148 mnResult = main.Mininet1.startNet( )
149 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
150 onpass="Mininet Started",
151 onfail="Error starting Mininet" )
152
153 main.step( "Git checkout and pull " + gitBranch )
154 if PULLCODE:
155 main.ONOSbench.gitCheckout( gitBranch )
156 gitPullResult = main.ONOSbench.gitPull()
157 # values of 1 or 3 are good
158 utilities.assert_lesser( expect=0, actual=gitPullResult,
159 onpass="Git pull successful",
160 onfail="Git pull failed" )
161 main.ONOSbench.getVersion( report=True )
162
163 main.step( "Using mvn clean install" )
164 cleanInstallResult = main.TRUE
165 if PULLCODE and gitPullResult == main.TRUE:
166 cleanInstallResult = main.ONOSbench.cleanInstall()
167 else:
168 main.log.warn( "Did not pull new code so skipping mvn " +
169 "clean install" )
170 utilities.assert_equals( expect=main.TRUE,
171 actual=cleanInstallResult,
172 onpass="MCI successful",
173 onfail="MCI failed" )
174 # GRAPHS
175 # NOTE: important params here:
176 # job = name of Jenkins job
177 # Plot Name = Plot-HA, only can be used if multiple plots
178 # index = The number of the graph under plot name
179 job = "HAsanity"
180 plotName = "Plot-HA"
181 graphs = '<ac:structured-macro ac:name="html">\n'
182 graphs += '<ac:plain-text-body><![CDATA[\n'
183 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
184 '/plot/' + plotName + '/getPlot?index=0' +\
185 '&width=500&height=300"' +\
186 'noborder="0" width="500" height="300" scrolling="yes" ' +\
187 'seamless="seamless"></iframe>\n'
188 graphs += ']]></ac:plain-text-body>\n'
189 graphs += '</ac:structured-macro>\n'
190 main.log.wiki(graphs)
191
192 main.step( "Creating ONOS package" )
193 packageResult = main.ONOSbench.onosPackage()
194 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
195 onpass="ONOS package successful",
196 onfail="ONOS package failed" )
197
198 main.step( "Installing ONOS package" )
199 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700200 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700201 tmpResult = main.ONOSbench.onosInstall( options="-f",
202 node=node.ip_address )
203 onosInstallResult = onosInstallResult and tmpResult
204 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
205 onpass="ONOS install successful",
206 onfail="ONOS install failed" )
207
208 main.step( "Checking if ONOS is up yet" )
209 for i in range( 2 ):
210 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700211 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700212 started = main.ONOSbench.isup( node.ip_address )
213 if not started:
214 main.log.error( node.name + " didn't start!" )
215 main.ONOSbench.onosStop( node.ip_address )
216 main.ONOSbench.onosStart( node.ip_address )
217 onosIsupResult = onosIsupResult and started
218 if onosIsupResult == main.TRUE:
219 break
220 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
221 onpass="ONOS startup successful",
222 onfail="ONOS startup failed" )
223
224 main.log.step( "Starting ONOS CLI sessions" )
225 cliResults = main.TRUE
226 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700227 for i in range( main.numCtrls ):
228 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700229 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700230 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700231 threads.append( t )
232 t.start()
233
234 for t in threads:
235 t.join()
236 cliResults = cliResults and t.result
237 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
238 onpass="ONOS cli startup successful",
239 onfail="ONOS cli startup failed" )
240
241 if main.params[ 'tcpdump' ].lower() == "true":
242 main.step( "Start Packet Capture MN" )
243 main.Mininet2.startTcpdump(
244 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
245 + "-MN.pcap",
246 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
247 port=main.params[ 'MNtcpdump' ][ 'port' ] )
248
249 main.step( "App Ids check" )
Jon Hallf3d16e72015-12-16 17:45:08 -0800250 time.sleep(60)
Jon Hall5cf14d52015-07-16 12:15:19 -0700251 appCheck = main.TRUE
252 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700253 for i in range( main.numCtrls ):
254 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700255 name="appToIDCheck-" + str( i ),
256 args=[] )
257 threads.append( t )
258 t.start()
259
260 for t in threads:
261 t.join()
262 appCheck = appCheck and t.result
263 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700264 main.log.warn( main.CLIs[0].apps() )
265 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700266 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
267 onpass="App Ids seem to be correct",
268 onfail="Something is wrong with app Ids" )
269
270 if cliResults == main.FALSE:
271 main.log.error( "Failed to start ONOS, stopping test" )
272 main.cleanup()
273 main.exit()
274
275 def CASE2( self, main ):
276 """
277 Assign devices to controllers
278 """
279 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700280 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700281 assert main, "main not defined"
282 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700283 assert main.CLIs, "main.CLIs not defined"
284 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700285 assert ONOS1Port, "ONOS1Port not defined"
286 assert ONOS2Port, "ONOS2Port not defined"
287 assert ONOS3Port, "ONOS3Port not defined"
288 assert ONOS4Port, "ONOS4Port not defined"
289 assert ONOS5Port, "ONOS5Port not defined"
290 assert ONOS6Port, "ONOS6Port not defined"
291 assert ONOS7Port, "ONOS7Port not defined"
292
293 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700294 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700295 "and check that an ONOS node becomes the " +\
296 "master of the device."
297 main.step( "Assign switches to controllers" )
298
299 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700300 for i in range( main.numCtrls ):
301 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700302 swList = []
303 for i in range( 1, 29 ):
304 swList.append( "s" + str( i ) )
305 main.Mininet1.assignSwController( sw=swList, ip=ipList )
306
307 mastershipCheck = main.TRUE
308 for i in range( 1, 29 ):
309 response = main.Mininet1.getSwController( "s" + str( i ) )
310 try:
311 main.log.info( str( response ) )
312 except Exception:
313 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700314 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700315 if re.search( "tcp:" + node.ip_address, response ):
316 mastershipCheck = mastershipCheck and main.TRUE
317 else:
318 main.log.error( "Error, node " + node.ip_address + " is " +
319 "not in the list of controllers s" +
320 str( i ) + " is connecting to." )
321 mastershipCheck = main.FALSE
322 utilities.assert_equals(
323 expect=main.TRUE,
324 actual=mastershipCheck,
325 onpass="Switch mastership assigned correctly",
326 onfail="Switches not assigned correctly to controllers" )
327
328 def CASE21( self, main ):
329 """
330 Assign mastership to controllers
331 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700332 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700333 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700334 assert main, "main not defined"
335 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700336 assert main.CLIs, "main.CLIs not defined"
337 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700338 assert ONOS1Port, "ONOS1Port not defined"
339 assert ONOS2Port, "ONOS2Port not defined"
340 assert ONOS3Port, "ONOS3Port not defined"
341 assert ONOS4Port, "ONOS4Port not defined"
342 assert ONOS5Port, "ONOS5Port not defined"
343 assert ONOS6Port, "ONOS6Port not defined"
344 assert ONOS7Port, "ONOS7Port not defined"
345
346 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700347 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700348 "device. Then manually assign" +\
349 " mastership to specific ONOS nodes using" +\
350 " 'device-role'"
351 main.step( "Assign mastership of switches to specific controllers" )
352 # Manually assign mastership to the controller we want
353 roleCall = main.TRUE
354
355 ipList = [ ]
356 deviceList = []
357 try:
358 # Assign mastership to specific controllers. This assignment was
359 # determined for a 7 node cluser, but will work with any sized
360 # cluster
361 for i in range( 1, 29 ): # switches 1 through 28
362 # set up correct variables:
363 if i == 1:
364 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700365 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700366 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
367 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700368 c = 1 % main.numCtrls
369 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700370 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
371 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700372 c = 1 % main.numCtrls
373 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700374 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
375 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700376 c = 3 % main.numCtrls
377 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700378 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
379 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700380 c = 2 % main.numCtrls
381 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700382 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
383 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700384 c = 2 % main.numCtrls
385 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700386 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
387 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700388 c = 5 % main.numCtrls
389 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700390 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
391 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700392 c = 4 % main.numCtrls
393 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700394 dpid = '3' + str( i ).zfill( 3 )
395 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
396 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700397 c = 6 % main.numCtrls
398 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700399 dpid = '6' + str( i ).zfill( 3 )
400 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
401 elif i == 28:
402 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700403 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700404 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
405 else:
406 main.log.error( "You didn't write an else statement for " +
407 "switch s" + str( i ) )
408 roleCall = main.FALSE
409 # Assign switch
410 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
411 # TODO: make this controller dynamic
412 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
413 ip )
414 ipList.append( ip )
415 deviceList.append( deviceId )
416 except ( AttributeError, AssertionError ):
417 main.log.exception( "Something is wrong with ONOS device view" )
418 main.log.info( main.ONOScli1.devices() )
419 utilities.assert_equals(
420 expect=main.TRUE,
421 actual=roleCall,
422 onpass="Re-assigned switch mastership to designated controller",
423 onfail="Something wrong with deviceRole calls" )
424
425 main.step( "Check mastership was correctly assigned" )
426 roleCheck = main.TRUE
427 # NOTE: This is due to the fact that device mastership change is not
428 # atomic and is actually a multi step process
429 time.sleep( 5 )
430 for i in range( len( ipList ) ):
431 ip = ipList[i]
432 deviceId = deviceList[i]
433 # Check assignment
434 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
435 if ip in master:
436 roleCheck = roleCheck and main.TRUE
437 else:
438 roleCheck = roleCheck and main.FALSE
439 main.log.error( "Error, controller " + ip + " is not" +
440 " master " + "of device " +
441 str( deviceId ) + ". Master is " +
442 repr( master ) + "." )
443 utilities.assert_equals(
444 expect=main.TRUE,
445 actual=roleCheck,
446 onpass="Switches were successfully reassigned to designated " +
447 "controller",
448 onfail="Switches were not successfully reassigned" )
449
450 def CASE3( self, main ):
451 """
452 Assign intents
453 """
454 import time
455 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700456 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700457 assert main, "main not defined"
458 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700459 assert main.CLIs, "main.CLIs not defined"
460 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700461 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700462 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 "assign predetermined host-to-host intents." +\
464 " After installation, check that the intent" +\
465 " is distributed to all nodes and the state" +\
466 " is INSTALLED"
467
468 # install onos-app-fwd
469 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700470 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700471 utilities.assert_equals( expect=main.TRUE, actual=installResults,
472 onpass="Install fwd successful",
473 onfail="Install fwd failed" )
474
475 main.step( "Check app ids" )
476 appCheck = main.TRUE
477 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700478 for i in range( main.numCtrls ):
479 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700480 name="appToIDCheck-" + str( i ),
481 args=[] )
482 threads.append( t )
483 t.start()
484
485 for t in threads:
486 t.join()
487 appCheck = appCheck and t.result
488 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700489 main.log.warn( main.CLIs[0].apps() )
490 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700491 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
492 onpass="App Ids seem to be correct",
493 onfail="Something is wrong with app Ids" )
494
495 main.step( "Discovering Hosts( Via pingall for now )" )
496 # FIXME: Once we have a host discovery mechanism, use that instead
497 # REACTIVE FWD test
498 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700499 passMsg = "Reactive Pingall test passed"
500 time1 = time.time()
501 pingResult = main.Mininet1.pingall()
502 time2 = time.time()
503 if not pingResult:
504 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700505 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700506 passMsg += " on the second try"
507 utilities.assert_equals(
508 expect=main.TRUE,
509 actual=pingResult,
510 onpass= passMsg,
511 onfail="Reactive Pingall failed, " +
512 "one or more ping pairs failed" )
513 main.log.info( "Time for pingall: %2f seconds" %
514 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700515 # timeout for fwd flows
516 time.sleep( 11 )
517 # uninstall onos-app-fwd
518 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700519 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700520 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
521 onpass="Uninstall fwd successful",
522 onfail="Uninstall fwd failed" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700523
524 main.step( "Check app ids" )
525 threads = []
526 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700527 for i in range( main.numCtrls ):
528 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700529 name="appToIDCheck-" + str( i ),
530 args=[] )
531 threads.append( t )
532 t.start()
533
534 for t in threads:
535 t.join()
536 appCheck2 = appCheck2 and t.result
537 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700538 main.log.warn( main.CLIs[0].apps() )
539 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700540 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
541 onpass="App Ids seem to be correct",
542 onfail="Something is wrong with app Ids" )
543
544 main.step( "Add host intents via cli" )
545 intentIds = []
546 # TODO: move the host numbers to params
547 # Maybe look at all the paths we ping?
548 intentAddResult = True
549 hostResult = main.TRUE
550 for i in range( 8, 18 ):
551 main.log.info( "Adding host intent between h" + str( i ) +
552 " and h" + str( i + 10 ) )
553 host1 = "00:00:00:00:00:" + \
554 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
555 host2 = "00:00:00:00:00:" + \
556 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
557 # NOTE: getHost can return None
558 host1Dict = main.ONOScli1.getHost( host1 )
559 host2Dict = main.ONOScli1.getHost( host2 )
560 host1Id = None
561 host2Id = None
562 if host1Dict and host2Dict:
563 host1Id = host1Dict.get( 'id', None )
564 host2Id = host2Dict.get( 'id', None )
565 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700566 nodeNum = ( i % main.numCtrls )
567 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700568 if tmpId:
569 main.log.info( "Added intent with id: " + tmpId )
570 intentIds.append( tmpId )
571 else:
572 main.log.error( "addHostIntent returned: " +
573 repr( tmpId ) )
574 else:
575 main.log.error( "Error, getHost() failed for h" + str( i ) +
576 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700577 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700578 main.log.warn( "Hosts output: " )
579 try:
580 main.log.warn( json.dumps( json.loads( hosts ),
581 sort_keys=True,
582 indent=4,
583 separators=( ',', ': ' ) ) )
584 except ( ValueError, TypeError ):
585 main.log.warn( repr( hosts ) )
586 hostResult = main.FALSE
587 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
588 onpass="Found a host id for each host",
589 onfail="Error looking up host ids" )
590
591 intentStart = time.time()
592 onosIds = main.ONOScli1.getAllIntentsId()
593 main.log.info( "Submitted intents: " + str( intentIds ) )
594 main.log.info( "Intents in ONOS: " + str( onosIds ) )
595 for intent in intentIds:
596 if intent in onosIds:
597 pass # intent submitted is in onos
598 else:
599 intentAddResult = False
600 if intentAddResult:
601 intentStop = time.time()
602 else:
603 intentStop = None
604 # Print the intent states
605 intents = main.ONOScli1.intents()
606 intentStates = []
607 installedCheck = True
608 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
609 count = 0
610 try:
611 for intent in json.loads( intents ):
612 state = intent.get( 'state', None )
613 if "INSTALLED" not in state:
614 installedCheck = False
615 intentId = intent.get( 'id', None )
616 intentStates.append( ( intentId, state ) )
617 except ( ValueError, TypeError ):
618 main.log.exception( "Error parsing intents" )
619 # add submitted intents not in the store
620 tmplist = [ i for i, s in intentStates ]
621 missingIntents = False
622 for i in intentIds:
623 if i not in tmplist:
624 intentStates.append( ( i, " - " ) )
625 missingIntents = True
626 intentStates.sort()
627 for i, s in intentStates:
628 count += 1
629 main.log.info( "%-6s%-15s%-15s" %
630 ( str( count ), str( i ), str( s ) ) )
631 leaders = main.ONOScli1.leaders()
632 try:
633 missing = False
634 if leaders:
635 parsedLeaders = json.loads( leaders )
636 main.log.warn( json.dumps( parsedLeaders,
637 sort_keys=True,
638 indent=4,
639 separators=( ',', ': ' ) ) )
640 # check for all intent partitions
641 topics = []
642 for i in range( 14 ):
643 topics.append( "intent-partition-" + str( i ) )
644 main.log.debug( topics )
645 ONOStopics = [ j['topic'] for j in parsedLeaders ]
646 for topic in topics:
647 if topic not in ONOStopics:
648 main.log.error( "Error: " + topic +
649 " not in leaders" )
650 missing = True
651 else:
652 main.log.error( "leaders() returned None" )
653 except ( ValueError, TypeError ):
654 main.log.exception( "Error parsing leaders" )
655 main.log.error( repr( leaders ) )
656 # Check all nodes
657 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700658 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700659 response = node.leaders( jsonFormat=False)
660 main.log.warn( str( node.name ) + " leaders output: \n" +
661 str( response ) )
662
663 partitions = main.ONOScli1.partitions()
664 try:
665 if partitions :
666 parsedPartitions = json.loads( partitions )
667 main.log.warn( json.dumps( parsedPartitions,
668 sort_keys=True,
669 indent=4,
670 separators=( ',', ': ' ) ) )
671 # TODO check for a leader in all paritions
672 # TODO check for consistency among nodes
673 else:
674 main.log.error( "partitions() returned None" )
675 except ( ValueError, TypeError ):
676 main.log.exception( "Error parsing partitions" )
677 main.log.error( repr( partitions ) )
678 pendingMap = main.ONOScli1.pendingMap()
679 try:
680 if pendingMap :
681 parsedPending = json.loads( pendingMap )
682 main.log.warn( json.dumps( parsedPending,
683 sort_keys=True,
684 indent=4,
685 separators=( ',', ': ' ) ) )
686 # TODO check something here?
687 else:
688 main.log.error( "pendingMap() returned None" )
689 except ( ValueError, TypeError ):
690 main.log.exception( "Error parsing pending map" )
691 main.log.error( repr( pendingMap ) )
692
693 intentAddResult = bool( intentAddResult and not missingIntents and
694 installedCheck )
695 if not intentAddResult:
696 main.log.error( "Error in pushing host intents to ONOS" )
697
698 main.step( "Intent Anti-Entropy dispersion" )
699 for i in range(100):
700 correct = True
701 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700702 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700703 onosIds = []
704 ids = cli.getAllIntentsId()
705 onosIds.append( ids )
706 main.log.debug( "Intents in " + cli.name + ": " +
707 str( sorted( onosIds ) ) )
708 if sorted( ids ) != sorted( intentIds ):
709 main.log.warn( "Set of intent IDs doesn't match" )
710 correct = False
711 break
712 else:
713 intents = json.loads( cli.intents() )
714 for intent in intents:
715 if intent[ 'state' ] != "INSTALLED":
716 main.log.warn( "Intent " + intent[ 'id' ] +
717 " is " + intent[ 'state' ] )
718 correct = False
719 break
720 if correct:
721 break
722 else:
723 time.sleep(1)
724 if not intentStop:
725 intentStop = time.time()
726 global gossipTime
727 gossipTime = intentStop - intentStart
728 main.log.info( "It took about " + str( gossipTime ) +
729 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700730 gossipPeriod = int( main.params['timers']['gossip'] )
731 maxGossipTime = gossipPeriod * len( main.nodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700732 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700733 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700734 onpass="ECM anti-entropy for intents worked within " +
735 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700736 onfail="Intent ECM anti-entropy took too long. " +
737 "Expected time:{}, Actual time:{}".format( maxGossipTime,
738 gossipTime ) )
739 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700740 intentAddResult = True
741
742 if not intentAddResult or "key" in pendingMap:
743 import time
744 installedCheck = True
745 main.log.info( "Sleeping 60 seconds to see if intents are found" )
746 time.sleep( 60 )
747 onosIds = main.ONOScli1.getAllIntentsId()
748 main.log.info( "Submitted intents: " + str( intentIds ) )
749 main.log.info( "Intents in ONOS: " + str( onosIds ) )
750 # Print the intent states
751 intents = main.ONOScli1.intents()
752 intentStates = []
753 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
754 count = 0
755 try:
756 for intent in json.loads( intents ):
757 # Iter through intents of a node
758 state = intent.get( 'state', None )
759 if "INSTALLED" not in state:
760 installedCheck = False
761 intentId = intent.get( 'id', None )
762 intentStates.append( ( intentId, state ) )
763 except ( ValueError, TypeError ):
764 main.log.exception( "Error parsing intents" )
765 # add submitted intents not in the store
766 tmplist = [ i for i, s in intentStates ]
767 for i in intentIds:
768 if i not in tmplist:
769 intentStates.append( ( i, " - " ) )
770 intentStates.sort()
771 for i, s in intentStates:
772 count += 1
773 main.log.info( "%-6s%-15s%-15s" %
774 ( str( count ), str( i ), str( s ) ) )
775 leaders = main.ONOScli1.leaders()
776 try:
777 missing = False
778 if leaders:
779 parsedLeaders = json.loads( leaders )
780 main.log.warn( json.dumps( parsedLeaders,
781 sort_keys=True,
782 indent=4,
783 separators=( ',', ': ' ) ) )
784 # check for all intent partitions
785 # check for election
786 topics = []
787 for i in range( 14 ):
788 topics.append( "intent-partition-" + str( i ) )
789 # FIXME: this should only be after we start the app
790 topics.append( "org.onosproject.election" )
791 main.log.debug( topics )
792 ONOStopics = [ j['topic'] for j in parsedLeaders ]
793 for topic in topics:
794 if topic not in ONOStopics:
795 main.log.error( "Error: " + topic +
796 " not in leaders" )
797 missing = True
798 else:
799 main.log.error( "leaders() returned None" )
800 except ( ValueError, TypeError ):
801 main.log.exception( "Error parsing leaders" )
802 main.log.error( repr( leaders ) )
803 # Check all nodes
804 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700805 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700806 response = node.leaders( jsonFormat=False)
807 main.log.warn( str( node.name ) + " leaders output: \n" +
808 str( response ) )
809
810 partitions = main.ONOScli1.partitions()
811 try:
812 if partitions :
813 parsedPartitions = json.loads( partitions )
814 main.log.warn( json.dumps( parsedPartitions,
815 sort_keys=True,
816 indent=4,
817 separators=( ',', ': ' ) ) )
818 # TODO check for a leader in all paritions
819 # TODO check for consistency among nodes
820 else:
821 main.log.error( "partitions() returned None" )
822 except ( ValueError, TypeError ):
823 main.log.exception( "Error parsing partitions" )
824 main.log.error( repr( partitions ) )
825 pendingMap = main.ONOScli1.pendingMap()
826 try:
827 if pendingMap :
828 parsedPending = json.loads( pendingMap )
829 main.log.warn( json.dumps( parsedPending,
830 sort_keys=True,
831 indent=4,
832 separators=( ',', ': ' ) ) )
833 # TODO check something here?
834 else:
835 main.log.error( "pendingMap() returned None" )
836 except ( ValueError, TypeError ):
837 main.log.exception( "Error parsing pending map" )
838 main.log.error( repr( pendingMap ) )
839
840 def CASE4( self, main ):
841 """
842 Ping across added host intents
843 """
844 import json
845 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700846 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700847 assert main, "main not defined"
848 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700849 assert main.CLIs, "main.CLIs not defined"
850 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700851 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700852 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700853 "functionality and check the state of " +\
854 "the intent"
855 main.step( "Ping across added host intents" )
856 PingResult = main.TRUE
857 for i in range( 8, 18 ):
858 ping = main.Mininet1.pingHost( src="h" + str( i ),
859 target="h" + str( i + 10 ) )
860 PingResult = PingResult and ping
861 if ping == main.FALSE:
862 main.log.warn( "Ping failed between h" + str( i ) +
863 " and h" + str( i + 10 ) )
864 elif ping == main.TRUE:
865 main.log.info( "Ping test passed!" )
866 # Don't set PingResult or you'd override failures
867 if PingResult == main.FALSE:
868 main.log.error(
869 "Intents have not been installed correctly, pings failed." )
870 # TODO: pretty print
871 main.log.warn( "ONOS1 intents: " )
872 try:
873 tmpIntents = main.ONOScli1.intents()
874 main.log.warn( json.dumps( json.loads( tmpIntents ),
875 sort_keys=True,
876 indent=4,
877 separators=( ',', ': ' ) ) )
878 except ( ValueError, TypeError ):
879 main.log.warn( repr( tmpIntents ) )
880 utilities.assert_equals(
881 expect=main.TRUE,
882 actual=PingResult,
883 onpass="Intents have been installed correctly and pings work",
884 onfail="Intents have not been installed correctly, pings failed." )
885
886 main.step( "Check Intent state" )
887 installedCheck = False
888 loopCount = 0
889 while not installedCheck and loopCount < 40:
890 installedCheck = True
891 # Print the intent states
892 intents = main.ONOScli1.intents()
893 intentStates = []
894 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
895 count = 0
896 # Iter through intents of a node
897 try:
898 for intent in json.loads( intents ):
899 state = intent.get( 'state', None )
900 if "INSTALLED" not in state:
901 installedCheck = False
902 intentId = intent.get( 'id', None )
903 intentStates.append( ( intentId, state ) )
904 except ( ValueError, TypeError ):
905 main.log.exception( "Error parsing intents." )
906 # Print states
907 intentStates.sort()
908 for i, s in intentStates:
909 count += 1
910 main.log.info( "%-6s%-15s%-15s" %
911 ( str( count ), str( i ), str( s ) ) )
912 if not installedCheck:
913 time.sleep( 1 )
914 loopCount += 1
915 utilities.assert_equals( expect=True, actual=installedCheck,
916 onpass="Intents are all INSTALLED",
917 onfail="Intents are not all in " +
918 "INSTALLED state" )
919
920 main.step( "Check leadership of topics" )
921 leaders = main.ONOScli1.leaders()
922 topicCheck = main.TRUE
923 try:
924 if leaders:
925 parsedLeaders = json.loads( leaders )
926 main.log.warn( json.dumps( parsedLeaders,
927 sort_keys=True,
928 indent=4,
929 separators=( ',', ': ' ) ) )
930 # check for all intent partitions
931 # check for election
932 # TODO: Look at Devices as topics now that it uses this system
933 topics = []
934 for i in range( 14 ):
935 topics.append( "intent-partition-" + str( i ) )
936 # FIXME: this should only be after we start the app
937 # FIXME: topics.append( "org.onosproject.election" )
938 # Print leaders output
939 main.log.debug( topics )
940 ONOStopics = [ j['topic'] for j in parsedLeaders ]
941 for topic in topics:
942 if topic not in ONOStopics:
943 main.log.error( "Error: " + topic +
944 " not in leaders" )
945 topicCheck = main.FALSE
946 else:
947 main.log.error( "leaders() returned None" )
948 topicCheck = main.FALSE
949 except ( ValueError, TypeError ):
950 topicCheck = main.FALSE
951 main.log.exception( "Error parsing leaders" )
952 main.log.error( repr( leaders ) )
953 # TODO: Check for a leader of these topics
954 # Check all nodes
955 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700956 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700957 response = node.leaders( jsonFormat=False)
958 main.log.warn( str( node.name ) + " leaders output: \n" +
959 str( response ) )
960
961 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
962 onpass="intent Partitions is in leaders",
963 onfail="Some topics were lost " )
964 # Print partitions
965 partitions = main.ONOScli1.partitions()
966 try:
967 if partitions :
968 parsedPartitions = json.loads( partitions )
969 main.log.warn( json.dumps( parsedPartitions,
970 sort_keys=True,
971 indent=4,
972 separators=( ',', ': ' ) ) )
973 # TODO check for a leader in all paritions
974 # TODO check for consistency among nodes
975 else:
976 main.log.error( "partitions() returned None" )
977 except ( ValueError, TypeError ):
978 main.log.exception( "Error parsing partitions" )
979 main.log.error( repr( partitions ) )
980 # Print Pending Map
981 pendingMap = main.ONOScli1.pendingMap()
982 try:
983 if pendingMap :
984 parsedPending = json.loads( pendingMap )
985 main.log.warn( json.dumps( parsedPending,
986 sort_keys=True,
987 indent=4,
988 separators=( ',', ': ' ) ) )
989 # TODO check something here?
990 else:
991 main.log.error( "pendingMap() returned None" )
992 except ( ValueError, TypeError ):
993 main.log.exception( "Error parsing pending map" )
994 main.log.error( repr( pendingMap ) )
995
996 if not installedCheck:
997 main.log.info( "Waiting 60 seconds to see if the state of " +
998 "intents change" )
999 time.sleep( 60 )
1000 # Print the intent states
1001 intents = main.ONOScli1.intents()
1002 intentStates = []
1003 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1004 count = 0
1005 # Iter through intents of a node
1006 try:
1007 for intent in json.loads( intents ):
1008 state = intent.get( 'state', None )
1009 if "INSTALLED" not in state:
1010 installedCheck = False
1011 intentId = intent.get( 'id', None )
1012 intentStates.append( ( intentId, state ) )
1013 except ( ValueError, TypeError ):
1014 main.log.exception( "Error parsing intents." )
1015 intentStates.sort()
1016 for i, s in intentStates:
1017 count += 1
1018 main.log.info( "%-6s%-15s%-15s" %
1019 ( str( count ), str( i ), str( s ) ) )
1020 leaders = main.ONOScli1.leaders()
1021 try:
1022 missing = False
1023 if leaders:
1024 parsedLeaders = json.loads( leaders )
1025 main.log.warn( json.dumps( parsedLeaders,
1026 sort_keys=True,
1027 indent=4,
1028 separators=( ',', ': ' ) ) )
1029 # check for all intent partitions
1030 # check for election
1031 topics = []
1032 for i in range( 14 ):
1033 topics.append( "intent-partition-" + str( i ) )
1034 # FIXME: this should only be after we start the app
1035 topics.append( "org.onosproject.election" )
1036 main.log.debug( topics )
1037 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1038 for topic in topics:
1039 if topic not in ONOStopics:
1040 main.log.error( "Error: " + topic +
1041 " not in leaders" )
1042 missing = True
1043 else:
1044 main.log.error( "leaders() returned None" )
1045 except ( ValueError, TypeError ):
1046 main.log.exception( "Error parsing leaders" )
1047 main.log.error( repr( leaders ) )
1048 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001049 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001050 response = node.leaders( jsonFormat=False)
1051 main.log.warn( str( node.name ) + " leaders output: \n" +
1052 str( response ) )
1053
1054 partitions = main.ONOScli1.partitions()
1055 try:
1056 if partitions :
1057 parsedPartitions = json.loads( partitions )
1058 main.log.warn( json.dumps( parsedPartitions,
1059 sort_keys=True,
1060 indent=4,
1061 separators=( ',', ': ' ) ) )
1062 # TODO check for a leader in all paritions
1063 # TODO check for consistency among nodes
1064 else:
1065 main.log.error( "partitions() returned None" )
1066 except ( ValueError, TypeError ):
1067 main.log.exception( "Error parsing partitions" )
1068 main.log.error( repr( partitions ) )
1069 pendingMap = main.ONOScli1.pendingMap()
1070 try:
1071 if pendingMap :
1072 parsedPending = json.loads( pendingMap )
1073 main.log.warn( json.dumps( parsedPending,
1074 sort_keys=True,
1075 indent=4,
1076 separators=( ',', ': ' ) ) )
1077 # TODO check something here?
1078 else:
1079 main.log.error( "pendingMap() returned None" )
1080 except ( ValueError, TypeError ):
1081 main.log.exception( "Error parsing pending map" )
1082 main.log.error( repr( pendingMap ) )
1083 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001084 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001085 main.step( "Wait a minute then ping again" )
1086 # the wait is above
1087 PingResult = main.TRUE
1088 for i in range( 8, 18 ):
1089 ping = main.Mininet1.pingHost( src="h" + str( i ),
1090 target="h" + str( i + 10 ) )
1091 PingResult = PingResult and ping
1092 if ping == main.FALSE:
1093 main.log.warn( "Ping failed between h" + str( i ) +
1094 " and h" + str( i + 10 ) )
1095 elif ping == main.TRUE:
1096 main.log.info( "Ping test passed!" )
1097 # Don't set PingResult or you'd override failures
1098 if PingResult == main.FALSE:
1099 main.log.error(
1100 "Intents have not been installed correctly, pings failed." )
1101 # TODO: pretty print
1102 main.log.warn( "ONOS1 intents: " )
1103 try:
1104 tmpIntents = main.ONOScli1.intents()
1105 main.log.warn( json.dumps( json.loads( tmpIntents ),
1106 sort_keys=True,
1107 indent=4,
1108 separators=( ',', ': ' ) ) )
1109 except ( ValueError, TypeError ):
1110 main.log.warn( repr( tmpIntents ) )
1111 utilities.assert_equals(
1112 expect=main.TRUE,
1113 actual=PingResult,
1114 onpass="Intents have been installed correctly and pings work",
1115 onfail="Intents have not been installed correctly, pings failed." )
1116
1117 def CASE5( self, main ):
1118 """
1119 Reading state of ONOS
1120 """
1121 import json
1122 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001123 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001124 assert main, "main not defined"
1125 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001126 assert main.CLIs, "main.CLIs not defined"
1127 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001128
1129 main.case( "Setting up and gathering data for current state" )
1130 # The general idea for this test case is to pull the state of
1131 # ( intents,flows, topology,... ) from each ONOS node
1132 # We can then compare them with each other and also with past states
1133
1134 main.step( "Check that each switch has a master" )
1135 global mastershipState
1136 mastershipState = '[]'
1137
1138 # Assert that each device has a master
1139 rolesNotNull = main.TRUE
1140 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001141 for i in range( main.numCtrls ):
1142 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001143 name="rolesNotNull-" + str( i ),
1144 args=[] )
1145 threads.append( t )
1146 t.start()
1147
1148 for t in threads:
1149 t.join()
1150 rolesNotNull = rolesNotNull and t.result
1151 utilities.assert_equals(
1152 expect=main.TRUE,
1153 actual=rolesNotNull,
1154 onpass="Each device has a master",
1155 onfail="Some devices don't have a master assigned" )
1156
1157 main.step( "Get the Mastership of each switch from each controller" )
1158 ONOSMastership = []
1159 mastershipCheck = main.FALSE
1160 consistentMastership = True
1161 rolesResults = True
1162 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001163 for i in range( main.numCtrls ):
1164 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001165 name="roles-" + str( i ),
1166 args=[] )
1167 threads.append( t )
1168 t.start()
1169
1170 for t in threads:
1171 t.join()
1172 ONOSMastership.append( t.result )
1173
Jon Halle1a3b752015-07-22 13:02:46 -07001174 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001175 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1176 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1177 " roles" )
1178 main.log.warn(
1179 "ONOS" + str( i + 1 ) + " mastership response: " +
1180 repr( ONOSMastership[i] ) )
1181 rolesResults = False
1182 utilities.assert_equals(
1183 expect=True,
1184 actual=rolesResults,
1185 onpass="No error in reading roles output",
1186 onfail="Error in reading roles from ONOS" )
1187
1188 main.step( "Check for consistency in roles from each controller" )
1189 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1190 main.log.info(
1191 "Switch roles are consistent across all ONOS nodes" )
1192 else:
1193 consistentMastership = False
1194 utilities.assert_equals(
1195 expect=True,
1196 actual=consistentMastership,
1197 onpass="Switch roles are consistent across all ONOS nodes",
1198 onfail="ONOS nodes have different views of switch roles" )
1199
1200 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001201 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001202 try:
1203 main.log.warn(
1204 "ONOS" + str( i + 1 ) + " roles: ",
1205 json.dumps(
1206 json.loads( ONOSMastership[ i ] ),
1207 sort_keys=True,
1208 indent=4,
1209 separators=( ',', ': ' ) ) )
1210 except ( ValueError, TypeError ):
1211 main.log.warn( repr( ONOSMastership[ i ] ) )
1212 elif rolesResults and consistentMastership:
1213 mastershipCheck = main.TRUE
1214 mastershipState = ONOSMastership[ 0 ]
1215
1216 main.step( "Get the intents from each controller" )
1217 global intentState
1218 intentState = []
1219 ONOSIntents = []
1220 intentCheck = main.FALSE
1221 consistentIntents = True
1222 intentsResults = True
1223 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001224 for i in range( main.numCtrls ):
1225 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001226 name="intents-" + str( i ),
1227 args=[],
1228 kwargs={ 'jsonFormat': True } )
1229 threads.append( t )
1230 t.start()
1231
1232 for t in threads:
1233 t.join()
1234 ONOSIntents.append( t.result )
1235
Jon Halle1a3b752015-07-22 13:02:46 -07001236 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001237 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1238 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1239 " intents" )
1240 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1241 repr( ONOSIntents[ i ] ) )
1242 intentsResults = False
1243 utilities.assert_equals(
1244 expect=True,
1245 actual=intentsResults,
1246 onpass="No error in reading intents output",
1247 onfail="Error in reading intents from ONOS" )
1248
1249 main.step( "Check for consistency in Intents from each controller" )
1250 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1251 main.log.info( "Intents are consistent across all ONOS " +
1252 "nodes" )
1253 else:
1254 consistentIntents = False
1255 main.log.error( "Intents not consistent" )
1256 utilities.assert_equals(
1257 expect=True,
1258 actual=consistentIntents,
1259 onpass="Intents are consistent across all ONOS nodes",
1260 onfail="ONOS nodes have different views of intents" )
1261
1262 if intentsResults:
1263 # Try to make it easy to figure out what is happening
1264 #
1265 # Intent ONOS1 ONOS2 ...
1266 # 0x01 INSTALLED INSTALLING
1267 # ... ... ...
1268 # ... ... ...
1269 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001270 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001271 title += " " * 10 + "ONOS" + str( n + 1 )
1272 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001273 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001274 keys = []
1275 try:
1276 # Get the set of all intent keys
1277 for nodeStr in ONOSIntents:
1278 node = json.loads( nodeStr )
1279 for intent in node:
1280 keys.append( intent.get( 'id' ) )
1281 keys = set( keys )
1282 # For each intent key, print the state on each node
1283 for key in keys:
1284 row = "%-13s" % key
1285 for nodeStr in ONOSIntents:
1286 node = json.loads( nodeStr )
1287 for intent in node:
1288 if intent.get( 'id', "Error" ) == key:
1289 row += "%-15s" % intent.get( 'state' )
1290 main.log.warn( row )
1291 # End of intent state table
1292 except ValueError as e:
1293 main.log.exception( e )
1294 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1295
1296 if intentsResults and not consistentIntents:
1297 # print the json objects
1298 n = len(ONOSIntents)
1299 main.log.debug( "ONOS" + str( n ) + " intents: " )
1300 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1301 sort_keys=True,
1302 indent=4,
1303 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001304 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001305 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1306 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1307 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1308 sort_keys=True,
1309 indent=4,
1310 separators=( ',', ': ' ) ) )
1311 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001312 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001313 str( n ) + " intents" )
1314 elif intentsResults and consistentIntents:
1315 intentCheck = main.TRUE
1316 intentState = ONOSIntents[ 0 ]
1317
1318 main.step( "Get the flows from each controller" )
1319 global flowState
1320 flowState = []
1321 ONOSFlows = []
1322 ONOSFlowsJson = []
1323 flowCheck = main.FALSE
1324 consistentFlows = True
1325 flowsResults = True
1326 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001327 for i in range( main.numCtrls ):
1328 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001329 name="flows-" + str( i ),
1330 args=[],
1331 kwargs={ 'jsonFormat': True } )
1332 threads.append( t )
1333 t.start()
1334
1335 # NOTE: Flows command can take some time to run
1336 time.sleep(30)
1337 for t in threads:
1338 t.join()
1339 result = t.result
1340 ONOSFlows.append( result )
1341
Jon Halle1a3b752015-07-22 13:02:46 -07001342 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001343 num = str( i + 1 )
1344 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1345 main.log.error( "Error in getting ONOS" + num + " flows" )
1346 main.log.warn( "ONOS" + num + " flows response: " +
1347 repr( ONOSFlows[ i ] ) )
1348 flowsResults = False
1349 ONOSFlowsJson.append( None )
1350 else:
1351 try:
1352 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1353 except ( ValueError, TypeError ):
1354 # FIXME: change this to log.error?
1355 main.log.exception( "Error in parsing ONOS" + num +
1356 " response as json." )
1357 main.log.error( repr( ONOSFlows[ i ] ) )
1358 ONOSFlowsJson.append( None )
1359 flowsResults = False
1360 utilities.assert_equals(
1361 expect=True,
1362 actual=flowsResults,
1363 onpass="No error in reading flows output",
1364 onfail="Error in reading flows from ONOS" )
1365
1366 main.step( "Check for consistency in Flows from each controller" )
1367 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1368 if all( tmp ):
1369 main.log.info( "Flow count is consistent across all ONOS nodes" )
1370 else:
1371 consistentFlows = False
1372 utilities.assert_equals(
1373 expect=True,
1374 actual=consistentFlows,
1375 onpass="The flow count is consistent across all ONOS nodes",
1376 onfail="ONOS nodes have different flow counts" )
1377
1378 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001379 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001380 try:
1381 main.log.warn(
1382 "ONOS" + str( i + 1 ) + " flows: " +
1383 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1384 indent=4, separators=( ',', ': ' ) ) )
1385 except ( ValueError, TypeError ):
1386 main.log.warn(
1387 "ONOS" + str( i + 1 ) + " flows: " +
1388 repr( ONOSFlows[ i ] ) )
1389 elif flowsResults and consistentFlows:
1390 flowCheck = main.TRUE
1391 flowState = ONOSFlows[ 0 ]
1392
1393 main.step( "Get the OF Table entries" )
1394 global flows
1395 flows = []
1396 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001397 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001398 if flowCheck == main.FALSE:
1399 for table in flows:
1400 main.log.warn( table )
GlennRC68467eb2015-11-16 18:01:01 -08001401
Jon Hall5cf14d52015-07-16 12:15:19 -07001402 # TODO: Compare switch flow tables with ONOS flow tables
1403
1404 main.step( "Start continuous pings" )
1405 main.Mininet2.pingLong(
1406 src=main.params[ 'PING' ][ 'source1' ],
1407 target=main.params[ 'PING' ][ 'target1' ],
1408 pingTime=500 )
1409 main.Mininet2.pingLong(
1410 src=main.params[ 'PING' ][ 'source2' ],
1411 target=main.params[ 'PING' ][ 'target2' ],
1412 pingTime=500 )
1413 main.Mininet2.pingLong(
1414 src=main.params[ 'PING' ][ 'source3' ],
1415 target=main.params[ 'PING' ][ 'target3' ],
1416 pingTime=500 )
1417 main.Mininet2.pingLong(
1418 src=main.params[ 'PING' ][ 'source4' ],
1419 target=main.params[ 'PING' ][ 'target4' ],
1420 pingTime=500 )
1421 main.Mininet2.pingLong(
1422 src=main.params[ 'PING' ][ 'source5' ],
1423 target=main.params[ 'PING' ][ 'target5' ],
1424 pingTime=500 )
1425 main.Mininet2.pingLong(
1426 src=main.params[ 'PING' ][ 'source6' ],
1427 target=main.params[ 'PING' ][ 'target6' ],
1428 pingTime=500 )
1429 main.Mininet2.pingLong(
1430 src=main.params[ 'PING' ][ 'source7' ],
1431 target=main.params[ 'PING' ][ 'target7' ],
1432 pingTime=500 )
1433 main.Mininet2.pingLong(
1434 src=main.params[ 'PING' ][ 'source8' ],
1435 target=main.params[ 'PING' ][ 'target8' ],
1436 pingTime=500 )
1437 main.Mininet2.pingLong(
1438 src=main.params[ 'PING' ][ 'source9' ],
1439 target=main.params[ 'PING' ][ 'target9' ],
1440 pingTime=500 )
1441 main.Mininet2.pingLong(
1442 src=main.params[ 'PING' ][ 'source10' ],
1443 target=main.params[ 'PING' ][ 'target10' ],
1444 pingTime=500 )
1445
1446 main.step( "Collecting topology information from ONOS" )
1447 devices = []
1448 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001449 for i in range( main.numCtrls ):
1450 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001451 name="devices-" + str( i ),
1452 args=[ ] )
1453 threads.append( t )
1454 t.start()
1455
1456 for t in threads:
1457 t.join()
1458 devices.append( t.result )
1459 hosts = []
1460 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001461 for i in range( main.numCtrls ):
1462 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001463 name="hosts-" + str( i ),
1464 args=[ ] )
1465 threads.append( t )
1466 t.start()
1467
1468 for t in threads:
1469 t.join()
1470 try:
1471 hosts.append( json.loads( t.result ) )
1472 except ( ValueError, TypeError ):
1473 # FIXME: better handling of this, print which node
1474 # Maybe use thread name?
1475 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001476 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001477 hosts.append( None )
1478
1479 ports = []
1480 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001481 for i in range( main.numCtrls ):
1482 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001483 name="ports-" + str( i ),
1484 args=[ ] )
1485 threads.append( t )
1486 t.start()
1487
1488 for t in threads:
1489 t.join()
1490 ports.append( t.result )
1491 links = []
1492 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001493 for i in range( main.numCtrls ):
1494 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001495 name="links-" + str( i ),
1496 args=[ ] )
1497 threads.append( t )
1498 t.start()
1499
1500 for t in threads:
1501 t.join()
1502 links.append( t.result )
1503 clusters = []
1504 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001505 for i in range( main.numCtrls ):
1506 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001507 name="clusters-" + str( i ),
1508 args=[ ] )
1509 threads.append( t )
1510 t.start()
1511
1512 for t in threads:
1513 t.join()
1514 clusters.append( t.result )
1515 # Compare json objects for hosts and dataplane clusters
1516
1517 # hosts
1518 main.step( "Host view is consistent across ONOS nodes" )
1519 consistentHostsResult = main.TRUE
1520 for controller in range( len( hosts ) ):
1521 controllerStr = str( controller + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001522 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001523 if hosts[ controller ] == hosts[ 0 ]:
1524 continue
1525 else: # hosts not consistent
1526 main.log.error( "hosts from ONOS" +
1527 controllerStr +
1528 " is inconsistent with ONOS1" )
1529 main.log.warn( repr( hosts[ controller ] ) )
1530 consistentHostsResult = main.FALSE
1531
1532 else:
1533 main.log.error( "Error in getting ONOS hosts from ONOS" +
1534 controllerStr )
1535 consistentHostsResult = main.FALSE
1536 main.log.warn( "ONOS" + controllerStr +
1537 " hosts response: " +
1538 repr( hosts[ controller ] ) )
1539 utilities.assert_equals(
1540 expect=main.TRUE,
1541 actual=consistentHostsResult,
1542 onpass="Hosts view is consistent across all ONOS nodes",
1543 onfail="ONOS nodes have different views of hosts" )
1544
1545 main.step( "Each host has an IP address" )
1546 ipResult = main.TRUE
1547 for controller in range( 0, len( hosts ) ):
1548 controllerStr = str( controller + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001549 if hosts[ controller ]:
1550 for host in hosts[ controller ]:
1551 if not host.get( 'ipAddresses', [ ] ):
1552 main.log.error( "Error with host ips on controller" +
1553 controllerStr + ": " + str( host ) )
1554 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001555 utilities.assert_equals(
1556 expect=main.TRUE,
1557 actual=ipResult,
1558 onpass="The ips of the hosts aren't empty",
1559 onfail="The ip of at least one host is missing" )
1560
1561 # Strongly connected clusters of devices
1562 main.step( "Cluster view is consistent across ONOS nodes" )
1563 consistentClustersResult = main.TRUE
1564 for controller in range( len( clusters ) ):
1565 controllerStr = str( controller + 1 )
1566 if "Error" not in clusters[ controller ]:
1567 if clusters[ controller ] == clusters[ 0 ]:
1568 continue
1569 else: # clusters not consistent
1570 main.log.error( "clusters from ONOS" + controllerStr +
1571 " is inconsistent with ONOS1" )
1572 consistentClustersResult = main.FALSE
1573
1574 else:
1575 main.log.error( "Error in getting dataplane clusters " +
1576 "from ONOS" + controllerStr )
1577 consistentClustersResult = main.FALSE
1578 main.log.warn( "ONOS" + controllerStr +
1579 " clusters response: " +
1580 repr( clusters[ controller ] ) )
1581 utilities.assert_equals(
1582 expect=main.TRUE,
1583 actual=consistentClustersResult,
1584 onpass="Clusters view is consistent across all ONOS nodes",
1585 onfail="ONOS nodes have different views of clusters" )
1586 # there should always only be one cluster
1587 main.step( "Cluster view correct across ONOS nodes" )
1588 try:
1589 numClusters = len( json.loads( clusters[ 0 ] ) )
1590 except ( ValueError, TypeError ):
1591 main.log.exception( "Error parsing clusters[0]: " +
1592 repr( clusters[ 0 ] ) )
1593 clusterResults = main.FALSE
1594 if numClusters == 1:
1595 clusterResults = main.TRUE
1596 utilities.assert_equals(
1597 expect=1,
1598 actual=numClusters,
1599 onpass="ONOS shows 1 SCC",
1600 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1601
1602 main.step( "Comparing ONOS topology to MN" )
1603 devicesResults = main.TRUE
1604 linksResults = main.TRUE
1605 hostsResults = main.TRUE
1606 mnSwitches = main.Mininet1.getSwitches()
1607 mnLinks = main.Mininet1.getLinks()
1608 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001609 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001610 controllerStr = str( controller + 1 )
1611 if devices[ controller ] and ports[ controller ] and\
1612 "Error" not in devices[ controller ] and\
1613 "Error" not in ports[ controller ]:
1614
1615 currentDevicesResult = main.Mininet1.compareSwitches(
1616 mnSwitches,
1617 json.loads( devices[ controller ] ),
1618 json.loads( ports[ controller ] ) )
1619 else:
1620 currentDevicesResult = main.FALSE
1621 utilities.assert_equals( expect=main.TRUE,
1622 actual=currentDevicesResult,
1623 onpass="ONOS" + controllerStr +
1624 " Switches view is correct",
1625 onfail="ONOS" + controllerStr +
1626 " Switches view is incorrect" )
1627 if links[ controller ] and "Error" not in links[ controller ]:
1628 currentLinksResult = main.Mininet1.compareLinks(
1629 mnSwitches, mnLinks,
1630 json.loads( links[ controller ] ) )
1631 else:
1632 currentLinksResult = main.FALSE
1633 utilities.assert_equals( expect=main.TRUE,
1634 actual=currentLinksResult,
1635 onpass="ONOS" + controllerStr +
1636 " links view is correct",
1637 onfail="ONOS" + controllerStr +
1638 " links view is incorrect" )
1639
Jon Hall657cdf62015-12-17 14:40:51 -08001640 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001641 currentHostsResult = main.Mininet1.compareHosts(
1642 mnHosts,
1643 hosts[ controller ] )
1644 else:
1645 currentHostsResult = main.FALSE
1646 utilities.assert_equals( expect=main.TRUE,
1647 actual=currentHostsResult,
1648 onpass="ONOS" + controllerStr +
1649 " hosts exist in Mininet",
1650 onfail="ONOS" + controllerStr +
1651 " hosts don't match Mininet" )
1652
1653 devicesResults = devicesResults and currentDevicesResult
1654 linksResults = linksResults and currentLinksResult
1655 hostsResults = hostsResults and currentHostsResult
1656
1657 main.step( "Device information is correct" )
1658 utilities.assert_equals(
1659 expect=main.TRUE,
1660 actual=devicesResults,
1661 onpass="Device information is correct",
1662 onfail="Device information is incorrect" )
1663
1664 main.step( "Links are correct" )
1665 utilities.assert_equals(
1666 expect=main.TRUE,
1667 actual=linksResults,
1668 onpass="Link are correct",
1669 onfail="Links are incorrect" )
1670
1671 main.step( "Hosts are correct" )
1672 utilities.assert_equals(
1673 expect=main.TRUE,
1674 actual=hostsResults,
1675 onpass="Hosts are correct",
1676 onfail="Hosts are incorrect" )
1677
1678 def CASE6( self, main ):
1679 """
1680 The Failure case. Since this is the Sanity test, we do nothing.
1681 """
1682 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001683 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001684 assert main, "main not defined"
1685 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001686 assert main.CLIs, "main.CLIs not defined"
1687 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001688 main.case( "Wait 60 seconds instead of inducing a failure" )
1689 time.sleep( 60 )
1690 utilities.assert_equals(
1691 expect=main.TRUE,
1692 actual=main.TRUE,
1693 onpass="Sleeping 60 seconds",
1694 onfail="Something is terribly wrong with my math" )
1695
1696 def CASE7( self, main ):
1697 """
1698 Check state after ONOS failure
1699 """
1700 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001701 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001702 assert main, "main not defined"
1703 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001704 assert main.CLIs, "main.CLIs not defined"
1705 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001706 main.case( "Running ONOS Constant State Tests" )
1707
1708 main.step( "Check that each switch has a master" )
1709 # Assert that each device has a master
1710 rolesNotNull = main.TRUE
1711 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001712 for i in range( main.numCtrls ):
1713 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001714 name="rolesNotNull-" + str( i ),
1715 args=[ ] )
1716 threads.append( t )
1717 t.start()
1718
1719 for t in threads:
1720 t.join()
1721 rolesNotNull = rolesNotNull and t.result
1722 utilities.assert_equals(
1723 expect=main.TRUE,
1724 actual=rolesNotNull,
1725 onpass="Each device has a master",
1726 onfail="Some devices don't have a master assigned" )
1727
1728 main.step( "Read device roles from ONOS" )
1729 ONOSMastership = []
1730 mastershipCheck = main.FALSE
1731 consistentMastership = True
1732 rolesResults = True
1733 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001734 for i in range( main.numCtrls ):
1735 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001736 name="roles-" + str( i ),
1737 args=[] )
1738 threads.append( t )
1739 t.start()
1740
1741 for t in threads:
1742 t.join()
1743 ONOSMastership.append( t.result )
1744
Jon Halle1a3b752015-07-22 13:02:46 -07001745 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001746 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1747 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1748 " roles" )
1749 main.log.warn(
1750 "ONOS" + str( i + 1 ) + " mastership response: " +
1751 repr( ONOSMastership[i] ) )
1752 rolesResults = False
1753 utilities.assert_equals(
1754 expect=True,
1755 actual=rolesResults,
1756 onpass="No error in reading roles output",
1757 onfail="Error in reading roles from ONOS" )
1758
1759 main.step( "Check for consistency in roles from each controller" )
1760 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1761 main.log.info(
1762 "Switch roles are consistent across all ONOS nodes" )
1763 else:
1764 consistentMastership = False
1765 utilities.assert_equals(
1766 expect=True,
1767 actual=consistentMastership,
1768 onpass="Switch roles are consistent across all ONOS nodes",
1769 onfail="ONOS nodes have different views of switch roles" )
1770
1771 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001772 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001773 main.log.warn(
1774 "ONOS" + str( i + 1 ) + " roles: ",
1775 json.dumps(
1776 json.loads( ONOSMastership[ i ] ),
1777 sort_keys=True,
1778 indent=4,
1779 separators=( ',', ': ' ) ) )
1780 elif rolesResults and not consistentMastership:
1781 mastershipCheck = main.TRUE
1782
1783 description2 = "Compare switch roles from before failure"
1784 main.step( description2 )
1785 try:
1786 currentJson = json.loads( ONOSMastership[0] )
1787 oldJson = json.loads( mastershipState )
1788 except ( ValueError, TypeError ):
1789 main.log.exception( "Something is wrong with parsing " +
1790 "ONOSMastership[0] or mastershipState" )
1791 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1792 main.log.error( "mastershipState" + repr( mastershipState ) )
1793 main.cleanup()
1794 main.exit()
1795 mastershipCheck = main.TRUE
1796 for i in range( 1, 29 ):
1797 switchDPID = str(
1798 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1799 current = [ switch[ 'master' ] for switch in currentJson
1800 if switchDPID in switch[ 'id' ] ]
1801 old = [ switch[ 'master' ] for switch in oldJson
1802 if switchDPID in switch[ 'id' ] ]
1803 if current == old:
1804 mastershipCheck = mastershipCheck and main.TRUE
1805 else:
1806 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1807 mastershipCheck = main.FALSE
1808 utilities.assert_equals(
1809 expect=main.TRUE,
1810 actual=mastershipCheck,
1811 onpass="Mastership of Switches was not changed",
1812 onfail="Mastership of some switches changed" )
1813 mastershipCheck = mastershipCheck and consistentMastership
1814
1815 main.step( "Get the intents and compare across all nodes" )
1816 ONOSIntents = []
1817 intentCheck = main.FALSE
1818 consistentIntents = True
1819 intentsResults = True
1820 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001821 for i in range( main.numCtrls ):
1822 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001823 name="intents-" + str( i ),
1824 args=[],
1825 kwargs={ 'jsonFormat': True } )
1826 threads.append( t )
1827 t.start()
1828
1829 for t in threads:
1830 t.join()
1831 ONOSIntents.append( t.result )
1832
Jon Halle1a3b752015-07-22 13:02:46 -07001833 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001834 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1835 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1836 " intents" )
1837 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1838 repr( ONOSIntents[ i ] ) )
1839 intentsResults = False
1840 utilities.assert_equals(
1841 expect=True,
1842 actual=intentsResults,
1843 onpass="No error in reading intents output",
1844 onfail="Error in reading intents from ONOS" )
1845
1846 main.step( "Check for consistency in Intents from each controller" )
1847 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1848 main.log.info( "Intents are consistent across all ONOS " +
1849 "nodes" )
1850 else:
1851 consistentIntents = False
1852
1853 # Try to make it easy to figure out what is happening
1854 #
1855 # Intent ONOS1 ONOS2 ...
1856 # 0x01 INSTALLED INSTALLING
1857 # ... ... ...
1858 # ... ... ...
1859 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001860 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001861 title += " " * 10 + "ONOS" + str( n + 1 )
1862 main.log.warn( title )
1863 # get all intent keys in the cluster
1864 keys = []
1865 for nodeStr in ONOSIntents:
1866 node = json.loads( nodeStr )
1867 for intent in node:
1868 keys.append( intent.get( 'id' ) )
1869 keys = set( keys )
1870 for key in keys:
1871 row = "%-13s" % key
1872 for nodeStr in ONOSIntents:
1873 node = json.loads( nodeStr )
1874 for intent in node:
1875 if intent.get( 'id' ) == key:
1876 row += "%-15s" % intent.get( 'state' )
1877 main.log.warn( row )
1878 # End table view
1879
1880 utilities.assert_equals(
1881 expect=True,
1882 actual=consistentIntents,
1883 onpass="Intents are consistent across all ONOS nodes",
1884 onfail="ONOS nodes have different views of intents" )
1885 intentStates = []
1886 for node in ONOSIntents: # Iter through ONOS nodes
1887 nodeStates = []
1888 # Iter through intents of a node
1889 try:
1890 for intent in json.loads( node ):
1891 nodeStates.append( intent[ 'state' ] )
1892 except ( ValueError, TypeError ):
1893 main.log.exception( "Error in parsing intents" )
1894 main.log.error( repr( node ) )
1895 intentStates.append( nodeStates )
1896 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1897 main.log.info( dict( out ) )
1898
1899 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001900 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001901 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1902 main.log.warn( json.dumps(
1903 json.loads( ONOSIntents[ i ] ),
1904 sort_keys=True,
1905 indent=4,
1906 separators=( ',', ': ' ) ) )
1907 elif intentsResults and consistentIntents:
1908 intentCheck = main.TRUE
1909
1910 # NOTE: Store has no durability, so intents are lost across system
1911 # restarts
1912 main.step( "Compare current intents with intents before the failure" )
1913 # NOTE: this requires case 5 to pass for intentState to be set.
1914 # maybe we should stop the test if that fails?
1915 sameIntents = main.FALSE
1916 if intentState and intentState == ONOSIntents[ 0 ]:
1917 sameIntents = main.TRUE
1918 main.log.info( "Intents are consistent with before failure" )
1919 # TODO: possibly the states have changed? we may need to figure out
1920 # what the acceptable states are
1921 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1922 sameIntents = main.TRUE
1923 try:
1924 before = json.loads( intentState )
1925 after = json.loads( ONOSIntents[ 0 ] )
1926 for intent in before:
1927 if intent not in after:
1928 sameIntents = main.FALSE
1929 main.log.debug( "Intent is not currently in ONOS " +
1930 "(at least in the same form):" )
1931 main.log.debug( json.dumps( intent ) )
1932 except ( ValueError, TypeError ):
1933 main.log.exception( "Exception printing intents" )
1934 main.log.debug( repr( ONOSIntents[0] ) )
1935 main.log.debug( repr( intentState ) )
1936 if sameIntents == main.FALSE:
1937 try:
1938 main.log.debug( "ONOS intents before: " )
1939 main.log.debug( json.dumps( json.loads( intentState ),
1940 sort_keys=True, indent=4,
1941 separators=( ',', ': ' ) ) )
1942 main.log.debug( "Current ONOS intents: " )
1943 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1944 sort_keys=True, indent=4,
1945 separators=( ',', ': ' ) ) )
1946 except ( ValueError, TypeError ):
1947 main.log.exception( "Exception printing intents" )
1948 main.log.debug( repr( ONOSIntents[0] ) )
1949 main.log.debug( repr( intentState ) )
1950 utilities.assert_equals(
1951 expect=main.TRUE,
1952 actual=sameIntents,
1953 onpass="Intents are consistent with before failure",
1954 onfail="The Intents changed during failure" )
1955 intentCheck = intentCheck and sameIntents
1956
1957 main.step( "Get the OF Table entries and compare to before " +
1958 "component failure" )
1959 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07001960 for i in range( 28 ):
1961 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08001962 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
1963 FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
Jon Hall5cf14d52015-07-16 12:15:19 -07001964 if FlowTables == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08001965 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
1966
Jon Hall5cf14d52015-07-16 12:15:19 -07001967 utilities.assert_equals(
1968 expect=main.TRUE,
1969 actual=FlowTables,
1970 onpass="No changes were found in the flow tables",
1971 onfail="Changes were found in the flow tables" )
1972
1973 main.Mininet2.pingLongKill()
1974 '''
1975 main.step( "Check the continuous pings to ensure that no packets " +
1976 "were dropped during component failure" )
1977 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1978 main.params[ 'TESTONIP' ] )
1979 LossInPings = main.FALSE
1980 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1981 for i in range( 8, 18 ):
1982 main.log.info(
1983 "Checking for a loss in pings along flow from s" +
1984 str( i ) )
1985 LossInPings = main.Mininet2.checkForLoss(
1986 "/tmp/ping.h" +
1987 str( i ) ) or LossInPings
1988 if LossInPings == main.TRUE:
1989 main.log.info( "Loss in ping detected" )
1990 elif LossInPings == main.ERROR:
1991 main.log.info( "There are multiple mininet process running" )
1992 elif LossInPings == main.FALSE:
1993 main.log.info( "No Loss in the pings" )
1994 main.log.info( "No loss of dataplane connectivity" )
1995 utilities.assert_equals(
1996 expect=main.FALSE,
1997 actual=LossInPings,
1998 onpass="No Loss of connectivity",
1999 onfail="Loss of dataplane connectivity detected" )
2000 '''
2001
2002 main.step( "Leadership Election is still functional" )
2003 # Test of LeadershipElection
2004 # NOTE: this only works for the sanity test. In case of failures,
2005 # leader will likely change
Jon Halle1a3b752015-07-22 13:02:46 -07002006 leader = main.nodes[ 0 ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002007 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002008 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002009 leaderN = cli.electionTestLeader()
2010 # verify leader is ONOS1
2011 if leaderN == leader:
2012 # all is well
2013 # NOTE: In failure scenario, this could be a new node, maybe
2014 # check != ONOS1
2015 pass
2016 elif leaderN == main.FALSE:
2017 # error in response
2018 main.log.error( "Something is wrong with " +
2019 "electionTestLeader function, check the" +
2020 " error logs" )
2021 leaderResult = main.FALSE
2022 elif leader != leaderN:
2023 leaderResult = main.FALSE
2024 main.log.error( cli.name + " sees " + str( leaderN ) +
2025 " as the leader of the election app. " +
2026 "Leader should be " + str( leader ) )
2027 utilities.assert_equals(
2028 expect=main.TRUE,
2029 actual=leaderResult,
2030 onpass="Leadership election passed",
2031 onfail="Something went wrong with Leadership election" )
2032
2033 def CASE8( self, main ):
2034 """
2035 Compare topo
2036 """
2037 import json
2038 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002039 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002040 assert main, "main not defined"
2041 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002042 assert main.CLIs, "main.CLIs not defined"
2043 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002044
2045 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002046 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002047 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002048 topoResult = main.FALSE
2049 elapsed = 0
2050 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002051 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002052 startTime = time.time()
2053 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002054 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002055 devicesResults = main.TRUE
2056 linksResults = main.TRUE
2057 hostsResults = main.TRUE
2058 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002059 count += 1
2060 cliStart = time.time()
2061 devices = []
2062 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002063 for i in range( main.numCtrls ):
2064 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002065 name="devices-" + str( i ),
2066 args=[ ] )
2067 threads.append( t )
2068 t.start()
2069
2070 for t in threads:
2071 t.join()
2072 devices.append( t.result )
2073 hosts = []
2074 ipResult = main.TRUE
2075 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002076 for i in range( main.numCtrls ):
Jon Hallbaf53162015-12-17 17:04:34 -08002077 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002078 name="hosts-" + str( i ),
Jon Hallbaf53162015-12-17 17:04:34 -08002079 args=[ main.CLIs[i].hosts, [ None ] ],
2080 kwargs= { 'sleep': 5, 'attempts': 5,
2081 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002082 threads.append( t )
2083 t.start()
2084
2085 for t in threads:
2086 t.join()
2087 try:
2088 hosts.append( json.loads( t.result ) )
2089 except ( ValueError, TypeError ):
2090 main.log.exception( "Error parsing hosts results" )
2091 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002092 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002093 for controller in range( 0, len( hosts ) ):
2094 controllerStr = str( controller + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002095 if hosts[ controller ]:
2096 for host in hosts[ controller ]:
2097 if host is None or host.get( 'ipAddresses', [] ) == []:
2098 main.log.error(
2099 "Error with host ipAddresses on controller" +
2100 controllerStr + ": " + str( host ) )
2101 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002102 ports = []
2103 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002104 for i in range( main.numCtrls ):
2105 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002106 name="ports-" + str( i ),
2107 args=[ ] )
2108 threads.append( t )
2109 t.start()
2110
2111 for t in threads:
2112 t.join()
2113 ports.append( t.result )
2114 links = []
2115 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002116 for i in range( main.numCtrls ):
2117 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002118 name="links-" + str( i ),
2119 args=[ ] )
2120 threads.append( t )
2121 t.start()
2122
2123 for t in threads:
2124 t.join()
2125 links.append( t.result )
2126 clusters = []
2127 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002128 for i in range( main.numCtrls ):
2129 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002130 name="clusters-" + str( i ),
2131 args=[ ] )
2132 threads.append( t )
2133 t.start()
2134
2135 for t in threads:
2136 t.join()
2137 clusters.append( t.result )
2138
2139 elapsed = time.time() - startTime
2140 cliTime = time.time() - cliStart
2141 print "Elapsed time: " + str( elapsed )
2142 print "CLI time: " + str( cliTime )
2143
2144 mnSwitches = main.Mininet1.getSwitches()
2145 mnLinks = main.Mininet1.getLinks()
2146 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002147 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002148 controllerStr = str( controller + 1 )
2149 if devices[ controller ] and ports[ controller ] and\
2150 "Error" not in devices[ controller ] and\
2151 "Error" not in ports[ controller ]:
2152
2153 currentDevicesResult = main.Mininet1.compareSwitches(
2154 mnSwitches,
2155 json.loads( devices[ controller ] ),
2156 json.loads( ports[ controller ] ) )
2157 else:
2158 currentDevicesResult = main.FALSE
2159 utilities.assert_equals( expect=main.TRUE,
2160 actual=currentDevicesResult,
2161 onpass="ONOS" + controllerStr +
2162 " Switches view is correct",
2163 onfail="ONOS" + controllerStr +
2164 " Switches view is incorrect" )
2165
2166 if links[ controller ] and "Error" not in links[ controller ]:
2167 currentLinksResult = main.Mininet1.compareLinks(
2168 mnSwitches, mnLinks,
2169 json.loads( links[ controller ] ) )
2170 else:
2171 currentLinksResult = main.FALSE
2172 utilities.assert_equals( expect=main.TRUE,
2173 actual=currentLinksResult,
2174 onpass="ONOS" + controllerStr +
2175 " links view is correct",
2176 onfail="ONOS" + controllerStr +
2177 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002178 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002179 currentHostsResult = main.Mininet1.compareHosts(
2180 mnHosts,
2181 hosts[ controller ] )
Jon Hallaf39dcc2016-01-05 12:17:01 -08002182 elif hosts[ controller ] == []:
2183 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002184 else:
2185 currentHostsResult = main.FALSE
2186 utilities.assert_equals( expect=main.TRUE,
2187 actual=currentHostsResult,
2188 onpass="ONOS" + controllerStr +
2189 " hosts exist in Mininet",
2190 onfail="ONOS" + controllerStr +
2191 " hosts don't match Mininet" )
2192 # CHECKING HOST ATTACHMENT POINTS
2193 hostAttachment = True
2194 zeroHosts = False
2195 # FIXME: topo-HA/obelisk specific mappings:
2196 # key is mac and value is dpid
2197 mappings = {}
2198 for i in range( 1, 29 ): # hosts 1 through 28
2199 # set up correct variables:
2200 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2201 if i == 1:
2202 deviceId = "1000".zfill(16)
2203 elif i == 2:
2204 deviceId = "2000".zfill(16)
2205 elif i == 3:
2206 deviceId = "3000".zfill(16)
2207 elif i == 4:
2208 deviceId = "3004".zfill(16)
2209 elif i == 5:
2210 deviceId = "5000".zfill(16)
2211 elif i == 6:
2212 deviceId = "6000".zfill(16)
2213 elif i == 7:
2214 deviceId = "6007".zfill(16)
2215 elif i >= 8 and i <= 17:
2216 dpid = '3' + str( i ).zfill( 3 )
2217 deviceId = dpid.zfill(16)
2218 elif i >= 18 and i <= 27:
2219 dpid = '6' + str( i ).zfill( 3 )
2220 deviceId = dpid.zfill(16)
2221 elif i == 28:
2222 deviceId = "2800".zfill(16)
2223 mappings[ macId ] = deviceId
Jon Hallbaf53162015-12-17 17:04:34 -08002224 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002225 if hosts[ controller ] == []:
2226 main.log.warn( "There are no hosts discovered" )
2227 zeroHosts = True
2228 else:
2229 for host in hosts[ controller ]:
2230 mac = None
2231 location = None
2232 device = None
2233 port = None
2234 try:
2235 mac = host.get( 'mac' )
2236 assert mac, "mac field could not be found for this host object"
2237
2238 location = host.get( 'location' )
2239 assert location, "location field could not be found for this host object"
2240
2241 # Trim the protocol identifier off deviceId
2242 device = str( location.get( 'elementId' ) ).split(':')[1]
2243 assert device, "elementId field could not be found for this host location object"
2244
2245 port = location.get( 'port' )
2246 assert port, "port field could not be found for this host location object"
2247
2248 # Now check if this matches where they should be
2249 if mac and device and port:
2250 if str( port ) != "1":
2251 main.log.error( "The attachment port is incorrect for " +
2252 "host " + str( mac ) +
2253 ". Expected: 1 Actual: " + str( port) )
2254 hostAttachment = False
2255 if device != mappings[ str( mac ) ]:
2256 main.log.error( "The attachment device is incorrect for " +
2257 "host " + str( mac ) +
2258 ". Expected: " + mappings[ str( mac ) ] +
2259 " Actual: " + device )
2260 hostAttachment = False
2261 else:
2262 hostAttachment = False
2263 except AssertionError:
2264 main.log.exception( "Json object not as expected" )
2265 main.log.error( repr( host ) )
2266 hostAttachment = False
2267 else:
2268 main.log.error( "No hosts json output or \"Error\"" +
2269 " in output. hosts = " +
2270 repr( hosts[ controller ] ) )
2271 if zeroHosts is False:
2272 hostAttachment = True
2273
2274 # END CHECKING HOST ATTACHMENT POINTS
2275 devicesResults = devicesResults and currentDevicesResult
2276 linksResults = linksResults and currentLinksResult
2277 hostsResults = hostsResults and currentHostsResult
2278 hostAttachmentResults = hostAttachmentResults and\
2279 hostAttachment
2280 topoResult = ( devicesResults and linksResults
2281 and hostsResults and ipResult and
2282 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002283 utilities.assert_equals( expect=True,
2284 actual=topoResult,
2285 onpass="ONOS topology matches Mininet",
2286 onfail="ONOS topology don't match Mininet" )
2287 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002288
2289 # Compare json objects for hosts and dataplane clusters
2290
2291 # hosts
2292 main.step( "Hosts view is consistent across all ONOS nodes" )
2293 consistentHostsResult = main.TRUE
2294 for controller in range( len( hosts ) ):
2295 controllerStr = str( controller + 1 )
Jon Hallaf39dcc2016-01-05 12:17:01 -08002296 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002297 if hosts[ controller ] == hosts[ 0 ]:
2298 continue
2299 else: # hosts not consistent
2300 main.log.error( "hosts from ONOS" + controllerStr +
2301 " is inconsistent with ONOS1" )
2302 main.log.warn( repr( hosts[ controller ] ) )
2303 consistentHostsResult = main.FALSE
2304
2305 else:
2306 main.log.error( "Error in getting ONOS hosts from ONOS" +
2307 controllerStr )
2308 consistentHostsResult = main.FALSE
2309 main.log.warn( "ONOS" + controllerStr +
2310 " hosts response: " +
2311 repr( hosts[ controller ] ) )
2312 utilities.assert_equals(
2313 expect=main.TRUE,
2314 actual=consistentHostsResult,
2315 onpass="Hosts view is consistent across all ONOS nodes",
2316 onfail="ONOS nodes have different views of hosts" )
2317
2318 main.step( "Hosts information is correct" )
2319 hostsResults = hostsResults and ipResult
2320 utilities.assert_equals(
2321 expect=main.TRUE,
2322 actual=hostsResults,
2323 onpass="Host information is correct",
2324 onfail="Host information is incorrect" )
2325
2326 main.step( "Host attachment points to the network" )
2327 utilities.assert_equals(
2328 expect=True,
2329 actual=hostAttachmentResults,
2330 onpass="Hosts are correctly attached to the network",
2331 onfail="ONOS did not correctly attach hosts to the network" )
2332
2333 # Strongly connected clusters of devices
2334 main.step( "Clusters view is consistent across all ONOS nodes" )
2335 consistentClustersResult = main.TRUE
2336 for controller in range( len( clusters ) ):
2337 controllerStr = str( controller + 1 )
2338 if "Error" not in clusters[ controller ]:
2339 if clusters[ controller ] == clusters[ 0 ]:
2340 continue
2341 else: # clusters not consistent
2342 main.log.error( "clusters from ONOS" +
2343 controllerStr +
2344 " is inconsistent with ONOS1" )
2345 consistentClustersResult = main.FALSE
2346
2347 else:
2348 main.log.error( "Error in getting dataplane clusters " +
2349 "from ONOS" + controllerStr )
2350 consistentClustersResult = main.FALSE
2351 main.log.warn( "ONOS" + controllerStr +
2352 " clusters response: " +
2353 repr( clusters[ controller ] ) )
2354 utilities.assert_equals(
2355 expect=main.TRUE,
2356 actual=consistentClustersResult,
2357 onpass="Clusters view is consistent across all ONOS nodes",
2358 onfail="ONOS nodes have different views of clusters" )
2359
2360 main.step( "There is only one SCC" )
2361 # there should always only be one cluster
2362 try:
2363 numClusters = len( json.loads( clusters[ 0 ] ) )
2364 except ( ValueError, TypeError ):
2365 main.log.exception( "Error parsing clusters[0]: " +
2366 repr( clusters[0] ) )
2367 clusterResults = main.FALSE
2368 if numClusters == 1:
2369 clusterResults = main.TRUE
2370 utilities.assert_equals(
2371 expect=1,
2372 actual=numClusters,
2373 onpass="ONOS shows 1 SCC",
2374 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2375
2376 topoResult = ( devicesResults and linksResults
2377 and hostsResults and consistentHostsResult
2378 and consistentClustersResult and clusterResults
2379 and ipResult and hostAttachmentResults )
2380
2381 topoResult = topoResult and int( count <= 2 )
2382 note = "note it takes about " + str( int( cliTime ) ) + \
2383 " seconds for the test to make all the cli calls to fetch " +\
2384 "the topology from each ONOS instance"
2385 main.log.info(
2386 "Very crass estimate for topology discovery/convergence( " +
2387 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2388 str( count ) + " tries" )
2389
2390 main.step( "Device information is correct" )
2391 utilities.assert_equals(
2392 expect=main.TRUE,
2393 actual=devicesResults,
2394 onpass="Device information is correct",
2395 onfail="Device information is incorrect" )
2396
2397 main.step( "Links are correct" )
2398 utilities.assert_equals(
2399 expect=main.TRUE,
2400 actual=linksResults,
2401 onpass="Link are correct",
2402 onfail="Links are incorrect" )
2403
2404 main.step( "Hosts are correct" )
2405 utilities.assert_equals(
2406 expect=main.TRUE,
2407 actual=hostsResults,
2408 onpass="Hosts are correct",
2409 onfail="Hosts are incorrect" )
2410
2411 # FIXME: move this to an ONOS state case
2412 main.step( "Checking ONOS nodes" )
2413 nodesOutput = []
2414 nodeResults = main.TRUE
2415 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002416 for i in range( main.numCtrls ):
2417 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002418 name="nodes-" + str( i ),
2419 args=[ ] )
2420 threads.append( t )
2421 t.start()
2422
2423 for t in threads:
2424 t.join()
2425 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002426 ips = [ node.ip_address for node in main.nodes ]
Jon Halle9b1fa32015-12-08 15:32:21 -08002427 ips.sort()
Jon Hall5cf14d52015-07-16 12:15:19 -07002428 for i in nodesOutput:
2429 try:
2430 current = json.loads( i )
Jon Halle9b1fa32015-12-08 15:32:21 -08002431 activeIps = []
2432 currentResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002433 for node in current:
Jon Halle9b1fa32015-12-08 15:32:21 -08002434 if node['state'] == 'ACTIVE':
2435 activeIps.append( node['ip'] )
2436 activeIps.sort()
2437 if ips == activeIps:
2438 currentResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002439 except ( ValueError, TypeError ):
2440 main.log.error( "Error parsing nodes output" )
2441 main.log.warn( repr( i ) )
Jon Halle9b1fa32015-12-08 15:32:21 -08002442 currentResult = main.FALSE
2443 nodeResults = nodeResults and currentResult
Jon Hall5cf14d52015-07-16 12:15:19 -07002444 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2445 onpass="Nodes check successful",
2446 onfail="Nodes check NOT successful" )
2447
2448 def CASE9( self, main ):
2449 """
2450 Link s3-s28 down
2451 """
2452 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002453 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002454 assert main, "main not defined"
2455 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002456 assert main.CLIs, "main.CLIs not defined"
2457 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002458 # NOTE: You should probably run a topology check after this
2459
2460 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2461
2462 description = "Turn off a link to ensure that Link Discovery " +\
2463 "is working properly"
2464 main.case( description )
2465
2466 main.step( "Kill Link between s3 and s28" )
2467 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2468 main.log.info( "Waiting " + str( linkSleep ) +
2469 " seconds for link down to be discovered" )
2470 time.sleep( linkSleep )
2471 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2472 onpass="Link down successful",
2473 onfail="Failed to bring link down" )
2474 # TODO do some sort of check here
2475
2476 def CASE10( self, main ):
2477 """
2478 Link s3-s28 up
2479 """
2480 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002481 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002482 assert main, "main not defined"
2483 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002484 assert main.CLIs, "main.CLIs not defined"
2485 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002486 # NOTE: You should probably run a topology check after this
2487
2488 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2489
2490 description = "Restore a link to ensure that Link Discovery is " + \
2491 "working properly"
2492 main.case( description )
2493
2494 main.step( "Bring link between s3 and s28 back up" )
2495 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2496 main.log.info( "Waiting " + str( linkSleep ) +
2497 " seconds for link up to be discovered" )
2498 time.sleep( linkSleep )
2499 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2500 onpass="Link up successful",
2501 onfail="Failed to bring link up" )
2502 # TODO do some sort of check here
2503
2504 def CASE11( self, main ):
2505 """
2506 Switch Down
2507 """
2508 # NOTE: You should probably run a topology check after this
2509 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002510 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002511 assert main, "main not defined"
2512 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002513 assert main.CLIs, "main.CLIs not defined"
2514 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002515
2516 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2517
2518 description = "Killing a switch to ensure it is discovered correctly"
2519 main.case( description )
2520 switch = main.params[ 'kill' ][ 'switch' ]
2521 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2522
2523 # TODO: Make this switch parameterizable
2524 main.step( "Kill " + switch )
2525 main.log.info( "Deleting " + switch )
2526 main.Mininet1.delSwitch( switch )
2527 main.log.info( "Waiting " + str( switchSleep ) +
2528 " seconds for switch down to be discovered" )
2529 time.sleep( switchSleep )
2530 device = main.ONOScli1.getDevice( dpid=switchDPID )
2531 # Peek at the deleted switch
2532 main.log.warn( str( device ) )
2533 result = main.FALSE
2534 if device and device[ 'available' ] is False:
2535 result = main.TRUE
2536 utilities.assert_equals( expect=main.TRUE, actual=result,
2537 onpass="Kill switch successful",
2538 onfail="Failed to kill switch?" )
2539
2540 def CASE12( self, main ):
2541 """
2542 Switch Up
2543 """
2544 # NOTE: You should probably run a topology check after this
2545 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002546 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002547 assert main, "main not defined"
2548 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002549 assert main.CLIs, "main.CLIs not defined"
2550 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002551 assert ONOS1Port, "ONOS1Port not defined"
2552 assert ONOS2Port, "ONOS2Port not defined"
2553 assert ONOS3Port, "ONOS3Port not defined"
2554 assert ONOS4Port, "ONOS4Port not defined"
2555 assert ONOS5Port, "ONOS5Port not defined"
2556 assert ONOS6Port, "ONOS6Port not defined"
2557 assert ONOS7Port, "ONOS7Port not defined"
2558
2559 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2560 switch = main.params[ 'kill' ][ 'switch' ]
2561 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2562 links = main.params[ 'kill' ][ 'links' ].split()
2563 description = "Adding a switch to ensure it is discovered correctly"
2564 main.case( description )
2565
2566 main.step( "Add back " + switch )
2567 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2568 for peer in links:
2569 main.Mininet1.addLink( switch, peer )
2570 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002571 for i in range( main.numCtrls ):
2572 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002573 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2574 main.log.info( "Waiting " + str( switchSleep ) +
2575 " seconds for switch up to be discovered" )
2576 time.sleep( switchSleep )
2577 device = main.ONOScli1.getDevice( dpid=switchDPID )
2578 # Peek at the deleted switch
2579 main.log.warn( str( device ) )
2580 result = main.FALSE
2581 if device and device[ 'available' ]:
2582 result = main.TRUE
2583 utilities.assert_equals( expect=main.TRUE, actual=result,
2584 onpass="add switch successful",
2585 onfail="Failed to add switch?" )
2586
2587 def CASE13( self, main ):
2588 """
2589 Clean up
2590 """
2591 import os
2592 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002593 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002594 assert main, "main not defined"
2595 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002596 assert main.CLIs, "main.CLIs not defined"
2597 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002598
2599 # printing colors to terminal
2600 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2601 'blue': '\033[94m', 'green': '\033[92m',
2602 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2603 main.case( "Test Cleanup" )
2604 main.step( "Killing tcpdumps" )
2605 main.Mininet2.stopTcpdump()
2606
2607 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002608 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002609 main.step( "Copying MN pcap and ONOS log files to test station" )
2610 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2611 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002612 # NOTE: MN Pcap file is being saved to logdir.
2613 # We scp this file as MN and TestON aren't necessarily the same vm
2614
2615 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002616 # TODO: Load these from params
2617 # NOTE: must end in /
2618 logFolder = "/opt/onos/log/"
2619 logFiles = [ "karaf.log", "karaf.log.1" ]
2620 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002621 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002622 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002623 dstName = main.logdir + "/" + node.name + "-" + f
2624 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2625 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002626 # std*.log's
2627 # NOTE: must end in /
2628 logFolder = "/opt/onos/var/"
2629 logFiles = [ "stderr.log", "stdout.log" ]
2630 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002631 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002632 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002633 dstName = main.logdir + "/" + node.name + "-" + f
2634 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2635 logFolder + f, dstName )
2636 else:
2637 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002638
2639 main.step( "Stopping Mininet" )
2640 mnResult = main.Mininet1.stopNet()
2641 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2642 onpass="Mininet stopped",
2643 onfail="MN cleanup NOT successful" )
2644
2645 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002646 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002647 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2648 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002649
2650 try:
2651 timerLog = open( main.logdir + "/Timers.csv", 'w')
2652 # Overwrite with empty line and close
2653 labels = "Gossip Intents"
2654 data = str( gossipTime )
2655 timerLog.write( labels + "\n" + data )
2656 timerLog.close()
2657 except NameError, e:
2658 main.log.exception(e)
2659
2660 def CASE14( self, main ):
2661 """
2662 start election app on all onos nodes
2663 """
Jon Halle1a3b752015-07-22 13:02:46 -07002664 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002665 assert main, "main not defined"
2666 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002667 assert main.CLIs, "main.CLIs not defined"
2668 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002669
2670 main.case("Start Leadership Election app")
2671 main.step( "Install leadership election app" )
2672 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2673 utilities.assert_equals(
2674 expect=main.TRUE,
2675 actual=appResult,
2676 onpass="Election app installed",
2677 onfail="Something went wrong with installing Leadership election" )
2678
2679 main.step( "Run for election on each node" )
2680 leaderResult = main.TRUE
2681 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002682 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002683 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002684 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002685 leader = cli.electionTestLeader()
2686 if leader is None or leader == main.FALSE:
2687 main.log.error( cli.name + ": Leader for the election app " +
2688 "should be an ONOS node, instead got '" +
2689 str( leader ) + "'" )
2690 leaderResult = main.FALSE
2691 leaders.append( leader )
2692 utilities.assert_equals(
2693 expect=main.TRUE,
2694 actual=leaderResult,
2695 onpass="Successfully ran for leadership",
2696 onfail="Failed to run for leadership" )
2697
2698 main.step( "Check that each node shows the same leader" )
2699 sameLeader = main.TRUE
2700 if len( set( leaders ) ) != 1:
2701 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002702 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002703 str( leaders ) )
2704 utilities.assert_equals(
2705 expect=main.TRUE,
2706 actual=sameLeader,
2707 onpass="Leadership is consistent for the election topic",
2708 onfail="Nodes have different leaders" )
2709
2710 def CASE15( self, main ):
2711 """
2712 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002713 15.1 Run election on each node
2714 15.2 Check that each node has the same leaders and candidates
2715 15.3 Find current leader and withdraw
2716 15.4 Check that a new node was elected leader
2717 15.5 Check that that new leader was the candidate of old leader
2718 15.6 Run for election on old leader
2719 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2720 15.8 Make sure that the old leader was added to the candidate list
2721
2722 old and new variable prefixes refer to data from before vs after
2723 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002724 """
2725 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002726 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002727 assert main, "main not defined"
2728 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002729 assert main.CLIs, "main.CLIs not defined"
2730 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002731
acsmars3a72bde2015-09-02 14:16:22 -07002732 description = "Check that Leadership Election App is still functional"
Jon Hall5cf14d52015-07-16 12:15:19 -07002733 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07002734 # NOTE: Need to re-run since being a canidate is not persistant
2735 # TODO: add check for "Command not found:" in the driver, this
2736 # means the election test app isn't loaded
Jon Hall5cf14d52015-07-16 12:15:19 -07002737
acsmars71adceb2015-08-31 15:09:26 -07002738 oldLeaders = [] # leaders by node before withdrawl from candidates
2739 newLeaders = [] # leaders by node after withdrawl from candidates
2740 oldAllCandidates = [] # list of lists of each nodes' candidates before
2741 newAllCandidates = [] # list of lists of each nodes' candidates after
2742 oldCandidates = [] # list of candidates from node 0 before withdrawl
2743 newCandidates = [] # list of candidates from node 0 after withdrawl
2744 oldLeader = '' # the old leader from oldLeaders, None if not same
2745 newLeader = '' # the new leaders fron newLoeaders, None if not same
2746 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2747 expectNoLeader = False # True when there is only one leader
2748 if main.numCtrls == 1:
2749 expectNoLeader = True
2750
2751 main.step( "Run for election on each node" )
2752 electionResult = main.TRUE
2753
2754 for cli in main.CLIs: # run test election on each node
2755 if cli.electionTestRun() == main.FALSE:
2756 electionResult = main.FALSE
2757
Jon Hall5cf14d52015-07-16 12:15:19 -07002758 utilities.assert_equals(
2759 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002760 actual=electionResult,
2761 onpass="All nodes successfully ran for leadership",
2762 onfail="At least one node failed to run for leadership" )
2763
acsmars3a72bde2015-09-02 14:16:22 -07002764 if electionResult == main.FALSE:
2765 main.log.error(
2766 "Skipping Test Case because Election Test isn't loaded" )
2767 main.skipCase()
2768
acsmars71adceb2015-08-31 15:09:26 -07002769 main.step( "Check that each node shows the same leader and candidates" )
2770 sameResult = main.TRUE
2771 failMessage = "Nodes have different leaders"
2772 for cli in main.CLIs:
2773 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2774 oldAllCandidates.append( node )
2775 oldLeaders.append( node[ 0 ] )
2776 oldCandidates = oldAllCandidates[ 0 ]
2777
2778 # Check that each node has the same leader. Defines oldLeader
2779 if len( set( oldLeaders ) ) != 1:
2780 sameResult = main.FALSE
2781 main.log.error( "More than one leader present:" + str( oldLeaders ) )
2782 oldLeader = None
2783 else:
2784 oldLeader = oldLeaders[ 0 ]
2785
2786 # Check that each node's candidate list is the same
acsmars29233db2015-11-04 11:15:00 -08002787 candidateDiscrepancy = False # Boolean of candidate mismatches
acsmars71adceb2015-08-31 15:09:26 -07002788 for candidates in oldAllCandidates:
2789 if set( candidates ) != set( oldCandidates ):
2790 sameResult = main.FALSE
acsmars29233db2015-11-04 11:15:00 -08002791 candidateDiscrepancy = True
2792
2793 if candidateDiscrepancy:
2794 failMessage += " and candidates"
acsmars71adceb2015-08-31 15:09:26 -07002795
2796 utilities.assert_equals(
2797 expect=main.TRUE,
2798 actual=sameResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002799 onpass="Leadership is consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002800 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002801
2802 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002803 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002804 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002805 if oldLeader is None:
2806 main.log.error( "Leadership isn't consistent." )
2807 withdrawResult = main.FALSE
2808 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002809 for i in range( len( main.CLIs ) ):
acsmars71adceb2015-08-31 15:09:26 -07002810 if oldLeader == main.nodes[ i ].ip_address:
2811 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002812 break
2813 else: # FOR/ELSE statement
2814 main.log.error( "Leader election, could not find current leader" )
2815 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002816 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002817 utilities.assert_equals(
2818 expect=main.TRUE,
2819 actual=withdrawResult,
2820 onpass="Node was withdrawn from election",
2821 onfail="Node was not withdrawn from election" )
2822
acsmars71adceb2015-08-31 15:09:26 -07002823 main.step( "Check that a new node was elected leader" )
2824
Jon Hall5cf14d52015-07-16 12:15:19 -07002825 # FIXME: use threads
acsmars71adceb2015-08-31 15:09:26 -07002826 newLeaderResult = main.TRUE
2827 failMessage = "Nodes have different leaders"
2828
2829 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002830 for cli in main.CLIs:
acsmars71adceb2015-08-31 15:09:26 -07002831 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2832 # elections might no have finished yet
2833 if node[ 0 ] == 'none' and not expectNoLeader:
2834 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2835 "sure elections are complete." )
2836 time.sleep(5)
2837 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2838 # election still isn't done or there is a problem
2839 if node[ 0 ] == 'none':
2840 main.log.error( "No leader was elected on at least 1 node" )
2841 newLeaderResult = main.FALSE
2842 newAllCandidates.append( node )
2843 newLeaders.append( node[ 0 ] )
2844 newCandidates = newAllCandidates[ 0 ]
2845
2846 # Check that each node has the same leader. Defines newLeader
2847 if len( set( newLeaders ) ) != 1:
2848 newLeaderResult = main.FALSE
2849 main.log.error( "Nodes have different leaders: " +
2850 str( newLeaders ) )
2851 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002852 else:
acsmars71adceb2015-08-31 15:09:26 -07002853 newLeader = newLeaders[ 0 ]
2854
2855 # Check that each node's candidate list is the same
2856 for candidates in newAllCandidates:
2857 if set( candidates ) != set( newCandidates ):
2858 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002859 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002860
2861 # Check that the new leader is not the older leader, which was withdrawn
2862 if newLeader == oldLeader:
2863 newLeaderResult = main.FALSE
2864 main.log.error( "All nodes still see old leader: " + oldLeader +
2865 " as the current leader" )
2866
Jon Hall5cf14d52015-07-16 12:15:19 -07002867 utilities.assert_equals(
2868 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002869 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002870 onpass="Leadership election passed",
2871 onfail="Something went wrong with Leadership election" )
2872
acsmars71adceb2015-08-31 15:09:26 -07002873 main.step( "Check that that new leader was the candidate of old leader")
2874 # candidates[ 2 ] should be come the top candidate after withdrawl
2875 correctCandidateResult = main.TRUE
2876 if expectNoLeader:
2877 if newLeader == 'none':
2878 main.log.info( "No leader expected. None found. Pass" )
2879 correctCandidateResult = main.TRUE
2880 else:
2881 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2882 correctCandidateResult = main.FALSE
2883 elif newLeader != oldCandidates[ 2 ]:
2884 correctCandidateResult = main.FALSE
2885 main.log.error( "Candidate " + newLeader + " was elected. " +
2886 oldCandidates[ 2 ] + " should have had priority." )
2887
2888 utilities.assert_equals(
2889 expect=main.TRUE,
2890 actual=correctCandidateResult,
2891 onpass="Correct Candidate Elected",
2892 onfail="Incorrect Candidate Elected" )
2893
Jon Hall5cf14d52015-07-16 12:15:19 -07002894 main.step( "Run for election on old leader( just so everyone " +
2895 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002896 if oldLeaderCLI is not None:
2897 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002898 else:
acsmars71adceb2015-08-31 15:09:26 -07002899 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002900 runResult = main.FALSE
2901 utilities.assert_equals(
2902 expect=main.TRUE,
2903 actual=runResult,
2904 onpass="App re-ran for election",
2905 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002906 main.step(
2907 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002908 # verify leader didn't just change
acsmars71adceb2015-08-31 15:09:26 -07002909 positionResult = main.TRUE
2910 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
2911
2912 # Reset and reuse the new candidate and leaders lists
2913 newAllCandidates = []
2914 newCandidates = []
2915 newLeaders = []
2916 for cli in main.CLIs:
2917 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2918 if oldLeader not in node: # election might no have finished yet
2919 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2920 "be sure elections are complete" )
2921 time.sleep(5)
2922 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2923 if oldLeader not in node: # election still isn't done, errors
2924 main.log.error(
2925 "Old leader was not elected on at least one node" )
2926 positionResult = main.FALSE
2927 newAllCandidates.append( node )
2928 newLeaders.append( node[ 0 ] )
2929 newCandidates = newAllCandidates[ 0 ]
2930
2931 # Check that each node has the same leader. Defines newLeader
2932 if len( set( newLeaders ) ) != 1:
2933 positionResult = main.FALSE
2934 main.log.error( "Nodes have different leaders: " +
2935 str( newLeaders ) )
2936 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002937 else:
acsmars71adceb2015-08-31 15:09:26 -07002938 newLeader = newLeaders[ 0 ]
2939
2940 # Check that each node's candidate list is the same
2941 for candidates in newAllCandidates:
2942 if set( candidates ) != set( newCandidates ):
2943 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002944 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002945
2946 # Check that the re-elected node is last on the candidate List
2947 if oldLeader != newCandidates[ -1 ]:
2948 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
2949 str( newCandidates ) )
2950 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002951
2952 utilities.assert_equals(
2953 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002954 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002955 onpass="Old leader successfully re-ran for election",
2956 onfail="Something went wrong with Leadership election after " +
2957 "the old leader re-ran for election" )
2958
2959 def CASE16( self, main ):
2960 """
2961 Install Distributed Primitives app
2962 """
2963 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002964 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002965 assert main, "main not defined"
2966 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002967 assert main.CLIs, "main.CLIs not defined"
2968 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002969
2970 # Variables for the distributed primitives tests
2971 global pCounterName
2972 global iCounterName
2973 global pCounterValue
2974 global iCounterValue
2975 global onosSet
2976 global onosSetName
2977 pCounterName = "TestON-Partitions"
2978 iCounterName = "TestON-inMemory"
2979 pCounterValue = 0
2980 iCounterValue = 0
2981 onosSet = set([])
2982 onosSetName = "TestON-set"
2983
2984 description = "Install Primitives app"
2985 main.case( description )
2986 main.step( "Install Primitives app" )
2987 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002988 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002989 utilities.assert_equals( expect=main.TRUE,
2990 actual=appResults,
2991 onpass="Primitives app activated",
2992 onfail="Primitives app not activated" )
2993 time.sleep( 5 ) # To allow all nodes to activate
2994
2995 def CASE17( self, main ):
2996 """
2997 Check for basic functionality with distributed primitives
2998 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002999 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003000 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003001 assert main, "main not defined"
3002 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003003 assert main.CLIs, "main.CLIs not defined"
3004 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003005 assert pCounterName, "pCounterName not defined"
3006 assert iCounterName, "iCounterName not defined"
3007 assert onosSetName, "onosSetName not defined"
3008 # NOTE: assert fails if value is 0/None/Empty/False
3009 try:
3010 pCounterValue
3011 except NameError:
3012 main.log.error( "pCounterValue not defined, setting to 0" )
3013 pCounterValue = 0
3014 try:
3015 iCounterValue
3016 except NameError:
3017 main.log.error( "iCounterValue not defined, setting to 0" )
3018 iCounterValue = 0
3019 try:
3020 onosSet
3021 except NameError:
3022 main.log.error( "onosSet not defined, setting to empty Set" )
3023 onosSet = set([])
3024 # Variables for the distributed primitives tests. These are local only
3025 addValue = "a"
3026 addAllValue = "a b c d e f"
3027 retainValue = "c d e f"
3028
3029 description = "Check for basic functionality with distributed " +\
3030 "primitives"
3031 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003032 main.caseExplanation = "Test the methods of the distributed " +\
3033 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003034 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003035 # Partitioned counters
3036 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003037 pCounters = []
3038 threads = []
3039 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003040 for i in range( main.numCtrls ):
3041 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3042 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003043 args=[ pCounterName ] )
3044 pCounterValue += 1
3045 addedPValues.append( pCounterValue )
3046 threads.append( t )
3047 t.start()
3048
3049 for t in threads:
3050 t.join()
3051 pCounters.append( t.result )
3052 # Check that counter incremented numController times
3053 pCounterResults = True
3054 for i in addedPValues:
3055 tmpResult = i in pCounters
3056 pCounterResults = pCounterResults and tmpResult
3057 if not tmpResult:
3058 main.log.error( str( i ) + " is not in partitioned "
3059 "counter incremented results" )
3060 utilities.assert_equals( expect=True,
3061 actual=pCounterResults,
3062 onpass="Default counter incremented",
3063 onfail="Error incrementing default" +
3064 " counter" )
3065
Jon Halle1a3b752015-07-22 13:02:46 -07003066 main.step( "Get then Increment a default counter on each node" )
3067 pCounters = []
3068 threads = []
3069 addedPValues = []
3070 for i in range( main.numCtrls ):
3071 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3072 name="counterGetAndAdd-" + str( i ),
3073 args=[ pCounterName ] )
3074 addedPValues.append( pCounterValue )
3075 pCounterValue += 1
3076 threads.append( t )
3077 t.start()
3078
3079 for t in threads:
3080 t.join()
3081 pCounters.append( t.result )
3082 # Check that counter incremented numController times
3083 pCounterResults = True
3084 for i in addedPValues:
3085 tmpResult = i in pCounters
3086 pCounterResults = pCounterResults and tmpResult
3087 if not tmpResult:
3088 main.log.error( str( i ) + " is not in partitioned "
3089 "counter incremented results" )
3090 utilities.assert_equals( expect=True,
3091 actual=pCounterResults,
3092 onpass="Default counter incremented",
3093 onfail="Error incrementing default" +
3094 " counter" )
3095
3096 main.step( "Counters we added have the correct values" )
3097 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3098 utilities.assert_equals( expect=main.TRUE,
3099 actual=incrementCheck,
3100 onpass="Added counters are correct",
3101 onfail="Added counters are incorrect" )
3102
3103 main.step( "Add -8 to then get a default counter on each node" )
3104 pCounters = []
3105 threads = []
3106 addedPValues = []
3107 for i in range( main.numCtrls ):
3108 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3109 name="counterIncrement-" + str( i ),
3110 args=[ pCounterName ],
3111 kwargs={ "delta": -8 } )
3112 pCounterValue += -8
3113 addedPValues.append( pCounterValue )
3114 threads.append( t )
3115 t.start()
3116
3117 for t in threads:
3118 t.join()
3119 pCounters.append( t.result )
3120 # Check that counter incremented numController times
3121 pCounterResults = True
3122 for i in addedPValues:
3123 tmpResult = i in pCounters
3124 pCounterResults = pCounterResults and tmpResult
3125 if not tmpResult:
3126 main.log.error( str( i ) + " is not in partitioned "
3127 "counter incremented results" )
3128 utilities.assert_equals( expect=True,
3129 actual=pCounterResults,
3130 onpass="Default counter incremented",
3131 onfail="Error incrementing default" +
3132 " counter" )
3133
3134 main.step( "Add 5 to then get a default counter on each node" )
3135 pCounters = []
3136 threads = []
3137 addedPValues = []
3138 for i in range( main.numCtrls ):
3139 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3140 name="counterIncrement-" + str( i ),
3141 args=[ pCounterName ],
3142 kwargs={ "delta": 5 } )
3143 pCounterValue += 5
3144 addedPValues.append( pCounterValue )
3145 threads.append( t )
3146 t.start()
3147
3148 for t in threads:
3149 t.join()
3150 pCounters.append( t.result )
3151 # Check that counter incremented numController times
3152 pCounterResults = True
3153 for i in addedPValues:
3154 tmpResult = i in pCounters
3155 pCounterResults = pCounterResults and tmpResult
3156 if not tmpResult:
3157 main.log.error( str( i ) + " is not in partitioned "
3158 "counter incremented results" )
3159 utilities.assert_equals( expect=True,
3160 actual=pCounterResults,
3161 onpass="Default counter incremented",
3162 onfail="Error incrementing default" +
3163 " counter" )
3164
3165 main.step( "Get then add 5 to a default counter on each node" )
3166 pCounters = []
3167 threads = []
3168 addedPValues = []
3169 for i in range( main.numCtrls ):
3170 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3171 name="counterIncrement-" + str( i ),
3172 args=[ pCounterName ],
3173 kwargs={ "delta": 5 } )
3174 addedPValues.append( pCounterValue )
3175 pCounterValue += 5
3176 threads.append( t )
3177 t.start()
3178
3179 for t in threads:
3180 t.join()
3181 pCounters.append( t.result )
3182 # Check that counter incremented numController times
3183 pCounterResults = True
3184 for i in addedPValues:
3185 tmpResult = i in pCounters
3186 pCounterResults = pCounterResults and tmpResult
3187 if not tmpResult:
3188 main.log.error( str( i ) + " is not in partitioned "
3189 "counter incremented results" )
3190 utilities.assert_equals( expect=True,
3191 actual=pCounterResults,
3192 onpass="Default counter incremented",
3193 onfail="Error incrementing default" +
3194 " counter" )
3195
3196 main.step( "Counters we added have the correct values" )
3197 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3198 utilities.assert_equals( expect=main.TRUE,
3199 actual=incrementCheck,
3200 onpass="Added counters are correct",
3201 onfail="Added counters are incorrect" )
3202
3203 # In-Memory counters
3204 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003205 iCounters = []
3206 addedIValues = []
3207 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003208 for i in range( main.numCtrls ):
3209 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003210 name="icounterIncrement-" + str( i ),
3211 args=[ iCounterName ],
3212 kwargs={ "inMemory": True } )
3213 iCounterValue += 1
3214 addedIValues.append( iCounterValue )
3215 threads.append( t )
3216 t.start()
3217
3218 for t in threads:
3219 t.join()
3220 iCounters.append( t.result )
3221 # Check that counter incremented numController times
3222 iCounterResults = True
3223 for i in addedIValues:
3224 tmpResult = i in iCounters
3225 iCounterResults = iCounterResults and tmpResult
3226 if not tmpResult:
3227 main.log.error( str( i ) + " is not in the in-memory "
3228 "counter incremented results" )
3229 utilities.assert_equals( expect=True,
3230 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003231 onpass="In-memory counter incremented",
3232 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003233 " counter" )
3234
Jon Halle1a3b752015-07-22 13:02:46 -07003235 main.step( "Get then Increment a in-memory counter on each node" )
3236 iCounters = []
3237 threads = []
3238 addedIValues = []
3239 for i in range( main.numCtrls ):
3240 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3241 name="counterGetAndAdd-" + str( i ),
3242 args=[ iCounterName ],
3243 kwargs={ "inMemory": True } )
3244 addedIValues.append( iCounterValue )
3245 iCounterValue += 1
3246 threads.append( t )
3247 t.start()
3248
3249 for t in threads:
3250 t.join()
3251 iCounters.append( t.result )
3252 # Check that counter incremented numController times
3253 iCounterResults = True
3254 for i in addedIValues:
3255 tmpResult = i in iCounters
3256 iCounterResults = iCounterResults and tmpResult
3257 if not tmpResult:
3258 main.log.error( str( i ) + " is not in in-memory "
3259 "counter incremented results" )
3260 utilities.assert_equals( expect=True,
3261 actual=iCounterResults,
3262 onpass="In-memory counter incremented",
3263 onfail="Error incrementing in-memory" +
3264 " counter" )
3265
3266 main.step( "Counters we added have the correct values" )
3267 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3268 utilities.assert_equals( expect=main.TRUE,
3269 actual=incrementCheck,
3270 onpass="Added counters are correct",
3271 onfail="Added counters are incorrect" )
3272
3273 main.step( "Add -8 to then get a in-memory counter on each node" )
3274 iCounters = []
3275 threads = []
3276 addedIValues = []
3277 for i in range( main.numCtrls ):
3278 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3279 name="counterIncrement-" + str( i ),
3280 args=[ iCounterName ],
3281 kwargs={ "delta": -8, "inMemory": True } )
3282 iCounterValue += -8
3283 addedIValues.append( iCounterValue )
3284 threads.append( t )
3285 t.start()
3286
3287 for t in threads:
3288 t.join()
3289 iCounters.append( t.result )
3290 # Check that counter incremented numController times
3291 iCounterResults = True
3292 for i in addedIValues:
3293 tmpResult = i in iCounters
3294 iCounterResults = iCounterResults and tmpResult
3295 if not tmpResult:
3296 main.log.error( str( i ) + " is not in in-memory "
3297 "counter incremented results" )
3298 utilities.assert_equals( expect=True,
3299 actual=pCounterResults,
3300 onpass="In-memory counter incremented",
3301 onfail="Error incrementing in-memory" +
3302 " counter" )
3303
3304 main.step( "Add 5 to then get a in-memory counter on each node" )
3305 iCounters = []
3306 threads = []
3307 addedIValues = []
3308 for i in range( main.numCtrls ):
3309 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3310 name="counterIncrement-" + str( i ),
3311 args=[ iCounterName ],
3312 kwargs={ "delta": 5, "inMemory": True } )
3313 iCounterValue += 5
3314 addedIValues.append( iCounterValue )
3315 threads.append( t )
3316 t.start()
3317
3318 for t in threads:
3319 t.join()
3320 iCounters.append( t.result )
3321 # Check that counter incremented numController times
3322 iCounterResults = True
3323 for i in addedIValues:
3324 tmpResult = i in iCounters
3325 iCounterResults = iCounterResults and tmpResult
3326 if not tmpResult:
3327 main.log.error( str( i ) + " is not in in-memory "
3328 "counter incremented results" )
3329 utilities.assert_equals( expect=True,
3330 actual=pCounterResults,
3331 onpass="In-memory counter incremented",
3332 onfail="Error incrementing in-memory" +
3333 " counter" )
3334
3335 main.step( "Get then add 5 to a in-memory counter on each node" )
3336 iCounters = []
3337 threads = []
3338 addedIValues = []
3339 for i in range( main.numCtrls ):
3340 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3341 name="counterIncrement-" + str( i ),
3342 args=[ iCounterName ],
3343 kwargs={ "delta": 5, "inMemory": True } )
3344 addedIValues.append( iCounterValue )
3345 iCounterValue += 5
3346 threads.append( t )
3347 t.start()
3348
3349 for t in threads:
3350 t.join()
3351 iCounters.append( t.result )
3352 # Check that counter incremented numController times
3353 iCounterResults = True
3354 for i in addedIValues:
3355 tmpResult = i in iCounters
3356 iCounterResults = iCounterResults and tmpResult
3357 if not tmpResult:
3358 main.log.error( str( i ) + " is not in in-memory "
3359 "counter incremented results" )
3360 utilities.assert_equals( expect=True,
3361 actual=iCounterResults,
3362 onpass="In-memory counter incremented",
3363 onfail="Error incrementing in-memory" +
3364 " counter" )
3365
3366 main.step( "Counters we added have the correct values" )
3367 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3368 utilities.assert_equals( expect=main.TRUE,
3369 actual=incrementCheck,
3370 onpass="Added counters are correct",
3371 onfail="Added counters are incorrect" )
3372
Jon Hall5cf14d52015-07-16 12:15:19 -07003373 main.step( "Check counters are consistant across nodes" )
Jon Hall57b50432015-10-22 10:20:10 -07003374 onosCounters, consistentCounterResults = main.Counters.consistentCheck()
Jon Hall5cf14d52015-07-16 12:15:19 -07003375 utilities.assert_equals( expect=main.TRUE,
3376 actual=consistentCounterResults,
3377 onpass="ONOS counters are consistent " +
3378 "across nodes",
3379 onfail="ONOS Counters are inconsistent " +
3380 "across nodes" )
3381
3382 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003383 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3384 incrementCheck = incrementCheck and \
3385 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003386 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003387 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003388 onpass="Added counters are correct",
3389 onfail="Added counters are incorrect" )
3390 # DISTRIBUTED SETS
3391 main.step( "Distributed Set get" )
3392 size = len( onosSet )
3393 getResponses = []
3394 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003395 for i in range( main.numCtrls ):
3396 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003397 name="setTestGet-" + str( i ),
3398 args=[ onosSetName ] )
3399 threads.append( t )
3400 t.start()
3401 for t in threads:
3402 t.join()
3403 getResponses.append( t.result )
3404
3405 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003406 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003407 if isinstance( getResponses[ i ], list):
3408 current = set( getResponses[ i ] )
3409 if len( current ) == len( getResponses[ i ] ):
3410 # no repeats
3411 if onosSet != current:
3412 main.log.error( "ONOS" + str( i + 1 ) +
3413 " has incorrect view" +
3414 " of set " + onosSetName + ":\n" +
3415 str( getResponses[ i ] ) )
3416 main.log.debug( "Expected: " + str( onosSet ) )
3417 main.log.debug( "Actual: " + str( current ) )
3418 getResults = main.FALSE
3419 else:
3420 # error, set is not a set
3421 main.log.error( "ONOS" + str( i + 1 ) +
3422 " has repeat elements in" +
3423 " set " + onosSetName + ":\n" +
3424 str( getResponses[ i ] ) )
3425 getResults = main.FALSE
3426 elif getResponses[ i ] == main.ERROR:
3427 getResults = main.FALSE
3428 utilities.assert_equals( expect=main.TRUE,
3429 actual=getResults,
3430 onpass="Set elements are correct",
3431 onfail="Set elements are incorrect" )
3432
3433 main.step( "Distributed Set size" )
3434 sizeResponses = []
3435 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003436 for i in range( main.numCtrls ):
3437 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003438 name="setTestSize-" + str( i ),
3439 args=[ onosSetName ] )
3440 threads.append( t )
3441 t.start()
3442 for t in threads:
3443 t.join()
3444 sizeResponses.append( t.result )
3445
3446 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003447 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003448 if size != sizeResponses[ i ]:
3449 sizeResults = main.FALSE
3450 main.log.error( "ONOS" + str( i + 1 ) +
3451 " expected a size of " + str( size ) +
3452 " for set " + onosSetName +
3453 " but got " + str( sizeResponses[ i ] ) )
3454 utilities.assert_equals( expect=main.TRUE,
3455 actual=sizeResults,
3456 onpass="Set sizes are correct",
3457 onfail="Set sizes are incorrect" )
3458
3459 main.step( "Distributed Set add()" )
3460 onosSet.add( addValue )
3461 addResponses = []
3462 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003463 for i in range( main.numCtrls ):
3464 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003465 name="setTestAdd-" + str( i ),
3466 args=[ onosSetName, addValue ] )
3467 threads.append( t )
3468 t.start()
3469 for t in threads:
3470 t.join()
3471 addResponses.append( t.result )
3472
3473 # main.TRUE = successfully changed the set
3474 # main.FALSE = action resulted in no change in set
3475 # main.ERROR - Some error in executing the function
3476 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003477 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003478 if addResponses[ i ] == main.TRUE:
3479 # All is well
3480 pass
3481 elif addResponses[ i ] == main.FALSE:
3482 # Already in set, probably fine
3483 pass
3484 elif addResponses[ i ] == main.ERROR:
3485 # Error in execution
3486 addResults = main.FALSE
3487 else:
3488 # unexpected result
3489 addResults = main.FALSE
3490 if addResults != main.TRUE:
3491 main.log.error( "Error executing set add" )
3492
3493 # Check if set is still correct
3494 size = len( onosSet )
3495 getResponses = []
3496 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003497 for i in range( main.numCtrls ):
3498 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003499 name="setTestGet-" + str( i ),
3500 args=[ onosSetName ] )
3501 threads.append( t )
3502 t.start()
3503 for t in threads:
3504 t.join()
3505 getResponses.append( t.result )
3506 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003507 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003508 if isinstance( getResponses[ i ], list):
3509 current = set( getResponses[ i ] )
3510 if len( current ) == len( getResponses[ i ] ):
3511 # no repeats
3512 if onosSet != current:
3513 main.log.error( "ONOS" + str( i + 1 ) +
3514 " has incorrect view" +
3515 " of set " + onosSetName + ":\n" +
3516 str( getResponses[ i ] ) )
3517 main.log.debug( "Expected: " + str( onosSet ) )
3518 main.log.debug( "Actual: " + str( current ) )
3519 getResults = main.FALSE
3520 else:
3521 # error, set is not a set
3522 main.log.error( "ONOS" + str( i + 1 ) +
3523 " has repeat elements in" +
3524 " set " + onosSetName + ":\n" +
3525 str( getResponses[ i ] ) )
3526 getResults = main.FALSE
3527 elif getResponses[ i ] == main.ERROR:
3528 getResults = main.FALSE
3529 sizeResponses = []
3530 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003531 for i in range( main.numCtrls ):
3532 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003533 name="setTestSize-" + str( i ),
3534 args=[ onosSetName ] )
3535 threads.append( t )
3536 t.start()
3537 for t in threads:
3538 t.join()
3539 sizeResponses.append( t.result )
3540 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003541 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003542 if size != sizeResponses[ i ]:
3543 sizeResults = main.FALSE
3544 main.log.error( "ONOS" + str( i + 1 ) +
3545 " expected a size of " + str( size ) +
3546 " for set " + onosSetName +
3547 " but got " + str( sizeResponses[ i ] ) )
3548 addResults = addResults and getResults and sizeResults
3549 utilities.assert_equals( expect=main.TRUE,
3550 actual=addResults,
3551 onpass="Set add correct",
3552 onfail="Set add was incorrect" )
3553
3554 main.step( "Distributed Set addAll()" )
3555 onosSet.update( addAllValue.split() )
3556 addResponses = []
3557 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003558 for i in range( main.numCtrls ):
3559 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003560 name="setTestAddAll-" + str( i ),
3561 args=[ onosSetName, addAllValue ] )
3562 threads.append( t )
3563 t.start()
3564 for t in threads:
3565 t.join()
3566 addResponses.append( t.result )
3567
3568 # main.TRUE = successfully changed the set
3569 # main.FALSE = action resulted in no change in set
3570 # main.ERROR - Some error in executing the function
3571 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003572 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003573 if addResponses[ i ] == main.TRUE:
3574 # All is well
3575 pass
3576 elif addResponses[ i ] == main.FALSE:
3577 # Already in set, probably fine
3578 pass
3579 elif addResponses[ i ] == main.ERROR:
3580 # Error in execution
3581 addAllResults = main.FALSE
3582 else:
3583 # unexpected result
3584 addAllResults = main.FALSE
3585 if addAllResults != main.TRUE:
3586 main.log.error( "Error executing set addAll" )
3587
3588 # Check if set is still correct
3589 size = len( onosSet )
3590 getResponses = []
3591 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003592 for i in range( main.numCtrls ):
3593 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003594 name="setTestGet-" + str( i ),
3595 args=[ onosSetName ] )
3596 threads.append( t )
3597 t.start()
3598 for t in threads:
3599 t.join()
3600 getResponses.append( t.result )
3601 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003602 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003603 if isinstance( getResponses[ i ], list):
3604 current = set( getResponses[ i ] )
3605 if len( current ) == len( getResponses[ i ] ):
3606 # no repeats
3607 if onosSet != current:
3608 main.log.error( "ONOS" + str( i + 1 ) +
3609 " has incorrect view" +
3610 " of set " + onosSetName + ":\n" +
3611 str( getResponses[ i ] ) )
3612 main.log.debug( "Expected: " + str( onosSet ) )
3613 main.log.debug( "Actual: " + str( current ) )
3614 getResults = main.FALSE
3615 else:
3616 # error, set is not a set
3617 main.log.error( "ONOS" + str( i + 1 ) +
3618 " has repeat elements in" +
3619 " set " + onosSetName + ":\n" +
3620 str( getResponses[ i ] ) )
3621 getResults = main.FALSE
3622 elif getResponses[ i ] == main.ERROR:
3623 getResults = main.FALSE
3624 sizeResponses = []
3625 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003626 for i in range( main.numCtrls ):
3627 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003628 name="setTestSize-" + str( i ),
3629 args=[ onosSetName ] )
3630 threads.append( t )
3631 t.start()
3632 for t in threads:
3633 t.join()
3634 sizeResponses.append( t.result )
3635 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003636 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003637 if size != sizeResponses[ i ]:
3638 sizeResults = main.FALSE
3639 main.log.error( "ONOS" + str( i + 1 ) +
3640 " expected a size of " + str( size ) +
3641 " for set " + onosSetName +
3642 " but got " + str( sizeResponses[ i ] ) )
3643 addAllResults = addAllResults and getResults and sizeResults
3644 utilities.assert_equals( expect=main.TRUE,
3645 actual=addAllResults,
3646 onpass="Set addAll correct",
3647 onfail="Set addAll was incorrect" )
3648
3649 main.step( "Distributed Set contains()" )
3650 containsResponses = []
3651 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003652 for i in range( main.numCtrls ):
3653 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003654 name="setContains-" + str( i ),
3655 args=[ onosSetName ],
3656 kwargs={ "values": addValue } )
3657 threads.append( t )
3658 t.start()
3659 for t in threads:
3660 t.join()
3661 # NOTE: This is the tuple
3662 containsResponses.append( t.result )
3663
3664 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003665 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003666 if containsResponses[ i ] == main.ERROR:
3667 containsResults = main.FALSE
3668 else:
3669 containsResults = containsResults and\
3670 containsResponses[ i ][ 1 ]
3671 utilities.assert_equals( expect=main.TRUE,
3672 actual=containsResults,
3673 onpass="Set contains is functional",
3674 onfail="Set contains failed" )
3675
3676 main.step( "Distributed Set containsAll()" )
3677 containsAllResponses = []
3678 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003679 for i in range( main.numCtrls ):
3680 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003681 name="setContainsAll-" + str( i ),
3682 args=[ onosSetName ],
3683 kwargs={ "values": addAllValue } )
3684 threads.append( t )
3685 t.start()
3686 for t in threads:
3687 t.join()
3688 # NOTE: This is the tuple
3689 containsAllResponses.append( t.result )
3690
3691 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003692 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003693 if containsResponses[ i ] == main.ERROR:
3694 containsResults = main.FALSE
3695 else:
3696 containsResults = containsResults and\
3697 containsResponses[ i ][ 1 ]
3698 utilities.assert_equals( expect=main.TRUE,
3699 actual=containsAllResults,
3700 onpass="Set containsAll is functional",
3701 onfail="Set containsAll failed" )
3702
3703 main.step( "Distributed Set remove()" )
3704 onosSet.remove( addValue )
3705 removeResponses = []
3706 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003707 for i in range( main.numCtrls ):
3708 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003709 name="setTestRemove-" + str( i ),
3710 args=[ onosSetName, addValue ] )
3711 threads.append( t )
3712 t.start()
3713 for t in threads:
3714 t.join()
3715 removeResponses.append( t.result )
3716
3717 # main.TRUE = successfully changed the set
3718 # main.FALSE = action resulted in no change in set
3719 # main.ERROR - Some error in executing the function
3720 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003721 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003722 if removeResponses[ i ] == main.TRUE:
3723 # All is well
3724 pass
3725 elif removeResponses[ i ] == main.FALSE:
3726 # not in set, probably fine
3727 pass
3728 elif removeResponses[ i ] == main.ERROR:
3729 # Error in execution
3730 removeResults = main.FALSE
3731 else:
3732 # unexpected result
3733 removeResults = main.FALSE
3734 if removeResults != main.TRUE:
3735 main.log.error( "Error executing set remove" )
3736
3737 # Check if set is still correct
3738 size = len( onosSet )
3739 getResponses = []
3740 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003741 for i in range( main.numCtrls ):
3742 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003743 name="setTestGet-" + str( i ),
3744 args=[ onosSetName ] )
3745 threads.append( t )
3746 t.start()
3747 for t in threads:
3748 t.join()
3749 getResponses.append( t.result )
3750 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003751 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003752 if isinstance( getResponses[ i ], list):
3753 current = set( getResponses[ i ] )
3754 if len( current ) == len( getResponses[ i ] ):
3755 # no repeats
3756 if onosSet != current:
3757 main.log.error( "ONOS" + str( i + 1 ) +
3758 " has incorrect view" +
3759 " of set " + onosSetName + ":\n" +
3760 str( getResponses[ i ] ) )
3761 main.log.debug( "Expected: " + str( onosSet ) )
3762 main.log.debug( "Actual: " + str( current ) )
3763 getResults = main.FALSE
3764 else:
3765 # error, set is not a set
3766 main.log.error( "ONOS" + str( i + 1 ) +
3767 " has repeat elements in" +
3768 " set " + onosSetName + ":\n" +
3769 str( getResponses[ i ] ) )
3770 getResults = main.FALSE
3771 elif getResponses[ i ] == main.ERROR:
3772 getResults = main.FALSE
3773 sizeResponses = []
3774 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003775 for i in range( main.numCtrls ):
3776 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003777 name="setTestSize-" + str( i ),
3778 args=[ onosSetName ] )
3779 threads.append( t )
3780 t.start()
3781 for t in threads:
3782 t.join()
3783 sizeResponses.append( t.result )
3784 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003785 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003786 if size != sizeResponses[ i ]:
3787 sizeResults = main.FALSE
3788 main.log.error( "ONOS" + str( i + 1 ) +
3789 " expected a size of " + str( size ) +
3790 " for set " + onosSetName +
3791 " but got " + str( sizeResponses[ i ] ) )
3792 removeResults = removeResults and getResults and sizeResults
3793 utilities.assert_equals( expect=main.TRUE,
3794 actual=removeResults,
3795 onpass="Set remove correct",
3796 onfail="Set remove was incorrect" )
3797
3798 main.step( "Distributed Set removeAll()" )
3799 onosSet.difference_update( addAllValue.split() )
3800 removeAllResponses = []
3801 threads = []
3802 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003803 for i in range( main.numCtrls ):
3804 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003805 name="setTestRemoveAll-" + str( i ),
3806 args=[ onosSetName, addAllValue ] )
3807 threads.append( t )
3808 t.start()
3809 for t in threads:
3810 t.join()
3811 removeAllResponses.append( t.result )
3812 except Exception, e:
3813 main.log.exception(e)
3814
3815 # main.TRUE = successfully changed the set
3816 # main.FALSE = action resulted in no change in set
3817 # main.ERROR - Some error in executing the function
3818 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003819 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003820 if removeAllResponses[ i ] == main.TRUE:
3821 # All is well
3822 pass
3823 elif removeAllResponses[ i ] == main.FALSE:
3824 # not in set, probably fine
3825 pass
3826 elif removeAllResponses[ i ] == main.ERROR:
3827 # Error in execution
3828 removeAllResults = main.FALSE
3829 else:
3830 # unexpected result
3831 removeAllResults = main.FALSE
3832 if removeAllResults != main.TRUE:
3833 main.log.error( "Error executing set removeAll" )
3834
3835 # Check if set is still correct
3836 size = len( onosSet )
3837 getResponses = []
3838 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003839 for i in range( main.numCtrls ):
3840 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003841 name="setTestGet-" + str( i ),
3842 args=[ onosSetName ] )
3843 threads.append( t )
3844 t.start()
3845 for t in threads:
3846 t.join()
3847 getResponses.append( t.result )
3848 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003849 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003850 if isinstance( getResponses[ i ], list):
3851 current = set( getResponses[ i ] )
3852 if len( current ) == len( getResponses[ i ] ):
3853 # no repeats
3854 if onosSet != current:
3855 main.log.error( "ONOS" + str( i + 1 ) +
3856 " has incorrect view" +
3857 " of set " + onosSetName + ":\n" +
3858 str( getResponses[ i ] ) )
3859 main.log.debug( "Expected: " + str( onosSet ) )
3860 main.log.debug( "Actual: " + str( current ) )
3861 getResults = main.FALSE
3862 else:
3863 # error, set is not a set
3864 main.log.error( "ONOS" + str( i + 1 ) +
3865 " has repeat elements in" +
3866 " set " + onosSetName + ":\n" +
3867 str( getResponses[ i ] ) )
3868 getResults = main.FALSE
3869 elif getResponses[ i ] == main.ERROR:
3870 getResults = main.FALSE
3871 sizeResponses = []
3872 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003873 for i in range( main.numCtrls ):
3874 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003875 name="setTestSize-" + str( i ),
3876 args=[ onosSetName ] )
3877 threads.append( t )
3878 t.start()
3879 for t in threads:
3880 t.join()
3881 sizeResponses.append( t.result )
3882 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003883 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003884 if size != sizeResponses[ i ]:
3885 sizeResults = main.FALSE
3886 main.log.error( "ONOS" + str( i + 1 ) +
3887 " expected a size of " + str( size ) +
3888 " for set " + onosSetName +
3889 " but got " + str( sizeResponses[ i ] ) )
3890 removeAllResults = removeAllResults and getResults and sizeResults
3891 utilities.assert_equals( expect=main.TRUE,
3892 actual=removeAllResults,
3893 onpass="Set removeAll correct",
3894 onfail="Set removeAll was incorrect" )
3895
3896 main.step( "Distributed Set addAll()" )
3897 onosSet.update( addAllValue.split() )
3898 addResponses = []
3899 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003900 for i in range( main.numCtrls ):
3901 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003902 name="setTestAddAll-" + str( i ),
3903 args=[ onosSetName, addAllValue ] )
3904 threads.append( t )
3905 t.start()
3906 for t in threads:
3907 t.join()
3908 addResponses.append( t.result )
3909
3910 # main.TRUE = successfully changed the set
3911 # main.FALSE = action resulted in no change in set
3912 # main.ERROR - Some error in executing the function
3913 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003914 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003915 if addResponses[ i ] == main.TRUE:
3916 # All is well
3917 pass
3918 elif addResponses[ i ] == main.FALSE:
3919 # Already in set, probably fine
3920 pass
3921 elif addResponses[ i ] == main.ERROR:
3922 # Error in execution
3923 addAllResults = main.FALSE
3924 else:
3925 # unexpected result
3926 addAllResults = main.FALSE
3927 if addAllResults != main.TRUE:
3928 main.log.error( "Error executing set addAll" )
3929
3930 # Check if set is still correct
3931 size = len( onosSet )
3932 getResponses = []
3933 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003934 for i in range( main.numCtrls ):
3935 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003936 name="setTestGet-" + str( i ),
3937 args=[ onosSetName ] )
3938 threads.append( t )
3939 t.start()
3940 for t in threads:
3941 t.join()
3942 getResponses.append( t.result )
3943 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003944 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003945 if isinstance( getResponses[ i ], list):
3946 current = set( getResponses[ i ] )
3947 if len( current ) == len( getResponses[ i ] ):
3948 # no repeats
3949 if onosSet != current:
3950 main.log.error( "ONOS" + str( i + 1 ) +
3951 " has incorrect view" +
3952 " of set " + onosSetName + ":\n" +
3953 str( getResponses[ i ] ) )
3954 main.log.debug( "Expected: " + str( onosSet ) )
3955 main.log.debug( "Actual: " + str( current ) )
3956 getResults = main.FALSE
3957 else:
3958 # error, set is not a set
3959 main.log.error( "ONOS" + str( i + 1 ) +
3960 " has repeat elements in" +
3961 " set " + onosSetName + ":\n" +
3962 str( getResponses[ i ] ) )
3963 getResults = main.FALSE
3964 elif getResponses[ i ] == main.ERROR:
3965 getResults = main.FALSE
3966 sizeResponses = []
3967 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003968 for i in range( main.numCtrls ):
3969 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003970 name="setTestSize-" + str( i ),
3971 args=[ onosSetName ] )
3972 threads.append( t )
3973 t.start()
3974 for t in threads:
3975 t.join()
3976 sizeResponses.append( t.result )
3977 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003978 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003979 if size != sizeResponses[ i ]:
3980 sizeResults = main.FALSE
3981 main.log.error( "ONOS" + str( i + 1 ) +
3982 " expected a size of " + str( size ) +
3983 " for set " + onosSetName +
3984 " but got " + str( sizeResponses[ i ] ) )
3985 addAllResults = addAllResults and getResults and sizeResults
3986 utilities.assert_equals( expect=main.TRUE,
3987 actual=addAllResults,
3988 onpass="Set addAll correct",
3989 onfail="Set addAll was incorrect" )
3990
3991 main.step( "Distributed Set clear()" )
3992 onosSet.clear()
3993 clearResponses = []
3994 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003995 for i in range( main.numCtrls ):
3996 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003997 name="setTestClear-" + str( i ),
3998 args=[ onosSetName, " "], # Values doesn't matter
3999 kwargs={ "clear": True } )
4000 threads.append( t )
4001 t.start()
4002 for t in threads:
4003 t.join()
4004 clearResponses.append( t.result )
4005
4006 # main.TRUE = successfully changed the set
4007 # main.FALSE = action resulted in no change in set
4008 # main.ERROR - Some error in executing the function
4009 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004010 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004011 if clearResponses[ i ] == main.TRUE:
4012 # All is well
4013 pass
4014 elif clearResponses[ i ] == main.FALSE:
4015 # Nothing set, probably fine
4016 pass
4017 elif clearResponses[ i ] == main.ERROR:
4018 # Error in execution
4019 clearResults = main.FALSE
4020 else:
4021 # unexpected result
4022 clearResults = main.FALSE
4023 if clearResults != main.TRUE:
4024 main.log.error( "Error executing set clear" )
4025
4026 # Check if set is still correct
4027 size = len( onosSet )
4028 getResponses = []
4029 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004030 for i in range( main.numCtrls ):
4031 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004032 name="setTestGet-" + str( i ),
4033 args=[ onosSetName ] )
4034 threads.append( t )
4035 t.start()
4036 for t in threads:
4037 t.join()
4038 getResponses.append( t.result )
4039 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004040 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004041 if isinstance( getResponses[ i ], list):
4042 current = set( getResponses[ i ] )
4043 if len( current ) == len( getResponses[ i ] ):
4044 # no repeats
4045 if onosSet != current:
4046 main.log.error( "ONOS" + str( i + 1 ) +
4047 " has incorrect view" +
4048 " of set " + onosSetName + ":\n" +
4049 str( getResponses[ i ] ) )
4050 main.log.debug( "Expected: " + str( onosSet ) )
4051 main.log.debug( "Actual: " + str( current ) )
4052 getResults = main.FALSE
4053 else:
4054 # error, set is not a set
4055 main.log.error( "ONOS" + str( i + 1 ) +
4056 " has repeat elements in" +
4057 " set " + onosSetName + ":\n" +
4058 str( getResponses[ i ] ) )
4059 getResults = main.FALSE
4060 elif getResponses[ i ] == main.ERROR:
4061 getResults = main.FALSE
4062 sizeResponses = []
4063 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004064 for i in range( main.numCtrls ):
4065 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004066 name="setTestSize-" + str( i ),
4067 args=[ onosSetName ] )
4068 threads.append( t )
4069 t.start()
4070 for t in threads:
4071 t.join()
4072 sizeResponses.append( t.result )
4073 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004074 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004075 if size != sizeResponses[ i ]:
4076 sizeResults = main.FALSE
4077 main.log.error( "ONOS" + str( i + 1 ) +
4078 " expected a size of " + str( size ) +
4079 " for set " + onosSetName +
4080 " but got " + str( sizeResponses[ i ] ) )
4081 clearResults = clearResults and getResults and sizeResults
4082 utilities.assert_equals( expect=main.TRUE,
4083 actual=clearResults,
4084 onpass="Set clear correct",
4085 onfail="Set clear was incorrect" )
4086
4087 main.step( "Distributed Set addAll()" )
4088 onosSet.update( addAllValue.split() )
4089 addResponses = []
4090 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004091 for i in range( main.numCtrls ):
4092 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004093 name="setTestAddAll-" + str( i ),
4094 args=[ onosSetName, addAllValue ] )
4095 threads.append( t )
4096 t.start()
4097 for t in threads:
4098 t.join()
4099 addResponses.append( t.result )
4100
4101 # main.TRUE = successfully changed the set
4102 # main.FALSE = action resulted in no change in set
4103 # main.ERROR - Some error in executing the function
4104 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004105 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004106 if addResponses[ i ] == main.TRUE:
4107 # All is well
4108 pass
4109 elif addResponses[ i ] == main.FALSE:
4110 # Already in set, probably fine
4111 pass
4112 elif addResponses[ i ] == main.ERROR:
4113 # Error in execution
4114 addAllResults = main.FALSE
4115 else:
4116 # unexpected result
4117 addAllResults = main.FALSE
4118 if addAllResults != main.TRUE:
4119 main.log.error( "Error executing set addAll" )
4120
4121 # Check if set is still correct
4122 size = len( onosSet )
4123 getResponses = []
4124 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004125 for i in range( main.numCtrls ):
4126 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004127 name="setTestGet-" + str( i ),
4128 args=[ onosSetName ] )
4129 threads.append( t )
4130 t.start()
4131 for t in threads:
4132 t.join()
4133 getResponses.append( t.result )
4134 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004135 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004136 if isinstance( getResponses[ i ], list):
4137 current = set( getResponses[ i ] )
4138 if len( current ) == len( getResponses[ i ] ):
4139 # no repeats
4140 if onosSet != current:
4141 main.log.error( "ONOS" + str( i + 1 ) +
4142 " has incorrect view" +
4143 " of set " + onosSetName + ":\n" +
4144 str( getResponses[ i ] ) )
4145 main.log.debug( "Expected: " + str( onosSet ) )
4146 main.log.debug( "Actual: " + str( current ) )
4147 getResults = main.FALSE
4148 else:
4149 # error, set is not a set
4150 main.log.error( "ONOS" + str( i + 1 ) +
4151 " has repeat elements in" +
4152 " set " + onosSetName + ":\n" +
4153 str( getResponses[ i ] ) )
4154 getResults = main.FALSE
4155 elif getResponses[ i ] == main.ERROR:
4156 getResults = main.FALSE
4157 sizeResponses = []
4158 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004159 for i in range( main.numCtrls ):
4160 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004161 name="setTestSize-" + str( i ),
4162 args=[ onosSetName ] )
4163 threads.append( t )
4164 t.start()
4165 for t in threads:
4166 t.join()
4167 sizeResponses.append( t.result )
4168 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004169 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004170 if size != sizeResponses[ i ]:
4171 sizeResults = main.FALSE
4172 main.log.error( "ONOS" + str( i + 1 ) +
4173 " expected a size of " + str( size ) +
4174 " for set " + onosSetName +
4175 " but got " + str( sizeResponses[ i ] ) )
4176 addAllResults = addAllResults and getResults and sizeResults
4177 utilities.assert_equals( expect=main.TRUE,
4178 actual=addAllResults,
4179 onpass="Set addAll correct",
4180 onfail="Set addAll was incorrect" )
4181
4182 main.step( "Distributed Set retain()" )
4183 onosSet.intersection_update( retainValue.split() )
4184 retainResponses = []
4185 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004186 for i in range( main.numCtrls ):
4187 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004188 name="setTestRetain-" + str( i ),
4189 args=[ onosSetName, retainValue ],
4190 kwargs={ "retain": True } )
4191 threads.append( t )
4192 t.start()
4193 for t in threads:
4194 t.join()
4195 retainResponses.append( t.result )
4196
4197 # main.TRUE = successfully changed the set
4198 # main.FALSE = action resulted in no change in set
4199 # main.ERROR - Some error in executing the function
4200 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004201 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004202 if retainResponses[ i ] == main.TRUE:
4203 # All is well
4204 pass
4205 elif retainResponses[ i ] == main.FALSE:
4206 # Already in set, probably fine
4207 pass
4208 elif retainResponses[ i ] == main.ERROR:
4209 # Error in execution
4210 retainResults = main.FALSE
4211 else:
4212 # unexpected result
4213 retainResults = main.FALSE
4214 if retainResults != main.TRUE:
4215 main.log.error( "Error executing set retain" )
4216
4217 # Check if set is still correct
4218 size = len( onosSet )
4219 getResponses = []
4220 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004221 for i in range( main.numCtrls ):
4222 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004223 name="setTestGet-" + str( i ),
4224 args=[ onosSetName ] )
4225 threads.append( t )
4226 t.start()
4227 for t in threads:
4228 t.join()
4229 getResponses.append( t.result )
4230 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004231 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004232 if isinstance( getResponses[ i ], list):
4233 current = set( getResponses[ i ] )
4234 if len( current ) == len( getResponses[ i ] ):
4235 # no repeats
4236 if onosSet != current:
4237 main.log.error( "ONOS" + str( i + 1 ) +
4238 " has incorrect view" +
4239 " of set " + onosSetName + ":\n" +
4240 str( getResponses[ i ] ) )
4241 main.log.debug( "Expected: " + str( onosSet ) )
4242 main.log.debug( "Actual: " + str( current ) )
4243 getResults = main.FALSE
4244 else:
4245 # error, set is not a set
4246 main.log.error( "ONOS" + str( i + 1 ) +
4247 " has repeat elements in" +
4248 " set " + onosSetName + ":\n" +
4249 str( getResponses[ i ] ) )
4250 getResults = main.FALSE
4251 elif getResponses[ i ] == main.ERROR:
4252 getResults = main.FALSE
4253 sizeResponses = []
4254 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004255 for i in range( main.numCtrls ):
4256 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004257 name="setTestSize-" + str( i ),
4258 args=[ onosSetName ] )
4259 threads.append( t )
4260 t.start()
4261 for t in threads:
4262 t.join()
4263 sizeResponses.append( t.result )
4264 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004265 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004266 if size != sizeResponses[ i ]:
4267 sizeResults = main.FALSE
4268 main.log.error( "ONOS" + str( i + 1 ) +
4269 " expected a size of " +
4270 str( size ) + " for set " + onosSetName +
4271 " but got " + str( sizeResponses[ i ] ) )
4272 retainResults = retainResults and getResults and sizeResults
4273 utilities.assert_equals( expect=main.TRUE,
4274 actual=retainResults,
4275 onpass="Set retain correct",
4276 onfail="Set retain was incorrect" )
4277
Jon Hall2a5002c2015-08-21 16:49:11 -07004278 # Transactional maps
4279 main.step( "Partitioned Transactional maps put" )
4280 tMapValue = "Testing"
4281 numKeys = 100
4282 putResult = True
4283 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4284 if len( putResponses ) == 100:
4285 for i in putResponses:
4286 if putResponses[ i ][ 'value' ] != tMapValue:
4287 putResult = False
4288 else:
4289 putResult = False
4290 if not putResult:
4291 main.log.debug( "Put response values: " + str( putResponses ) )
4292 utilities.assert_equals( expect=True,
4293 actual=putResult,
4294 onpass="Partitioned Transactional Map put successful",
4295 onfail="Partitioned Transactional Map put values are incorrect" )
4296
4297 main.step( "Partitioned Transactional maps get" )
4298 getCheck = True
4299 for n in range( 1, numKeys + 1 ):
4300 getResponses = []
4301 threads = []
4302 valueCheck = True
4303 for i in range( main.numCtrls ):
4304 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4305 name="TMap-get-" + str( i ),
4306 args=[ "Key" + str ( n ) ] )
4307 threads.append( t )
4308 t.start()
4309 for t in threads:
4310 t.join()
4311 getResponses.append( t.result )
4312 for node in getResponses:
4313 if node != tMapValue:
4314 valueCheck = False
4315 if not valueCheck:
4316 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4317 main.log.warn( getResponses )
4318 getCheck = getCheck and valueCheck
4319 utilities.assert_equals( expect=True,
4320 actual=getCheck,
4321 onpass="Partitioned Transactional Map get values were correct",
4322 onfail="Partitioned Transactional Map values incorrect" )
4323
4324 main.step( "In-memory Transactional maps put" )
4325 tMapValue = "Testing"
4326 numKeys = 100
4327 putResult = True
4328 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4329 if len( putResponses ) == 100:
4330 for i in putResponses:
4331 if putResponses[ i ][ 'value' ] != tMapValue:
4332 putResult = False
4333 else:
4334 putResult = False
4335 if not putResult:
4336 main.log.debug( "Put response values: " + str( putResponses ) )
4337 utilities.assert_equals( expect=True,
4338 actual=putResult,
4339 onpass="In-Memory Transactional Map put successful",
4340 onfail="In-Memory Transactional Map put values are incorrect" )
4341
4342 main.step( "In-Memory Transactional maps get" )
4343 getCheck = True
4344 for n in range( 1, numKeys + 1 ):
4345 getResponses = []
4346 threads = []
4347 valueCheck = True
4348 for i in range( main.numCtrls ):
4349 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4350 name="TMap-get-" + str( i ),
4351 args=[ "Key" + str ( n ) ],
4352 kwargs={ "inMemory": True } )
4353 threads.append( t )
4354 t.start()
4355 for t in threads:
4356 t.join()
4357 getResponses.append( t.result )
4358 for node in getResponses:
4359 if node != tMapValue:
4360 valueCheck = False
4361 if not valueCheck:
4362 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4363 main.log.warn( getResponses )
4364 getCheck = getCheck and valueCheck
4365 utilities.assert_equals( expect=True,
4366 actual=getCheck,
4367 onpass="In-Memory Transactional Map get values were correct",
4368 onfail="In-Memory Transactional Map values incorrect" )