blob: e404da47aea1adc1f48e40ac6a945ba4f74b15a1 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
28class HAsanity:
29
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall5cf14d52015-07-16 12:15:19 -070052 main.log.info( "ONOS HA Sanity test - initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070071 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79
80 # FIXME: just get controller port from params?
81 # TODO: do we really need all these?
82 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
83 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
84 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
85 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
86 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
87 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
88 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
89
Jon Halle1a3b752015-07-22 13:02:46 -070090 try:
91 fileName = "Counters"
92 path = main.params[ 'imports' ][ 'path' ]
93 main.Counters = imp.load_source( fileName,
94 path + fileName + ".py" )
95 except Exception as e:
96 main.log.exception( e )
97 main.cleanup()
98 main.exit()
99
100 main.CLIs = []
101 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700102 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700103 for i in range( 1, main.numCtrls + 1 ):
104 try:
105 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
106 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
107 ipList.append( main.nodes[ -1 ].ip_address )
108 except AttributeError:
109 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700110
111 main.step( "Create cell file" )
112 cellAppString = main.params[ 'ENV' ][ 'appString' ]
113 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
114 main.Mininet1.ip_address,
115 cellAppString, ipList )
116 main.step( "Applying cell variable to environment" )
117 cellResult = main.ONOSbench.setCell( cellName )
118 verifyResult = main.ONOSbench.verifyCell()
119
120 # FIXME:this is short term fix
121 main.log.info( "Removing raft logs" )
122 main.ONOSbench.onosRemoveRaftLogs()
123
124 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700125 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700126 main.ONOSbench.onosUninstall( node.ip_address )
127
128 # Make sure ONOS is DEAD
129 main.log.info( "Killing any ONOS processes" )
130 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700131 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700132 killed = main.ONOSbench.onosKill( node.ip_address )
133 killResults = killResults and killed
134
135 cleanInstallResult = main.TRUE
136 gitPullResult = main.TRUE
137
138 main.step( "Starting Mininet" )
139 # scp topo file to mininet
140 # TODO: move to params?
141 topoName = "obelisk.py"
142 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700143 main.ONOSbench.scp( main.Mininet1,
144 filePath + topoName,
145 main.Mininet1.home,
146 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700147 mnResult = main.Mininet1.startNet( )
148 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
149 onpass="Mininet Started",
150 onfail="Error starting Mininet" )
151
152 main.step( "Git checkout and pull " + gitBranch )
153 if PULLCODE:
154 main.ONOSbench.gitCheckout( gitBranch )
155 gitPullResult = main.ONOSbench.gitPull()
156 # values of 1 or 3 are good
157 utilities.assert_lesser( expect=0, actual=gitPullResult,
158 onpass="Git pull successful",
159 onfail="Git pull failed" )
160 main.ONOSbench.getVersion( report=True )
161
162 main.step( "Using mvn clean install" )
163 cleanInstallResult = main.TRUE
164 if PULLCODE and gitPullResult == main.TRUE:
165 cleanInstallResult = main.ONOSbench.cleanInstall()
166 else:
167 main.log.warn( "Did not pull new code so skipping mvn " +
168 "clean install" )
169 utilities.assert_equals( expect=main.TRUE,
170 actual=cleanInstallResult,
171 onpass="MCI successful",
172 onfail="MCI failed" )
173 # GRAPHS
174 # NOTE: important params here:
175 # job = name of Jenkins job
176 # Plot Name = Plot-HA, only can be used if multiple plots
177 # index = The number of the graph under plot name
178 job = "HAsanity"
179 plotName = "Plot-HA"
180 graphs = '<ac:structured-macro ac:name="html">\n'
181 graphs += '<ac:plain-text-body><![CDATA[\n'
182 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
183 '/plot/' + plotName + '/getPlot?index=0' +\
184 '&width=500&height=300"' +\
185 'noborder="0" width="500" height="300" scrolling="yes" ' +\
186 'seamless="seamless"></iframe>\n'
187 graphs += ']]></ac:plain-text-body>\n'
188 graphs += '</ac:structured-macro>\n'
189 main.log.wiki(graphs)
190
191 main.step( "Creating ONOS package" )
192 packageResult = main.ONOSbench.onosPackage()
193 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
194 onpass="ONOS package successful",
195 onfail="ONOS package failed" )
196
197 main.step( "Installing ONOS package" )
198 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700199 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700200 tmpResult = main.ONOSbench.onosInstall( options="-f",
201 node=node.ip_address )
202 onosInstallResult = onosInstallResult and tmpResult
203 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
204 onpass="ONOS install successful",
205 onfail="ONOS install failed" )
206
207 main.step( "Checking if ONOS is up yet" )
208 for i in range( 2 ):
209 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700210 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700211 started = main.ONOSbench.isup( node.ip_address )
212 if not started:
213 main.log.error( node.name + " didn't start!" )
214 main.ONOSbench.onosStop( node.ip_address )
215 main.ONOSbench.onosStart( node.ip_address )
216 onosIsupResult = onosIsupResult and started
217 if onosIsupResult == main.TRUE:
218 break
219 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
220 onpass="ONOS startup successful",
221 onfail="ONOS startup failed" )
222
223 main.log.step( "Starting ONOS CLI sessions" )
224 cliResults = main.TRUE
225 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700226 for i in range( main.numCtrls ):
227 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700228 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700229 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 threads.append( t )
231 t.start()
232
233 for t in threads:
234 t.join()
235 cliResults = cliResults and t.result
236 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
237 onpass="ONOS cli startup successful",
238 onfail="ONOS cli startup failed" )
239
240 if main.params[ 'tcpdump' ].lower() == "true":
241 main.step( "Start Packet Capture MN" )
242 main.Mininet2.startTcpdump(
243 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
244 + "-MN.pcap",
245 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
246 port=main.params[ 'MNtcpdump' ][ 'port' ] )
247
248 main.step( "App Ids check" )
249 appCheck = main.TRUE
250 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700251 for i in range( main.numCtrls ):
252 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700253 name="appToIDCheck-" + str( i ),
254 args=[] )
255 threads.append( t )
256 t.start()
257
258 for t in threads:
259 t.join()
260 appCheck = appCheck and t.result
261 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700262 main.log.warn( main.CLIs[0].apps() )
263 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700264 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
265 onpass="App Ids seem to be correct",
266 onfail="Something is wrong with app Ids" )
267
268 if cliResults == main.FALSE:
269 main.log.error( "Failed to start ONOS, stopping test" )
270 main.cleanup()
271 main.exit()
272
273 def CASE2( self, main ):
274 """
275 Assign devices to controllers
276 """
277 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700278 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700279 assert main, "main not defined"
280 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700281 assert main.CLIs, "main.CLIs not defined"
282 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700283 assert ONOS1Port, "ONOS1Port not defined"
284 assert ONOS2Port, "ONOS2Port not defined"
285 assert ONOS3Port, "ONOS3Port not defined"
286 assert ONOS4Port, "ONOS4Port not defined"
287 assert ONOS5Port, "ONOS5Port not defined"
288 assert ONOS6Port, "ONOS6Port not defined"
289 assert ONOS7Port, "ONOS7Port not defined"
290
291 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700292 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700293 "and check that an ONOS node becomes the " +\
294 "master of the device."
295 main.step( "Assign switches to controllers" )
296
297 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700298 for i in range( main.numCtrls ):
299 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700300 swList = []
301 for i in range( 1, 29 ):
302 swList.append( "s" + str( i ) )
303 main.Mininet1.assignSwController( sw=swList, ip=ipList )
304
305 mastershipCheck = main.TRUE
306 for i in range( 1, 29 ):
307 response = main.Mininet1.getSwController( "s" + str( i ) )
308 try:
309 main.log.info( str( response ) )
310 except Exception:
311 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700312 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700313 if re.search( "tcp:" + node.ip_address, response ):
314 mastershipCheck = mastershipCheck and main.TRUE
315 else:
316 main.log.error( "Error, node " + node.ip_address + " is " +
317 "not in the list of controllers s" +
318 str( i ) + " is connecting to." )
319 mastershipCheck = main.FALSE
320 utilities.assert_equals(
321 expect=main.TRUE,
322 actual=mastershipCheck,
323 onpass="Switch mastership assigned correctly",
324 onfail="Switches not assigned correctly to controllers" )
325
326 def CASE21( self, main ):
327 """
328 Assign mastership to controllers
329 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700330 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700331 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700332 assert main, "main not defined"
333 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700334 assert main.CLIs, "main.CLIs not defined"
335 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700336 assert ONOS1Port, "ONOS1Port not defined"
337 assert ONOS2Port, "ONOS2Port not defined"
338 assert ONOS3Port, "ONOS3Port not defined"
339 assert ONOS4Port, "ONOS4Port not defined"
340 assert ONOS5Port, "ONOS5Port not defined"
341 assert ONOS6Port, "ONOS6Port not defined"
342 assert ONOS7Port, "ONOS7Port not defined"
343
344 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700345 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700346 "device. Then manually assign" +\
347 " mastership to specific ONOS nodes using" +\
348 " 'device-role'"
349 main.step( "Assign mastership of switches to specific controllers" )
350 # Manually assign mastership to the controller we want
351 roleCall = main.TRUE
352
353 ipList = [ ]
354 deviceList = []
355 try:
356 # Assign mastership to specific controllers. This assignment was
357 # determined for a 7 node cluser, but will work with any sized
358 # cluster
359 for i in range( 1, 29 ): # switches 1 through 28
360 # set up correct variables:
361 if i == 1:
362 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700363 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700364 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
365 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700366 c = 1 % main.numCtrls
367 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
369 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700370 c = 1 % main.numCtrls
371 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
373 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700374 c = 3 % main.numCtrls
375 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700376 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
377 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700378 c = 2 % main.numCtrls
379 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700380 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
381 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700382 c = 2 % main.numCtrls
383 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700384 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
385 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700386 c = 5 % main.numCtrls
387 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700388 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
389 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700390 c = 4 % main.numCtrls
391 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700392 dpid = '3' + str( i ).zfill( 3 )
393 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
394 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700395 c = 6 % main.numCtrls
396 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700397 dpid = '6' + str( i ).zfill( 3 )
398 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
399 elif i == 28:
400 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700401 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700402 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
403 else:
404 main.log.error( "You didn't write an else statement for " +
405 "switch s" + str( i ) )
406 roleCall = main.FALSE
407 # Assign switch
408 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
409 # TODO: make this controller dynamic
410 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
411 ip )
412 ipList.append( ip )
413 deviceList.append( deviceId )
414 except ( AttributeError, AssertionError ):
415 main.log.exception( "Something is wrong with ONOS device view" )
416 main.log.info( main.ONOScli1.devices() )
417 utilities.assert_equals(
418 expect=main.TRUE,
419 actual=roleCall,
420 onpass="Re-assigned switch mastership to designated controller",
421 onfail="Something wrong with deviceRole calls" )
422
423 main.step( "Check mastership was correctly assigned" )
424 roleCheck = main.TRUE
425 # NOTE: This is due to the fact that device mastership change is not
426 # atomic and is actually a multi step process
427 time.sleep( 5 )
428 for i in range( len( ipList ) ):
429 ip = ipList[i]
430 deviceId = deviceList[i]
431 # Check assignment
432 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
433 if ip in master:
434 roleCheck = roleCheck and main.TRUE
435 else:
436 roleCheck = roleCheck and main.FALSE
437 main.log.error( "Error, controller " + ip + " is not" +
438 " master " + "of device " +
439 str( deviceId ) + ". Master is " +
440 repr( master ) + "." )
441 utilities.assert_equals(
442 expect=main.TRUE,
443 actual=roleCheck,
444 onpass="Switches were successfully reassigned to designated " +
445 "controller",
446 onfail="Switches were not successfully reassigned" )
447
448 def CASE3( self, main ):
449 """
450 Assign intents
451 """
452 import time
453 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700454 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700455 assert main, "main not defined"
456 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700457 assert main.CLIs, "main.CLIs not defined"
458 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700460 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700461 "assign predetermined host-to-host intents." +\
462 " After installation, check that the intent" +\
463 " is distributed to all nodes and the state" +\
464 " is INSTALLED"
465
466 # install onos-app-fwd
467 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700468 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700469 utilities.assert_equals( expect=main.TRUE, actual=installResults,
470 onpass="Install fwd successful",
471 onfail="Install fwd failed" )
472
473 main.step( "Check app ids" )
474 appCheck = main.TRUE
475 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700476 for i in range( main.numCtrls ):
477 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700478 name="appToIDCheck-" + str( i ),
479 args=[] )
480 threads.append( t )
481 t.start()
482
483 for t in threads:
484 t.join()
485 appCheck = appCheck and t.result
486 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700487 main.log.warn( main.CLIs[0].apps() )
488 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700489 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
490 onpass="App Ids seem to be correct",
491 onfail="Something is wrong with app Ids" )
492
493 main.step( "Discovering Hosts( Via pingall for now )" )
494 # FIXME: Once we have a host discovery mechanism, use that instead
495 # REACTIVE FWD test
496 pingResult = main.FALSE
497 for i in range(2): # Retry if pingall fails first time
498 time1 = time.time()
499 pingResult = main.Mininet1.pingall()
500 if i == 0:
501 utilities.assert_equals(
502 expect=main.TRUE,
503 actual=pingResult,
504 onpass="Reactive Pingall test passed",
505 onfail="Reactive Pingall failed, " +
506 "one or more ping pairs failed" )
507 time2 = time.time()
508 main.log.info( "Time for pingall: %2f seconds" %
509 ( time2 - time1 ) )
510 # timeout for fwd flows
511 time.sleep( 11 )
512 # uninstall onos-app-fwd
513 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700514 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700515 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
516 onpass="Uninstall fwd successful",
517 onfail="Uninstall fwd failed" )
518 '''
519 main.Mininet1.handle.sendline( "py [ h.cmd( \"arping -c 1 10.1.1.1 \" ) for h in net.hosts ] ")
520 import time
521 time.sleep(60)
522 '''
523
524 main.step( "Check app ids" )
525 threads = []
526 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700527 for i in range( main.numCtrls ):
528 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700529 name="appToIDCheck-" + str( i ),
530 args=[] )
531 threads.append( t )
532 t.start()
533
534 for t in threads:
535 t.join()
536 appCheck2 = appCheck2 and t.result
537 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700538 main.log.warn( main.CLIs[0].apps() )
539 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700540 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
541 onpass="App Ids seem to be correct",
542 onfail="Something is wrong with app Ids" )
543
544 main.step( "Add host intents via cli" )
545 intentIds = []
546 # TODO: move the host numbers to params
547 # Maybe look at all the paths we ping?
548 intentAddResult = True
549 hostResult = main.TRUE
550 for i in range( 8, 18 ):
551 main.log.info( "Adding host intent between h" + str( i ) +
552 " and h" + str( i + 10 ) )
553 host1 = "00:00:00:00:00:" + \
554 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
555 host2 = "00:00:00:00:00:" + \
556 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
557 # NOTE: getHost can return None
558 host1Dict = main.ONOScli1.getHost( host1 )
559 host2Dict = main.ONOScli1.getHost( host2 )
560 host1Id = None
561 host2Id = None
562 if host1Dict and host2Dict:
563 host1Id = host1Dict.get( 'id', None )
564 host2Id = host2Dict.get( 'id', None )
565 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700566 nodeNum = ( i % main.numCtrls )
567 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700568 if tmpId:
569 main.log.info( "Added intent with id: " + tmpId )
570 intentIds.append( tmpId )
571 else:
572 main.log.error( "addHostIntent returned: " +
573 repr( tmpId ) )
574 else:
575 main.log.error( "Error, getHost() failed for h" + str( i ) +
576 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700577 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700578 main.log.warn( "Hosts output: " )
579 try:
580 main.log.warn( json.dumps( json.loads( hosts ),
581 sort_keys=True,
582 indent=4,
583 separators=( ',', ': ' ) ) )
584 except ( ValueError, TypeError ):
585 main.log.warn( repr( hosts ) )
586 hostResult = main.FALSE
587 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
588 onpass="Found a host id for each host",
589 onfail="Error looking up host ids" )
590
591 intentStart = time.time()
592 onosIds = main.ONOScli1.getAllIntentsId()
593 main.log.info( "Submitted intents: " + str( intentIds ) )
594 main.log.info( "Intents in ONOS: " + str( onosIds ) )
595 for intent in intentIds:
596 if intent in onosIds:
597 pass # intent submitted is in onos
598 else:
599 intentAddResult = False
600 if intentAddResult:
601 intentStop = time.time()
602 else:
603 intentStop = None
604 # Print the intent states
605 intents = main.ONOScli1.intents()
606 intentStates = []
607 installedCheck = True
608 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
609 count = 0
610 try:
611 for intent in json.loads( intents ):
612 state = intent.get( 'state', None )
613 if "INSTALLED" not in state:
614 installedCheck = False
615 intentId = intent.get( 'id', None )
616 intentStates.append( ( intentId, state ) )
617 except ( ValueError, TypeError ):
618 main.log.exception( "Error parsing intents" )
619 # add submitted intents not in the store
620 tmplist = [ i for i, s in intentStates ]
621 missingIntents = False
622 for i in intentIds:
623 if i not in tmplist:
624 intentStates.append( ( i, " - " ) )
625 missingIntents = True
626 intentStates.sort()
627 for i, s in intentStates:
628 count += 1
629 main.log.info( "%-6s%-15s%-15s" %
630 ( str( count ), str( i ), str( s ) ) )
631 leaders = main.ONOScli1.leaders()
632 try:
633 missing = False
634 if leaders:
635 parsedLeaders = json.loads( leaders )
636 main.log.warn( json.dumps( parsedLeaders,
637 sort_keys=True,
638 indent=4,
639 separators=( ',', ': ' ) ) )
640 # check for all intent partitions
641 topics = []
642 for i in range( 14 ):
643 topics.append( "intent-partition-" + str( i ) )
644 main.log.debug( topics )
645 ONOStopics = [ j['topic'] for j in parsedLeaders ]
646 for topic in topics:
647 if topic not in ONOStopics:
648 main.log.error( "Error: " + topic +
649 " not in leaders" )
650 missing = True
651 else:
652 main.log.error( "leaders() returned None" )
653 except ( ValueError, TypeError ):
654 main.log.exception( "Error parsing leaders" )
655 main.log.error( repr( leaders ) )
656 # Check all nodes
657 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700658 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700659 response = node.leaders( jsonFormat=False)
660 main.log.warn( str( node.name ) + " leaders output: \n" +
661 str( response ) )
662
663 partitions = main.ONOScli1.partitions()
664 try:
665 if partitions :
666 parsedPartitions = json.loads( partitions )
667 main.log.warn( json.dumps( parsedPartitions,
668 sort_keys=True,
669 indent=4,
670 separators=( ',', ': ' ) ) )
671 # TODO check for a leader in all paritions
672 # TODO check for consistency among nodes
673 else:
674 main.log.error( "partitions() returned None" )
675 except ( ValueError, TypeError ):
676 main.log.exception( "Error parsing partitions" )
677 main.log.error( repr( partitions ) )
678 pendingMap = main.ONOScli1.pendingMap()
679 try:
680 if pendingMap :
681 parsedPending = json.loads( pendingMap )
682 main.log.warn( json.dumps( parsedPending,
683 sort_keys=True,
684 indent=4,
685 separators=( ',', ': ' ) ) )
686 # TODO check something here?
687 else:
688 main.log.error( "pendingMap() returned None" )
689 except ( ValueError, TypeError ):
690 main.log.exception( "Error parsing pending map" )
691 main.log.error( repr( pendingMap ) )
692
693 intentAddResult = bool( intentAddResult and not missingIntents and
694 installedCheck )
695 if not intentAddResult:
696 main.log.error( "Error in pushing host intents to ONOS" )
697
698 main.step( "Intent Anti-Entropy dispersion" )
699 for i in range(100):
700 correct = True
701 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700702 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700703 onosIds = []
704 ids = cli.getAllIntentsId()
705 onosIds.append( ids )
706 main.log.debug( "Intents in " + cli.name + ": " +
707 str( sorted( onosIds ) ) )
708 if sorted( ids ) != sorted( intentIds ):
709 main.log.warn( "Set of intent IDs doesn't match" )
710 correct = False
711 break
712 else:
713 intents = json.loads( cli.intents() )
714 for intent in intents:
715 if intent[ 'state' ] != "INSTALLED":
716 main.log.warn( "Intent " + intent[ 'id' ] +
717 " is " + intent[ 'state' ] )
718 correct = False
719 break
720 if correct:
721 break
722 else:
723 time.sleep(1)
724 if not intentStop:
725 intentStop = time.time()
726 global gossipTime
727 gossipTime = intentStop - intentStart
728 main.log.info( "It took about " + str( gossipTime ) +
729 " seconds for all intents to appear in each node" )
730 # FIXME: make this time configurable/calculate based off of number of
731 # nodes and gossip rounds
732 utilities.assert_greater_equals(
733 expect=40, actual=gossipTime,
734 onpass="ECM anti-entropy for intents worked within " +
735 "expected time",
736 onfail="Intent ECM anti-entropy took too long" )
737 if gossipTime <= 40:
738 intentAddResult = True
739
740 if not intentAddResult or "key" in pendingMap:
741 import time
742 installedCheck = True
743 main.log.info( "Sleeping 60 seconds to see if intents are found" )
744 time.sleep( 60 )
745 onosIds = main.ONOScli1.getAllIntentsId()
746 main.log.info( "Submitted intents: " + str( intentIds ) )
747 main.log.info( "Intents in ONOS: " + str( onosIds ) )
748 # Print the intent states
749 intents = main.ONOScli1.intents()
750 intentStates = []
751 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
752 count = 0
753 try:
754 for intent in json.loads( intents ):
755 # Iter through intents of a node
756 state = intent.get( 'state', None )
757 if "INSTALLED" not in state:
758 installedCheck = False
759 intentId = intent.get( 'id', None )
760 intentStates.append( ( intentId, state ) )
761 except ( ValueError, TypeError ):
762 main.log.exception( "Error parsing intents" )
763 # add submitted intents not in the store
764 tmplist = [ i for i, s in intentStates ]
765 for i in intentIds:
766 if i not in tmplist:
767 intentStates.append( ( i, " - " ) )
768 intentStates.sort()
769 for i, s in intentStates:
770 count += 1
771 main.log.info( "%-6s%-15s%-15s" %
772 ( str( count ), str( i ), str( s ) ) )
773 leaders = main.ONOScli1.leaders()
774 try:
775 missing = False
776 if leaders:
777 parsedLeaders = json.loads( leaders )
778 main.log.warn( json.dumps( parsedLeaders,
779 sort_keys=True,
780 indent=4,
781 separators=( ',', ': ' ) ) )
782 # check for all intent partitions
783 # check for election
784 topics = []
785 for i in range( 14 ):
786 topics.append( "intent-partition-" + str( i ) )
787 # FIXME: this should only be after we start the app
788 topics.append( "org.onosproject.election" )
789 main.log.debug( topics )
790 ONOStopics = [ j['topic'] for j in parsedLeaders ]
791 for topic in topics:
792 if topic not in ONOStopics:
793 main.log.error( "Error: " + topic +
794 " not in leaders" )
795 missing = True
796 else:
797 main.log.error( "leaders() returned None" )
798 except ( ValueError, TypeError ):
799 main.log.exception( "Error parsing leaders" )
800 main.log.error( repr( leaders ) )
801 # Check all nodes
802 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700803 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700804 response = node.leaders( jsonFormat=False)
805 main.log.warn( str( node.name ) + " leaders output: \n" +
806 str( response ) )
807
808 partitions = main.ONOScli1.partitions()
809 try:
810 if partitions :
811 parsedPartitions = json.loads( partitions )
812 main.log.warn( json.dumps( parsedPartitions,
813 sort_keys=True,
814 indent=4,
815 separators=( ',', ': ' ) ) )
816 # TODO check for a leader in all paritions
817 # TODO check for consistency among nodes
818 else:
819 main.log.error( "partitions() returned None" )
820 except ( ValueError, TypeError ):
821 main.log.exception( "Error parsing partitions" )
822 main.log.error( repr( partitions ) )
823 pendingMap = main.ONOScli1.pendingMap()
824 try:
825 if pendingMap :
826 parsedPending = json.loads( pendingMap )
827 main.log.warn( json.dumps( parsedPending,
828 sort_keys=True,
829 indent=4,
830 separators=( ',', ': ' ) ) )
831 # TODO check something here?
832 else:
833 main.log.error( "pendingMap() returned None" )
834 except ( ValueError, TypeError ):
835 main.log.exception( "Error parsing pending map" )
836 main.log.error( repr( pendingMap ) )
837
838 def CASE4( self, main ):
839 """
840 Ping across added host intents
841 """
842 import json
843 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700844 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700845 assert main, "main not defined"
846 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700847 assert main.CLIs, "main.CLIs not defined"
848 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700849 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700850 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700851 "functionality and check the state of " +\
852 "the intent"
853 main.step( "Ping across added host intents" )
854 PingResult = main.TRUE
855 for i in range( 8, 18 ):
856 ping = main.Mininet1.pingHost( src="h" + str( i ),
857 target="h" + str( i + 10 ) )
858 PingResult = PingResult and ping
859 if ping == main.FALSE:
860 main.log.warn( "Ping failed between h" + str( i ) +
861 " and h" + str( i + 10 ) )
862 elif ping == main.TRUE:
863 main.log.info( "Ping test passed!" )
864 # Don't set PingResult or you'd override failures
865 if PingResult == main.FALSE:
866 main.log.error(
867 "Intents have not been installed correctly, pings failed." )
868 # TODO: pretty print
869 main.log.warn( "ONOS1 intents: " )
870 try:
871 tmpIntents = main.ONOScli1.intents()
872 main.log.warn( json.dumps( json.loads( tmpIntents ),
873 sort_keys=True,
874 indent=4,
875 separators=( ',', ': ' ) ) )
876 except ( ValueError, TypeError ):
877 main.log.warn( repr( tmpIntents ) )
878 utilities.assert_equals(
879 expect=main.TRUE,
880 actual=PingResult,
881 onpass="Intents have been installed correctly and pings work",
882 onfail="Intents have not been installed correctly, pings failed." )
883
884 main.step( "Check Intent state" )
885 installedCheck = False
886 loopCount = 0
887 while not installedCheck and loopCount < 40:
888 installedCheck = True
889 # Print the intent states
890 intents = main.ONOScli1.intents()
891 intentStates = []
892 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
893 count = 0
894 # Iter through intents of a node
895 try:
896 for intent in json.loads( intents ):
897 state = intent.get( 'state', None )
898 if "INSTALLED" not in state:
899 installedCheck = False
900 intentId = intent.get( 'id', None )
901 intentStates.append( ( intentId, state ) )
902 except ( ValueError, TypeError ):
903 main.log.exception( "Error parsing intents." )
904 # Print states
905 intentStates.sort()
906 for i, s in intentStates:
907 count += 1
908 main.log.info( "%-6s%-15s%-15s" %
909 ( str( count ), str( i ), str( s ) ) )
910 if not installedCheck:
911 time.sleep( 1 )
912 loopCount += 1
913 utilities.assert_equals( expect=True, actual=installedCheck,
914 onpass="Intents are all INSTALLED",
915 onfail="Intents are not all in " +
916 "INSTALLED state" )
917
918 main.step( "Check leadership of topics" )
919 leaders = main.ONOScli1.leaders()
920 topicCheck = main.TRUE
921 try:
922 if leaders:
923 parsedLeaders = json.loads( leaders )
924 main.log.warn( json.dumps( parsedLeaders,
925 sort_keys=True,
926 indent=4,
927 separators=( ',', ': ' ) ) )
928 # check for all intent partitions
929 # check for election
930 # TODO: Look at Devices as topics now that it uses this system
931 topics = []
932 for i in range( 14 ):
933 topics.append( "intent-partition-" + str( i ) )
934 # FIXME: this should only be after we start the app
935 # FIXME: topics.append( "org.onosproject.election" )
936 # Print leaders output
937 main.log.debug( topics )
938 ONOStopics = [ j['topic'] for j in parsedLeaders ]
939 for topic in topics:
940 if topic not in ONOStopics:
941 main.log.error( "Error: " + topic +
942 " not in leaders" )
943 topicCheck = main.FALSE
944 else:
945 main.log.error( "leaders() returned None" )
946 topicCheck = main.FALSE
947 except ( ValueError, TypeError ):
948 topicCheck = main.FALSE
949 main.log.exception( "Error parsing leaders" )
950 main.log.error( repr( leaders ) )
951 # TODO: Check for a leader of these topics
952 # Check all nodes
953 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700954 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700955 response = node.leaders( jsonFormat=False)
956 main.log.warn( str( node.name ) + " leaders output: \n" +
957 str( response ) )
958
959 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
960 onpass="intent Partitions is in leaders",
961 onfail="Some topics were lost " )
962 # Print partitions
963 partitions = main.ONOScli1.partitions()
964 try:
965 if partitions :
966 parsedPartitions = json.loads( partitions )
967 main.log.warn( json.dumps( parsedPartitions,
968 sort_keys=True,
969 indent=4,
970 separators=( ',', ': ' ) ) )
971 # TODO check for a leader in all paritions
972 # TODO check for consistency among nodes
973 else:
974 main.log.error( "partitions() returned None" )
975 except ( ValueError, TypeError ):
976 main.log.exception( "Error parsing partitions" )
977 main.log.error( repr( partitions ) )
978 # Print Pending Map
979 pendingMap = main.ONOScli1.pendingMap()
980 try:
981 if pendingMap :
982 parsedPending = json.loads( pendingMap )
983 main.log.warn( json.dumps( parsedPending,
984 sort_keys=True,
985 indent=4,
986 separators=( ',', ': ' ) ) )
987 # TODO check something here?
988 else:
989 main.log.error( "pendingMap() returned None" )
990 except ( ValueError, TypeError ):
991 main.log.exception( "Error parsing pending map" )
992 main.log.error( repr( pendingMap ) )
993
994 if not installedCheck:
995 main.log.info( "Waiting 60 seconds to see if the state of " +
996 "intents change" )
997 time.sleep( 60 )
998 # Print the intent states
999 intents = main.ONOScli1.intents()
1000 intentStates = []
1001 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1002 count = 0
1003 # Iter through intents of a node
1004 try:
1005 for intent in json.loads( intents ):
1006 state = intent.get( 'state', None )
1007 if "INSTALLED" not in state:
1008 installedCheck = False
1009 intentId = intent.get( 'id', None )
1010 intentStates.append( ( intentId, state ) )
1011 except ( ValueError, TypeError ):
1012 main.log.exception( "Error parsing intents." )
1013 intentStates.sort()
1014 for i, s in intentStates:
1015 count += 1
1016 main.log.info( "%-6s%-15s%-15s" %
1017 ( str( count ), str( i ), str( s ) ) )
1018 leaders = main.ONOScli1.leaders()
1019 try:
1020 missing = False
1021 if leaders:
1022 parsedLeaders = json.loads( leaders )
1023 main.log.warn( json.dumps( parsedLeaders,
1024 sort_keys=True,
1025 indent=4,
1026 separators=( ',', ': ' ) ) )
1027 # check for all intent partitions
1028 # check for election
1029 topics = []
1030 for i in range( 14 ):
1031 topics.append( "intent-partition-" + str( i ) )
1032 # FIXME: this should only be after we start the app
1033 topics.append( "org.onosproject.election" )
1034 main.log.debug( topics )
1035 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1036 for topic in topics:
1037 if topic not in ONOStopics:
1038 main.log.error( "Error: " + topic +
1039 " not in leaders" )
1040 missing = True
1041 else:
1042 main.log.error( "leaders() returned None" )
1043 except ( ValueError, TypeError ):
1044 main.log.exception( "Error parsing leaders" )
1045 main.log.error( repr( leaders ) )
1046 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001047 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001048 response = node.leaders( jsonFormat=False)
1049 main.log.warn( str( node.name ) + " leaders output: \n" +
1050 str( response ) )
1051
1052 partitions = main.ONOScli1.partitions()
1053 try:
1054 if partitions :
1055 parsedPartitions = json.loads( partitions )
1056 main.log.warn( json.dumps( parsedPartitions,
1057 sort_keys=True,
1058 indent=4,
1059 separators=( ',', ': ' ) ) )
1060 # TODO check for a leader in all paritions
1061 # TODO check for consistency among nodes
1062 else:
1063 main.log.error( "partitions() returned None" )
1064 except ( ValueError, TypeError ):
1065 main.log.exception( "Error parsing partitions" )
1066 main.log.error( repr( partitions ) )
1067 pendingMap = main.ONOScli1.pendingMap()
1068 try:
1069 if pendingMap :
1070 parsedPending = json.loads( pendingMap )
1071 main.log.warn( json.dumps( parsedPending,
1072 sort_keys=True,
1073 indent=4,
1074 separators=( ',', ': ' ) ) )
1075 # TODO check something here?
1076 else:
1077 main.log.error( "pendingMap() returned None" )
1078 except ( ValueError, TypeError ):
1079 main.log.exception( "Error parsing pending map" )
1080 main.log.error( repr( pendingMap ) )
1081 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001082 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001083 main.step( "Wait a minute then ping again" )
1084 # the wait is above
1085 PingResult = main.TRUE
1086 for i in range( 8, 18 ):
1087 ping = main.Mininet1.pingHost( src="h" + str( i ),
1088 target="h" + str( i + 10 ) )
1089 PingResult = PingResult and ping
1090 if ping == main.FALSE:
1091 main.log.warn( "Ping failed between h" + str( i ) +
1092 " and h" + str( i + 10 ) )
1093 elif ping == main.TRUE:
1094 main.log.info( "Ping test passed!" )
1095 # Don't set PingResult or you'd override failures
1096 if PingResult == main.FALSE:
1097 main.log.error(
1098 "Intents have not been installed correctly, pings failed." )
1099 # TODO: pretty print
1100 main.log.warn( "ONOS1 intents: " )
1101 try:
1102 tmpIntents = main.ONOScli1.intents()
1103 main.log.warn( json.dumps( json.loads( tmpIntents ),
1104 sort_keys=True,
1105 indent=4,
1106 separators=( ',', ': ' ) ) )
1107 except ( ValueError, TypeError ):
1108 main.log.warn( repr( tmpIntents ) )
1109 utilities.assert_equals(
1110 expect=main.TRUE,
1111 actual=PingResult,
1112 onpass="Intents have been installed correctly and pings work",
1113 onfail="Intents have not been installed correctly, pings failed." )
1114
1115 def CASE5( self, main ):
1116 """
1117 Reading state of ONOS
1118 """
1119 import json
1120 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001121 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001122 assert main, "main not defined"
1123 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001124 assert main.CLIs, "main.CLIs not defined"
1125 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001126
1127 main.case( "Setting up and gathering data for current state" )
1128 # The general idea for this test case is to pull the state of
1129 # ( intents,flows, topology,... ) from each ONOS node
1130 # We can then compare them with each other and also with past states
1131
1132 main.step( "Check that each switch has a master" )
1133 global mastershipState
1134 mastershipState = '[]'
1135
1136 # Assert that each device has a master
1137 rolesNotNull = main.TRUE
1138 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001139 for i in range( main.numCtrls ):
1140 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001141 name="rolesNotNull-" + str( i ),
1142 args=[] )
1143 threads.append( t )
1144 t.start()
1145
1146 for t in threads:
1147 t.join()
1148 rolesNotNull = rolesNotNull and t.result
1149 utilities.assert_equals(
1150 expect=main.TRUE,
1151 actual=rolesNotNull,
1152 onpass="Each device has a master",
1153 onfail="Some devices don't have a master assigned" )
1154
1155 main.step( "Get the Mastership of each switch from each controller" )
1156 ONOSMastership = []
1157 mastershipCheck = main.FALSE
1158 consistentMastership = True
1159 rolesResults = True
1160 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001161 for i in range( main.numCtrls ):
1162 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001163 name="roles-" + str( i ),
1164 args=[] )
1165 threads.append( t )
1166 t.start()
1167
1168 for t in threads:
1169 t.join()
1170 ONOSMastership.append( t.result )
1171
Jon Halle1a3b752015-07-22 13:02:46 -07001172 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001173 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1174 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1175 " roles" )
1176 main.log.warn(
1177 "ONOS" + str( i + 1 ) + " mastership response: " +
1178 repr( ONOSMastership[i] ) )
1179 rolesResults = False
1180 utilities.assert_equals(
1181 expect=True,
1182 actual=rolesResults,
1183 onpass="No error in reading roles output",
1184 onfail="Error in reading roles from ONOS" )
1185
1186 main.step( "Check for consistency in roles from each controller" )
1187 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1188 main.log.info(
1189 "Switch roles are consistent across all ONOS nodes" )
1190 else:
1191 consistentMastership = False
1192 utilities.assert_equals(
1193 expect=True,
1194 actual=consistentMastership,
1195 onpass="Switch roles are consistent across all ONOS nodes",
1196 onfail="ONOS nodes have different views of switch roles" )
1197
1198 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001199 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001200 try:
1201 main.log.warn(
1202 "ONOS" + str( i + 1 ) + " roles: ",
1203 json.dumps(
1204 json.loads( ONOSMastership[ i ] ),
1205 sort_keys=True,
1206 indent=4,
1207 separators=( ',', ': ' ) ) )
1208 except ( ValueError, TypeError ):
1209 main.log.warn( repr( ONOSMastership[ i ] ) )
1210 elif rolesResults and consistentMastership:
1211 mastershipCheck = main.TRUE
1212 mastershipState = ONOSMastership[ 0 ]
1213
1214 main.step( "Get the intents from each controller" )
1215 global intentState
1216 intentState = []
1217 ONOSIntents = []
1218 intentCheck = main.FALSE
1219 consistentIntents = True
1220 intentsResults = True
1221 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001222 for i in range( main.numCtrls ):
1223 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001224 name="intents-" + str( i ),
1225 args=[],
1226 kwargs={ 'jsonFormat': True } )
1227 threads.append( t )
1228 t.start()
1229
1230 for t in threads:
1231 t.join()
1232 ONOSIntents.append( t.result )
1233
Jon Halle1a3b752015-07-22 13:02:46 -07001234 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001235 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1236 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1237 " intents" )
1238 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1239 repr( ONOSIntents[ i ] ) )
1240 intentsResults = False
1241 utilities.assert_equals(
1242 expect=True,
1243 actual=intentsResults,
1244 onpass="No error in reading intents output",
1245 onfail="Error in reading intents from ONOS" )
1246
1247 main.step( "Check for consistency in Intents from each controller" )
1248 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1249 main.log.info( "Intents are consistent across all ONOS " +
1250 "nodes" )
1251 else:
1252 consistentIntents = False
1253 main.log.error( "Intents not consistent" )
1254 utilities.assert_equals(
1255 expect=True,
1256 actual=consistentIntents,
1257 onpass="Intents are consistent across all ONOS nodes",
1258 onfail="ONOS nodes have different views of intents" )
1259
1260 if intentsResults:
1261 # Try to make it easy to figure out what is happening
1262 #
1263 # Intent ONOS1 ONOS2 ...
1264 # 0x01 INSTALLED INSTALLING
1265 # ... ... ...
1266 # ... ... ...
1267 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001268 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001269 title += " " * 10 + "ONOS" + str( n + 1 )
1270 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001271 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001272 keys = []
1273 try:
1274 # Get the set of all intent keys
1275 for nodeStr in ONOSIntents:
1276 node = json.loads( nodeStr )
1277 for intent in node:
1278 keys.append( intent.get( 'id' ) )
1279 keys = set( keys )
1280 # For each intent key, print the state on each node
1281 for key in keys:
1282 row = "%-13s" % key
1283 for nodeStr in ONOSIntents:
1284 node = json.loads( nodeStr )
1285 for intent in node:
1286 if intent.get( 'id', "Error" ) == key:
1287 row += "%-15s" % intent.get( 'state' )
1288 main.log.warn( row )
1289 # End of intent state table
1290 except ValueError as e:
1291 main.log.exception( e )
1292 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1293
1294 if intentsResults and not consistentIntents:
1295 # print the json objects
1296 n = len(ONOSIntents)
1297 main.log.debug( "ONOS" + str( n ) + " intents: " )
1298 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1299 sort_keys=True,
1300 indent=4,
1301 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001302 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001303 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1304 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1305 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1306 sort_keys=True,
1307 indent=4,
1308 separators=( ',', ': ' ) ) )
1309 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001310 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001311 str( n ) + " intents" )
1312 elif intentsResults and consistentIntents:
1313 intentCheck = main.TRUE
1314 intentState = ONOSIntents[ 0 ]
1315
1316 main.step( "Get the flows from each controller" )
1317 global flowState
1318 flowState = []
1319 ONOSFlows = []
1320 ONOSFlowsJson = []
1321 flowCheck = main.FALSE
1322 consistentFlows = True
1323 flowsResults = True
1324 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001325 for i in range( main.numCtrls ):
1326 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001327 name="flows-" + str( i ),
1328 args=[],
1329 kwargs={ 'jsonFormat': True } )
1330 threads.append( t )
1331 t.start()
1332
1333 # NOTE: Flows command can take some time to run
1334 time.sleep(30)
1335 for t in threads:
1336 t.join()
1337 result = t.result
1338 ONOSFlows.append( result )
1339
Jon Halle1a3b752015-07-22 13:02:46 -07001340 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001341 num = str( i + 1 )
1342 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1343 main.log.error( "Error in getting ONOS" + num + " flows" )
1344 main.log.warn( "ONOS" + num + " flows response: " +
1345 repr( ONOSFlows[ i ] ) )
1346 flowsResults = False
1347 ONOSFlowsJson.append( None )
1348 else:
1349 try:
1350 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1351 except ( ValueError, TypeError ):
1352 # FIXME: change this to log.error?
1353 main.log.exception( "Error in parsing ONOS" + num +
1354 " response as json." )
1355 main.log.error( repr( ONOSFlows[ i ] ) )
1356 ONOSFlowsJson.append( None )
1357 flowsResults = False
1358 utilities.assert_equals(
1359 expect=True,
1360 actual=flowsResults,
1361 onpass="No error in reading flows output",
1362 onfail="Error in reading flows from ONOS" )
1363
1364 main.step( "Check for consistency in Flows from each controller" )
1365 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1366 if all( tmp ):
1367 main.log.info( "Flow count is consistent across all ONOS nodes" )
1368 else:
1369 consistentFlows = False
1370 utilities.assert_equals(
1371 expect=True,
1372 actual=consistentFlows,
1373 onpass="The flow count is consistent across all ONOS nodes",
1374 onfail="ONOS nodes have different flow counts" )
1375
1376 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001377 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001378 try:
1379 main.log.warn(
1380 "ONOS" + str( i + 1 ) + " flows: " +
1381 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1382 indent=4, separators=( ',', ': ' ) ) )
1383 except ( ValueError, TypeError ):
1384 main.log.warn(
1385 "ONOS" + str( i + 1 ) + " flows: " +
1386 repr( ONOSFlows[ i ] ) )
1387 elif flowsResults and consistentFlows:
1388 flowCheck = main.TRUE
1389 flowState = ONOSFlows[ 0 ]
1390
1391 main.step( "Get the OF Table entries" )
1392 global flows
1393 flows = []
1394 for i in range( 1, 29 ):
Jon Hall9043c902015-07-30 14:23:44 -07001395 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001396 if flowCheck == main.FALSE:
1397 for table in flows:
1398 main.log.warn( table )
1399 # TODO: Compare switch flow tables with ONOS flow tables
1400
1401 main.step( "Start continuous pings" )
1402 main.Mininet2.pingLong(
1403 src=main.params[ 'PING' ][ 'source1' ],
1404 target=main.params[ 'PING' ][ 'target1' ],
1405 pingTime=500 )
1406 main.Mininet2.pingLong(
1407 src=main.params[ 'PING' ][ 'source2' ],
1408 target=main.params[ 'PING' ][ 'target2' ],
1409 pingTime=500 )
1410 main.Mininet2.pingLong(
1411 src=main.params[ 'PING' ][ 'source3' ],
1412 target=main.params[ 'PING' ][ 'target3' ],
1413 pingTime=500 )
1414 main.Mininet2.pingLong(
1415 src=main.params[ 'PING' ][ 'source4' ],
1416 target=main.params[ 'PING' ][ 'target4' ],
1417 pingTime=500 )
1418 main.Mininet2.pingLong(
1419 src=main.params[ 'PING' ][ 'source5' ],
1420 target=main.params[ 'PING' ][ 'target5' ],
1421 pingTime=500 )
1422 main.Mininet2.pingLong(
1423 src=main.params[ 'PING' ][ 'source6' ],
1424 target=main.params[ 'PING' ][ 'target6' ],
1425 pingTime=500 )
1426 main.Mininet2.pingLong(
1427 src=main.params[ 'PING' ][ 'source7' ],
1428 target=main.params[ 'PING' ][ 'target7' ],
1429 pingTime=500 )
1430 main.Mininet2.pingLong(
1431 src=main.params[ 'PING' ][ 'source8' ],
1432 target=main.params[ 'PING' ][ 'target8' ],
1433 pingTime=500 )
1434 main.Mininet2.pingLong(
1435 src=main.params[ 'PING' ][ 'source9' ],
1436 target=main.params[ 'PING' ][ 'target9' ],
1437 pingTime=500 )
1438 main.Mininet2.pingLong(
1439 src=main.params[ 'PING' ][ 'source10' ],
1440 target=main.params[ 'PING' ][ 'target10' ],
1441 pingTime=500 )
1442
1443 main.step( "Collecting topology information from ONOS" )
1444 devices = []
1445 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001446 for i in range( main.numCtrls ):
1447 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001448 name="devices-" + str( i ),
1449 args=[ ] )
1450 threads.append( t )
1451 t.start()
1452
1453 for t in threads:
1454 t.join()
1455 devices.append( t.result )
1456 hosts = []
1457 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001458 for i in range( main.numCtrls ):
1459 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001460 name="hosts-" + str( i ),
1461 args=[ ] )
1462 threads.append( t )
1463 t.start()
1464
1465 for t in threads:
1466 t.join()
1467 try:
1468 hosts.append( json.loads( t.result ) )
1469 except ( ValueError, TypeError ):
1470 # FIXME: better handling of this, print which node
1471 # Maybe use thread name?
1472 main.log.exception( "Error parsing json output of hosts" )
1473 # FIXME: should this be an empty json object instead?
1474 hosts.append( None )
1475
1476 ports = []
1477 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001478 for i in range( main.numCtrls ):
1479 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001480 name="ports-" + str( i ),
1481 args=[ ] )
1482 threads.append( t )
1483 t.start()
1484
1485 for t in threads:
1486 t.join()
1487 ports.append( t.result )
1488 links = []
1489 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001490 for i in range( main.numCtrls ):
1491 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001492 name="links-" + str( i ),
1493 args=[ ] )
1494 threads.append( t )
1495 t.start()
1496
1497 for t in threads:
1498 t.join()
1499 links.append( t.result )
1500 clusters = []
1501 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001502 for i in range( main.numCtrls ):
1503 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001504 name="clusters-" + str( i ),
1505 args=[ ] )
1506 threads.append( t )
1507 t.start()
1508
1509 for t in threads:
1510 t.join()
1511 clusters.append( t.result )
1512 # Compare json objects for hosts and dataplane clusters
1513
1514 # hosts
1515 main.step( "Host view is consistent across ONOS nodes" )
1516 consistentHostsResult = main.TRUE
1517 for controller in range( len( hosts ) ):
1518 controllerStr = str( controller + 1 )
1519 if "Error" not in hosts[ controller ]:
1520 if hosts[ controller ] == hosts[ 0 ]:
1521 continue
1522 else: # hosts not consistent
1523 main.log.error( "hosts from ONOS" +
1524 controllerStr +
1525 " is inconsistent with ONOS1" )
1526 main.log.warn( repr( hosts[ controller ] ) )
1527 consistentHostsResult = main.FALSE
1528
1529 else:
1530 main.log.error( "Error in getting ONOS hosts from ONOS" +
1531 controllerStr )
1532 consistentHostsResult = main.FALSE
1533 main.log.warn( "ONOS" + controllerStr +
1534 " hosts response: " +
1535 repr( hosts[ controller ] ) )
1536 utilities.assert_equals(
1537 expect=main.TRUE,
1538 actual=consistentHostsResult,
1539 onpass="Hosts view is consistent across all ONOS nodes",
1540 onfail="ONOS nodes have different views of hosts" )
1541
1542 main.step( "Each host has an IP address" )
1543 ipResult = main.TRUE
1544 for controller in range( 0, len( hosts ) ):
1545 controllerStr = str( controller + 1 )
1546 for host in hosts[ controller ]:
1547 if not host.get( 'ipAddresses', [ ] ):
1548 main.log.error( "DEBUG:Error with host ips on controller" +
1549 controllerStr + ": " + str( host ) )
1550 ipResult = main.FALSE
1551 utilities.assert_equals(
1552 expect=main.TRUE,
1553 actual=ipResult,
1554 onpass="The ips of the hosts aren't empty",
1555 onfail="The ip of at least one host is missing" )
1556
1557 # Strongly connected clusters of devices
1558 main.step( "Cluster view is consistent across ONOS nodes" )
1559 consistentClustersResult = main.TRUE
1560 for controller in range( len( clusters ) ):
1561 controllerStr = str( controller + 1 )
1562 if "Error" not in clusters[ controller ]:
1563 if clusters[ controller ] == clusters[ 0 ]:
1564 continue
1565 else: # clusters not consistent
1566 main.log.error( "clusters from ONOS" + controllerStr +
1567 " is inconsistent with ONOS1" )
1568 consistentClustersResult = main.FALSE
1569
1570 else:
1571 main.log.error( "Error in getting dataplane clusters " +
1572 "from ONOS" + controllerStr )
1573 consistentClustersResult = main.FALSE
1574 main.log.warn( "ONOS" + controllerStr +
1575 " clusters response: " +
1576 repr( clusters[ controller ] ) )
1577 utilities.assert_equals(
1578 expect=main.TRUE,
1579 actual=consistentClustersResult,
1580 onpass="Clusters view is consistent across all ONOS nodes",
1581 onfail="ONOS nodes have different views of clusters" )
1582 # there should always only be one cluster
1583 main.step( "Cluster view correct across ONOS nodes" )
1584 try:
1585 numClusters = len( json.loads( clusters[ 0 ] ) )
1586 except ( ValueError, TypeError ):
1587 main.log.exception( "Error parsing clusters[0]: " +
1588 repr( clusters[ 0 ] ) )
1589 clusterResults = main.FALSE
1590 if numClusters == 1:
1591 clusterResults = main.TRUE
1592 utilities.assert_equals(
1593 expect=1,
1594 actual=numClusters,
1595 onpass="ONOS shows 1 SCC",
1596 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1597
1598 main.step( "Comparing ONOS topology to MN" )
1599 devicesResults = main.TRUE
1600 linksResults = main.TRUE
1601 hostsResults = main.TRUE
1602 mnSwitches = main.Mininet1.getSwitches()
1603 mnLinks = main.Mininet1.getLinks()
1604 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001605 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001606 controllerStr = str( controller + 1 )
1607 if devices[ controller ] and ports[ controller ] and\
1608 "Error" not in devices[ controller ] and\
1609 "Error" not in ports[ controller ]:
1610
1611 currentDevicesResult = main.Mininet1.compareSwitches(
1612 mnSwitches,
1613 json.loads( devices[ controller ] ),
1614 json.loads( ports[ controller ] ) )
1615 else:
1616 currentDevicesResult = main.FALSE
1617 utilities.assert_equals( expect=main.TRUE,
1618 actual=currentDevicesResult,
1619 onpass="ONOS" + controllerStr +
1620 " Switches view is correct",
1621 onfail="ONOS" + controllerStr +
1622 " Switches view is incorrect" )
1623 if links[ controller ] and "Error" not in links[ controller ]:
1624 currentLinksResult = main.Mininet1.compareLinks(
1625 mnSwitches, mnLinks,
1626 json.loads( links[ controller ] ) )
1627 else:
1628 currentLinksResult = main.FALSE
1629 utilities.assert_equals( expect=main.TRUE,
1630 actual=currentLinksResult,
1631 onpass="ONOS" + controllerStr +
1632 " links view is correct",
1633 onfail="ONOS" + controllerStr +
1634 " links view is incorrect" )
1635
1636 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1637 currentHostsResult = main.Mininet1.compareHosts(
1638 mnHosts,
1639 hosts[ controller ] )
1640 else:
1641 currentHostsResult = main.FALSE
1642 utilities.assert_equals( expect=main.TRUE,
1643 actual=currentHostsResult,
1644 onpass="ONOS" + controllerStr +
1645 " hosts exist in Mininet",
1646 onfail="ONOS" + controllerStr +
1647 " hosts don't match Mininet" )
1648
1649 devicesResults = devicesResults and currentDevicesResult
1650 linksResults = linksResults and currentLinksResult
1651 hostsResults = hostsResults and currentHostsResult
1652
1653 main.step( "Device information is correct" )
1654 utilities.assert_equals(
1655 expect=main.TRUE,
1656 actual=devicesResults,
1657 onpass="Device information is correct",
1658 onfail="Device information is incorrect" )
1659
1660 main.step( "Links are correct" )
1661 utilities.assert_equals(
1662 expect=main.TRUE,
1663 actual=linksResults,
1664 onpass="Link are correct",
1665 onfail="Links are incorrect" )
1666
1667 main.step( "Hosts are correct" )
1668 utilities.assert_equals(
1669 expect=main.TRUE,
1670 actual=hostsResults,
1671 onpass="Hosts are correct",
1672 onfail="Hosts are incorrect" )
1673
1674 def CASE6( self, main ):
1675 """
1676 The Failure case. Since this is the Sanity test, we do nothing.
1677 """
1678 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001679 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001680 assert main, "main not defined"
1681 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001682 assert main.CLIs, "main.CLIs not defined"
1683 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001684 main.case( "Wait 60 seconds instead of inducing a failure" )
1685 time.sleep( 60 )
1686 utilities.assert_equals(
1687 expect=main.TRUE,
1688 actual=main.TRUE,
1689 onpass="Sleeping 60 seconds",
1690 onfail="Something is terribly wrong with my math" )
1691
1692 def CASE7( self, main ):
1693 """
1694 Check state after ONOS failure
1695 """
1696 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001697 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001698 assert main, "main not defined"
1699 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001700 assert main.CLIs, "main.CLIs not defined"
1701 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001702 main.case( "Running ONOS Constant State Tests" )
1703
1704 main.step( "Check that each switch has a master" )
1705 # Assert that each device has a master
1706 rolesNotNull = main.TRUE
1707 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001708 for i in range( main.numCtrls ):
1709 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001710 name="rolesNotNull-" + str( i ),
1711 args=[ ] )
1712 threads.append( t )
1713 t.start()
1714
1715 for t in threads:
1716 t.join()
1717 rolesNotNull = rolesNotNull and t.result
1718 utilities.assert_equals(
1719 expect=main.TRUE,
1720 actual=rolesNotNull,
1721 onpass="Each device has a master",
1722 onfail="Some devices don't have a master assigned" )
1723
1724 main.step( "Read device roles from ONOS" )
1725 ONOSMastership = []
1726 mastershipCheck = main.FALSE
1727 consistentMastership = True
1728 rolesResults = True
1729 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001730 for i in range( main.numCtrls ):
1731 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001732 name="roles-" + str( i ),
1733 args=[] )
1734 threads.append( t )
1735 t.start()
1736
1737 for t in threads:
1738 t.join()
1739 ONOSMastership.append( t.result )
1740
Jon Halle1a3b752015-07-22 13:02:46 -07001741 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001742 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1743 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1744 " roles" )
1745 main.log.warn(
1746 "ONOS" + str( i + 1 ) + " mastership response: " +
1747 repr( ONOSMastership[i] ) )
1748 rolesResults = False
1749 utilities.assert_equals(
1750 expect=True,
1751 actual=rolesResults,
1752 onpass="No error in reading roles output",
1753 onfail="Error in reading roles from ONOS" )
1754
1755 main.step( "Check for consistency in roles from each controller" )
1756 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1757 main.log.info(
1758 "Switch roles are consistent across all ONOS nodes" )
1759 else:
1760 consistentMastership = False
1761 utilities.assert_equals(
1762 expect=True,
1763 actual=consistentMastership,
1764 onpass="Switch roles are consistent across all ONOS nodes",
1765 onfail="ONOS nodes have different views of switch roles" )
1766
1767 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001768 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001769 main.log.warn(
1770 "ONOS" + str( i + 1 ) + " roles: ",
1771 json.dumps(
1772 json.loads( ONOSMastership[ i ] ),
1773 sort_keys=True,
1774 indent=4,
1775 separators=( ',', ': ' ) ) )
1776 elif rolesResults and not consistentMastership:
1777 mastershipCheck = main.TRUE
1778
1779 description2 = "Compare switch roles from before failure"
1780 main.step( description2 )
1781 try:
1782 currentJson = json.loads( ONOSMastership[0] )
1783 oldJson = json.loads( mastershipState )
1784 except ( ValueError, TypeError ):
1785 main.log.exception( "Something is wrong with parsing " +
1786 "ONOSMastership[0] or mastershipState" )
1787 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1788 main.log.error( "mastershipState" + repr( mastershipState ) )
1789 main.cleanup()
1790 main.exit()
1791 mastershipCheck = main.TRUE
1792 for i in range( 1, 29 ):
1793 switchDPID = str(
1794 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1795 current = [ switch[ 'master' ] for switch in currentJson
1796 if switchDPID in switch[ 'id' ] ]
1797 old = [ switch[ 'master' ] for switch in oldJson
1798 if switchDPID in switch[ 'id' ] ]
1799 if current == old:
1800 mastershipCheck = mastershipCheck and main.TRUE
1801 else:
1802 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1803 mastershipCheck = main.FALSE
1804 utilities.assert_equals(
1805 expect=main.TRUE,
1806 actual=mastershipCheck,
1807 onpass="Mastership of Switches was not changed",
1808 onfail="Mastership of some switches changed" )
1809 mastershipCheck = mastershipCheck and consistentMastership
1810
1811 main.step( "Get the intents and compare across all nodes" )
1812 ONOSIntents = []
1813 intentCheck = main.FALSE
1814 consistentIntents = True
1815 intentsResults = True
1816 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001817 for i in range( main.numCtrls ):
1818 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001819 name="intents-" + str( i ),
1820 args=[],
1821 kwargs={ 'jsonFormat': True } )
1822 threads.append( t )
1823 t.start()
1824
1825 for t in threads:
1826 t.join()
1827 ONOSIntents.append( t.result )
1828
Jon Halle1a3b752015-07-22 13:02:46 -07001829 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001830 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1831 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1832 " intents" )
1833 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1834 repr( ONOSIntents[ i ] ) )
1835 intentsResults = False
1836 utilities.assert_equals(
1837 expect=True,
1838 actual=intentsResults,
1839 onpass="No error in reading intents output",
1840 onfail="Error in reading intents from ONOS" )
1841
1842 main.step( "Check for consistency in Intents from each controller" )
1843 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1844 main.log.info( "Intents are consistent across all ONOS " +
1845 "nodes" )
1846 else:
1847 consistentIntents = False
1848
1849 # Try to make it easy to figure out what is happening
1850 #
1851 # Intent ONOS1 ONOS2 ...
1852 # 0x01 INSTALLED INSTALLING
1853 # ... ... ...
1854 # ... ... ...
1855 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001856 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001857 title += " " * 10 + "ONOS" + str( n + 1 )
1858 main.log.warn( title )
1859 # get all intent keys in the cluster
1860 keys = []
1861 for nodeStr in ONOSIntents:
1862 node = json.loads( nodeStr )
1863 for intent in node:
1864 keys.append( intent.get( 'id' ) )
1865 keys = set( keys )
1866 for key in keys:
1867 row = "%-13s" % key
1868 for nodeStr in ONOSIntents:
1869 node = json.loads( nodeStr )
1870 for intent in node:
1871 if intent.get( 'id' ) == key:
1872 row += "%-15s" % intent.get( 'state' )
1873 main.log.warn( row )
1874 # End table view
1875
1876 utilities.assert_equals(
1877 expect=True,
1878 actual=consistentIntents,
1879 onpass="Intents are consistent across all ONOS nodes",
1880 onfail="ONOS nodes have different views of intents" )
1881 intentStates = []
1882 for node in ONOSIntents: # Iter through ONOS nodes
1883 nodeStates = []
1884 # Iter through intents of a node
1885 try:
1886 for intent in json.loads( node ):
1887 nodeStates.append( intent[ 'state' ] )
1888 except ( ValueError, TypeError ):
1889 main.log.exception( "Error in parsing intents" )
1890 main.log.error( repr( node ) )
1891 intentStates.append( nodeStates )
1892 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1893 main.log.info( dict( out ) )
1894
1895 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001896 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001897 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1898 main.log.warn( json.dumps(
1899 json.loads( ONOSIntents[ i ] ),
1900 sort_keys=True,
1901 indent=4,
1902 separators=( ',', ': ' ) ) )
1903 elif intentsResults and consistentIntents:
1904 intentCheck = main.TRUE
1905
1906 # NOTE: Store has no durability, so intents are lost across system
1907 # restarts
1908 main.step( "Compare current intents with intents before the failure" )
1909 # NOTE: this requires case 5 to pass for intentState to be set.
1910 # maybe we should stop the test if that fails?
1911 sameIntents = main.FALSE
1912 if intentState and intentState == ONOSIntents[ 0 ]:
1913 sameIntents = main.TRUE
1914 main.log.info( "Intents are consistent with before failure" )
1915 # TODO: possibly the states have changed? we may need to figure out
1916 # what the acceptable states are
1917 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1918 sameIntents = main.TRUE
1919 try:
1920 before = json.loads( intentState )
1921 after = json.loads( ONOSIntents[ 0 ] )
1922 for intent in before:
1923 if intent not in after:
1924 sameIntents = main.FALSE
1925 main.log.debug( "Intent is not currently in ONOS " +
1926 "(at least in the same form):" )
1927 main.log.debug( json.dumps( intent ) )
1928 except ( ValueError, TypeError ):
1929 main.log.exception( "Exception printing intents" )
1930 main.log.debug( repr( ONOSIntents[0] ) )
1931 main.log.debug( repr( intentState ) )
1932 if sameIntents == main.FALSE:
1933 try:
1934 main.log.debug( "ONOS intents before: " )
1935 main.log.debug( json.dumps( json.loads( intentState ),
1936 sort_keys=True, indent=4,
1937 separators=( ',', ': ' ) ) )
1938 main.log.debug( "Current ONOS intents: " )
1939 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1940 sort_keys=True, indent=4,
1941 separators=( ',', ': ' ) ) )
1942 except ( ValueError, TypeError ):
1943 main.log.exception( "Exception printing intents" )
1944 main.log.debug( repr( ONOSIntents[0] ) )
1945 main.log.debug( repr( intentState ) )
1946 utilities.assert_equals(
1947 expect=main.TRUE,
1948 actual=sameIntents,
1949 onpass="Intents are consistent with before failure",
1950 onfail="The Intents changed during failure" )
1951 intentCheck = intentCheck and sameIntents
1952
1953 main.step( "Get the OF Table entries and compare to before " +
1954 "component failure" )
1955 FlowTables = main.TRUE
1956 flows2 = []
1957 for i in range( 28 ):
1958 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall9043c902015-07-30 14:23:44 -07001959 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001960 flows2.append( tmpFlows )
Jon Hall9043c902015-07-30 14:23:44 -07001961 tempResult = main.Mininet1.flowComp(
Jon Hall5cf14d52015-07-16 12:15:19 -07001962 flow1=flows[ i ],
1963 flow2=tmpFlows )
1964 FlowTables = FlowTables and tempResult
1965 if FlowTables == main.FALSE:
1966 main.log.info( "Differences in flow table for switch: s" +
1967 str( i + 1 ) )
1968 utilities.assert_equals(
1969 expect=main.TRUE,
1970 actual=FlowTables,
1971 onpass="No changes were found in the flow tables",
1972 onfail="Changes were found in the flow tables" )
1973
1974 main.Mininet2.pingLongKill()
1975 '''
1976 main.step( "Check the continuous pings to ensure that no packets " +
1977 "were dropped during component failure" )
1978 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1979 main.params[ 'TESTONIP' ] )
1980 LossInPings = main.FALSE
1981 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1982 for i in range( 8, 18 ):
1983 main.log.info(
1984 "Checking for a loss in pings along flow from s" +
1985 str( i ) )
1986 LossInPings = main.Mininet2.checkForLoss(
1987 "/tmp/ping.h" +
1988 str( i ) ) or LossInPings
1989 if LossInPings == main.TRUE:
1990 main.log.info( "Loss in ping detected" )
1991 elif LossInPings == main.ERROR:
1992 main.log.info( "There are multiple mininet process running" )
1993 elif LossInPings == main.FALSE:
1994 main.log.info( "No Loss in the pings" )
1995 main.log.info( "No loss of dataplane connectivity" )
1996 utilities.assert_equals(
1997 expect=main.FALSE,
1998 actual=LossInPings,
1999 onpass="No Loss of connectivity",
2000 onfail="Loss of dataplane connectivity detected" )
2001 '''
2002
2003 main.step( "Leadership Election is still functional" )
2004 # Test of LeadershipElection
2005 # NOTE: this only works for the sanity test. In case of failures,
2006 # leader will likely change
Jon Halle1a3b752015-07-22 13:02:46 -07002007 leader = main.nodes[ 0 ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002008 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002009 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002010 leaderN = cli.electionTestLeader()
2011 # verify leader is ONOS1
2012 if leaderN == leader:
2013 # all is well
2014 # NOTE: In failure scenario, this could be a new node, maybe
2015 # check != ONOS1
2016 pass
2017 elif leaderN == main.FALSE:
2018 # error in response
2019 main.log.error( "Something is wrong with " +
2020 "electionTestLeader function, check the" +
2021 " error logs" )
2022 leaderResult = main.FALSE
2023 elif leader != leaderN:
2024 leaderResult = main.FALSE
2025 main.log.error( cli.name + " sees " + str( leaderN ) +
2026 " as the leader of the election app. " +
2027 "Leader should be " + str( leader ) )
2028 utilities.assert_equals(
2029 expect=main.TRUE,
2030 actual=leaderResult,
2031 onpass="Leadership election passed",
2032 onfail="Something went wrong with Leadership election" )
2033
2034 def CASE8( self, main ):
2035 """
2036 Compare topo
2037 """
2038 import json
2039 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002040 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002041 assert main, "main not defined"
2042 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002043 assert main.CLIs, "main.CLIs not defined"
2044 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002045
2046 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002047 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002048 " and ONOS"
2049
2050 main.step( "Comparing ONOS topology to MN" )
2051 devicesResults = main.TRUE
2052 linksResults = main.TRUE
2053 hostsResults = main.TRUE
2054 hostAttachmentResults = True
2055 topoResult = main.FALSE
2056 elapsed = 0
2057 count = 0
2058 main.step( "Collecting topology information from ONOS" )
2059 startTime = time.time()
2060 # Give time for Gossip to work
2061 while topoResult == main.FALSE and elapsed < 60:
2062 count += 1
2063 cliStart = time.time()
2064 devices = []
2065 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002066 for i in range( main.numCtrls ):
2067 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002068 name="devices-" + str( i ),
2069 args=[ ] )
2070 threads.append( t )
2071 t.start()
2072
2073 for t in threads:
2074 t.join()
2075 devices.append( t.result )
2076 hosts = []
2077 ipResult = main.TRUE
2078 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002079 for i in range( main.numCtrls ):
2080 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07002081 name="hosts-" + str( i ),
2082 args=[ ] )
2083 threads.append( t )
2084 t.start()
2085
2086 for t in threads:
2087 t.join()
2088 try:
2089 hosts.append( json.loads( t.result ) )
2090 except ( ValueError, TypeError ):
2091 main.log.exception( "Error parsing hosts results" )
2092 main.log.error( repr( t.result ) )
2093 for controller in range( 0, len( hosts ) ):
2094 controllerStr = str( controller + 1 )
2095 for host in hosts[ controller ]:
2096 if host is None or host.get( 'ipAddresses', [] ) == []:
2097 main.log.error(
2098 "DEBUG:Error with host ipAddresses on controller" +
2099 controllerStr + ": " + str( host ) )
2100 ipResult = main.FALSE
2101 ports = []
2102 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002103 for i in range( main.numCtrls ):
2104 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002105 name="ports-" + str( i ),
2106 args=[ ] )
2107 threads.append( t )
2108 t.start()
2109
2110 for t in threads:
2111 t.join()
2112 ports.append( t.result )
2113 links = []
2114 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002115 for i in range( main.numCtrls ):
2116 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002117 name="links-" + str( i ),
2118 args=[ ] )
2119 threads.append( t )
2120 t.start()
2121
2122 for t in threads:
2123 t.join()
2124 links.append( t.result )
2125 clusters = []
2126 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002127 for i in range( main.numCtrls ):
2128 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002129 name="clusters-" + str( i ),
2130 args=[ ] )
2131 threads.append( t )
2132 t.start()
2133
2134 for t in threads:
2135 t.join()
2136 clusters.append( t.result )
2137
2138 elapsed = time.time() - startTime
2139 cliTime = time.time() - cliStart
2140 print "Elapsed time: " + str( elapsed )
2141 print "CLI time: " + str( cliTime )
2142
2143 mnSwitches = main.Mininet1.getSwitches()
2144 mnLinks = main.Mininet1.getLinks()
2145 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002146 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002147 controllerStr = str( controller + 1 )
2148 if devices[ controller ] and ports[ controller ] and\
2149 "Error" not in devices[ controller ] and\
2150 "Error" not in ports[ controller ]:
2151
2152 currentDevicesResult = main.Mininet1.compareSwitches(
2153 mnSwitches,
2154 json.loads( devices[ controller ] ),
2155 json.loads( ports[ controller ] ) )
2156 else:
2157 currentDevicesResult = main.FALSE
2158 utilities.assert_equals( expect=main.TRUE,
2159 actual=currentDevicesResult,
2160 onpass="ONOS" + controllerStr +
2161 " Switches view is correct",
2162 onfail="ONOS" + controllerStr +
2163 " Switches view is incorrect" )
2164
2165 if links[ controller ] and "Error" not in links[ controller ]:
2166 currentLinksResult = main.Mininet1.compareLinks(
2167 mnSwitches, mnLinks,
2168 json.loads( links[ controller ] ) )
2169 else:
2170 currentLinksResult = main.FALSE
2171 utilities.assert_equals( expect=main.TRUE,
2172 actual=currentLinksResult,
2173 onpass="ONOS" + controllerStr +
2174 " links view is correct",
2175 onfail="ONOS" + controllerStr +
2176 " links view is incorrect" )
2177
2178 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2179 currentHostsResult = main.Mininet1.compareHosts(
2180 mnHosts,
2181 hosts[ controller ] )
2182 else:
2183 currentHostsResult = main.FALSE
2184 utilities.assert_equals( expect=main.TRUE,
2185 actual=currentHostsResult,
2186 onpass="ONOS" + controllerStr +
2187 " hosts exist in Mininet",
2188 onfail="ONOS" + controllerStr +
2189 " hosts don't match Mininet" )
2190 # CHECKING HOST ATTACHMENT POINTS
2191 hostAttachment = True
2192 zeroHosts = False
2193 # FIXME: topo-HA/obelisk specific mappings:
2194 # key is mac and value is dpid
2195 mappings = {}
2196 for i in range( 1, 29 ): # hosts 1 through 28
2197 # set up correct variables:
2198 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2199 if i == 1:
2200 deviceId = "1000".zfill(16)
2201 elif i == 2:
2202 deviceId = "2000".zfill(16)
2203 elif i == 3:
2204 deviceId = "3000".zfill(16)
2205 elif i == 4:
2206 deviceId = "3004".zfill(16)
2207 elif i == 5:
2208 deviceId = "5000".zfill(16)
2209 elif i == 6:
2210 deviceId = "6000".zfill(16)
2211 elif i == 7:
2212 deviceId = "6007".zfill(16)
2213 elif i >= 8 and i <= 17:
2214 dpid = '3' + str( i ).zfill( 3 )
2215 deviceId = dpid.zfill(16)
2216 elif i >= 18 and i <= 27:
2217 dpid = '6' + str( i ).zfill( 3 )
2218 deviceId = dpid.zfill(16)
2219 elif i == 28:
2220 deviceId = "2800".zfill(16)
2221 mappings[ macId ] = deviceId
2222 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2223 if hosts[ controller ] == []:
2224 main.log.warn( "There are no hosts discovered" )
2225 zeroHosts = True
2226 else:
2227 for host in hosts[ controller ]:
2228 mac = None
2229 location = None
2230 device = None
2231 port = None
2232 try:
2233 mac = host.get( 'mac' )
2234 assert mac, "mac field could not be found for this host object"
2235
2236 location = host.get( 'location' )
2237 assert location, "location field could not be found for this host object"
2238
2239 # Trim the protocol identifier off deviceId
2240 device = str( location.get( 'elementId' ) ).split(':')[1]
2241 assert device, "elementId field could not be found for this host location object"
2242
2243 port = location.get( 'port' )
2244 assert port, "port field could not be found for this host location object"
2245
2246 # Now check if this matches where they should be
2247 if mac and device and port:
2248 if str( port ) != "1":
2249 main.log.error( "The attachment port is incorrect for " +
2250 "host " + str( mac ) +
2251 ". Expected: 1 Actual: " + str( port) )
2252 hostAttachment = False
2253 if device != mappings[ str( mac ) ]:
2254 main.log.error( "The attachment device is incorrect for " +
2255 "host " + str( mac ) +
2256 ". Expected: " + mappings[ str( mac ) ] +
2257 " Actual: " + device )
2258 hostAttachment = False
2259 else:
2260 hostAttachment = False
2261 except AssertionError:
2262 main.log.exception( "Json object not as expected" )
2263 main.log.error( repr( host ) )
2264 hostAttachment = False
2265 else:
2266 main.log.error( "No hosts json output or \"Error\"" +
2267 " in output. hosts = " +
2268 repr( hosts[ controller ] ) )
2269 if zeroHosts is False:
2270 hostAttachment = True
2271
2272 # END CHECKING HOST ATTACHMENT POINTS
2273 devicesResults = devicesResults and currentDevicesResult
2274 linksResults = linksResults and currentLinksResult
2275 hostsResults = hostsResults and currentHostsResult
2276 hostAttachmentResults = hostAttachmentResults and\
2277 hostAttachment
2278 topoResult = ( devicesResults and linksResults
2279 and hostsResults and ipResult and
2280 hostAttachmentResults )
2281
2282 # Compare json objects for hosts and dataplane clusters
2283
2284 # hosts
2285 main.step( "Hosts view is consistent across all ONOS nodes" )
2286 consistentHostsResult = main.TRUE
2287 for controller in range( len( hosts ) ):
2288 controllerStr = str( controller + 1 )
2289 if "Error" not in hosts[ controller ]:
2290 if hosts[ controller ] == hosts[ 0 ]:
2291 continue
2292 else: # hosts not consistent
2293 main.log.error( "hosts from ONOS" + controllerStr +
2294 " is inconsistent with ONOS1" )
2295 main.log.warn( repr( hosts[ controller ] ) )
2296 consistentHostsResult = main.FALSE
2297
2298 else:
2299 main.log.error( "Error in getting ONOS hosts from ONOS" +
2300 controllerStr )
2301 consistentHostsResult = main.FALSE
2302 main.log.warn( "ONOS" + controllerStr +
2303 " hosts response: " +
2304 repr( hosts[ controller ] ) )
2305 utilities.assert_equals(
2306 expect=main.TRUE,
2307 actual=consistentHostsResult,
2308 onpass="Hosts view is consistent across all ONOS nodes",
2309 onfail="ONOS nodes have different views of hosts" )
2310
2311 main.step( "Hosts information is correct" )
2312 hostsResults = hostsResults and ipResult
2313 utilities.assert_equals(
2314 expect=main.TRUE,
2315 actual=hostsResults,
2316 onpass="Host information is correct",
2317 onfail="Host information is incorrect" )
2318
2319 main.step( "Host attachment points to the network" )
2320 utilities.assert_equals(
2321 expect=True,
2322 actual=hostAttachmentResults,
2323 onpass="Hosts are correctly attached to the network",
2324 onfail="ONOS did not correctly attach hosts to the network" )
2325
2326 # Strongly connected clusters of devices
2327 main.step( "Clusters view is consistent across all ONOS nodes" )
2328 consistentClustersResult = main.TRUE
2329 for controller in range( len( clusters ) ):
2330 controllerStr = str( controller + 1 )
2331 if "Error" not in clusters[ controller ]:
2332 if clusters[ controller ] == clusters[ 0 ]:
2333 continue
2334 else: # clusters not consistent
2335 main.log.error( "clusters from ONOS" +
2336 controllerStr +
2337 " is inconsistent with ONOS1" )
2338 consistentClustersResult = main.FALSE
2339
2340 else:
2341 main.log.error( "Error in getting dataplane clusters " +
2342 "from ONOS" + controllerStr )
2343 consistentClustersResult = main.FALSE
2344 main.log.warn( "ONOS" + controllerStr +
2345 " clusters response: " +
2346 repr( clusters[ controller ] ) )
2347 utilities.assert_equals(
2348 expect=main.TRUE,
2349 actual=consistentClustersResult,
2350 onpass="Clusters view is consistent across all ONOS nodes",
2351 onfail="ONOS nodes have different views of clusters" )
2352
2353 main.step( "There is only one SCC" )
2354 # there should always only be one cluster
2355 try:
2356 numClusters = len( json.loads( clusters[ 0 ] ) )
2357 except ( ValueError, TypeError ):
2358 main.log.exception( "Error parsing clusters[0]: " +
2359 repr( clusters[0] ) )
2360 clusterResults = main.FALSE
2361 if numClusters == 1:
2362 clusterResults = main.TRUE
2363 utilities.assert_equals(
2364 expect=1,
2365 actual=numClusters,
2366 onpass="ONOS shows 1 SCC",
2367 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2368
2369 topoResult = ( devicesResults and linksResults
2370 and hostsResults and consistentHostsResult
2371 and consistentClustersResult and clusterResults
2372 and ipResult and hostAttachmentResults )
2373
2374 topoResult = topoResult and int( count <= 2 )
2375 note = "note it takes about " + str( int( cliTime ) ) + \
2376 " seconds for the test to make all the cli calls to fetch " +\
2377 "the topology from each ONOS instance"
2378 main.log.info(
2379 "Very crass estimate for topology discovery/convergence( " +
2380 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2381 str( count ) + " tries" )
2382
2383 main.step( "Device information is correct" )
2384 utilities.assert_equals(
2385 expect=main.TRUE,
2386 actual=devicesResults,
2387 onpass="Device information is correct",
2388 onfail="Device information is incorrect" )
2389
2390 main.step( "Links are correct" )
2391 utilities.assert_equals(
2392 expect=main.TRUE,
2393 actual=linksResults,
2394 onpass="Link are correct",
2395 onfail="Links are incorrect" )
2396
2397 main.step( "Hosts are correct" )
2398 utilities.assert_equals(
2399 expect=main.TRUE,
2400 actual=hostsResults,
2401 onpass="Hosts are correct",
2402 onfail="Hosts are incorrect" )
2403
2404 # FIXME: move this to an ONOS state case
2405 main.step( "Checking ONOS nodes" )
2406 nodesOutput = []
2407 nodeResults = main.TRUE
2408 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002409 for i in range( main.numCtrls ):
2410 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002411 name="nodes-" + str( i ),
2412 args=[ ] )
2413 threads.append( t )
2414 t.start()
2415
2416 for t in threads:
2417 t.join()
2418 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002419 ips = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002420 for i in nodesOutput:
2421 try:
2422 current = json.loads( i )
2423 for node in current:
2424 currentResult = main.FALSE
2425 if node['ip'] in ips: # node in nodes() output is in cell
2426 if node['state'] == 'ACTIVE':
2427 currentResult = main.TRUE
2428 else:
2429 main.log.error( "Error in ONOS node availability" )
2430 main.log.error(
2431 json.dumps( current,
2432 sort_keys=True,
2433 indent=4,
2434 separators=( ',', ': ' ) ) )
2435 break
2436 nodeResults = nodeResults and currentResult
2437 except ( ValueError, TypeError ):
2438 main.log.error( "Error parsing nodes output" )
2439 main.log.warn( repr( i ) )
2440 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2441 onpass="Nodes check successful",
2442 onfail="Nodes check NOT successful" )
2443
2444 def CASE9( self, main ):
2445 """
2446 Link s3-s28 down
2447 """
2448 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002449 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002450 assert main, "main not defined"
2451 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002452 assert main.CLIs, "main.CLIs not defined"
2453 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002454 # NOTE: You should probably run a topology check after this
2455
2456 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2457
2458 description = "Turn off a link to ensure that Link Discovery " +\
2459 "is working properly"
2460 main.case( description )
2461
2462 main.step( "Kill Link between s3 and s28" )
2463 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2464 main.log.info( "Waiting " + str( linkSleep ) +
2465 " seconds for link down to be discovered" )
2466 time.sleep( linkSleep )
2467 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2468 onpass="Link down successful",
2469 onfail="Failed to bring link down" )
2470 # TODO do some sort of check here
2471
2472 def CASE10( self, main ):
2473 """
2474 Link s3-s28 up
2475 """
2476 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002477 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002478 assert main, "main not defined"
2479 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002480 assert main.CLIs, "main.CLIs not defined"
2481 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002482 # NOTE: You should probably run a topology check after this
2483
2484 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2485
2486 description = "Restore a link to ensure that Link Discovery is " + \
2487 "working properly"
2488 main.case( description )
2489
2490 main.step( "Bring link between s3 and s28 back up" )
2491 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2492 main.log.info( "Waiting " + str( linkSleep ) +
2493 " seconds for link up to be discovered" )
2494 time.sleep( linkSleep )
2495 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2496 onpass="Link up successful",
2497 onfail="Failed to bring link up" )
2498 # TODO do some sort of check here
2499
2500 def CASE11( self, main ):
2501 """
2502 Switch Down
2503 """
2504 # NOTE: You should probably run a topology check after this
2505 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002506 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002507 assert main, "main not defined"
2508 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002509 assert main.CLIs, "main.CLIs not defined"
2510 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002511
2512 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2513
2514 description = "Killing a switch to ensure it is discovered correctly"
2515 main.case( description )
2516 switch = main.params[ 'kill' ][ 'switch' ]
2517 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2518
2519 # TODO: Make this switch parameterizable
2520 main.step( "Kill " + switch )
2521 main.log.info( "Deleting " + switch )
2522 main.Mininet1.delSwitch( switch )
2523 main.log.info( "Waiting " + str( switchSleep ) +
2524 " seconds for switch down to be discovered" )
2525 time.sleep( switchSleep )
2526 device = main.ONOScli1.getDevice( dpid=switchDPID )
2527 # Peek at the deleted switch
2528 main.log.warn( str( device ) )
2529 result = main.FALSE
2530 if device and device[ 'available' ] is False:
2531 result = main.TRUE
2532 utilities.assert_equals( expect=main.TRUE, actual=result,
2533 onpass="Kill switch successful",
2534 onfail="Failed to kill switch?" )
2535
2536 def CASE12( self, main ):
2537 """
2538 Switch Up
2539 """
2540 # NOTE: You should probably run a topology check after this
2541 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002542 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002543 assert main, "main not defined"
2544 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002545 assert main.CLIs, "main.CLIs not defined"
2546 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002547 assert ONOS1Port, "ONOS1Port not defined"
2548 assert ONOS2Port, "ONOS2Port not defined"
2549 assert ONOS3Port, "ONOS3Port not defined"
2550 assert ONOS4Port, "ONOS4Port not defined"
2551 assert ONOS5Port, "ONOS5Port not defined"
2552 assert ONOS6Port, "ONOS6Port not defined"
2553 assert ONOS7Port, "ONOS7Port not defined"
2554
2555 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2556 switch = main.params[ 'kill' ][ 'switch' ]
2557 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2558 links = main.params[ 'kill' ][ 'links' ].split()
2559 description = "Adding a switch to ensure it is discovered correctly"
2560 main.case( description )
2561
2562 main.step( "Add back " + switch )
2563 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2564 for peer in links:
2565 main.Mininet1.addLink( switch, peer )
2566 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002567 for i in range( main.numCtrls ):
2568 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002569 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2570 main.log.info( "Waiting " + str( switchSleep ) +
2571 " seconds for switch up to be discovered" )
2572 time.sleep( switchSleep )
2573 device = main.ONOScli1.getDevice( dpid=switchDPID )
2574 # Peek at the deleted switch
2575 main.log.warn( str( device ) )
2576 result = main.FALSE
2577 if device and device[ 'available' ]:
2578 result = main.TRUE
2579 utilities.assert_equals( expect=main.TRUE, actual=result,
2580 onpass="add switch successful",
2581 onfail="Failed to add switch?" )
2582
2583 def CASE13( self, main ):
2584 """
2585 Clean up
2586 """
2587 import os
2588 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002589 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002590 assert main, "main not defined"
2591 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002592 assert main.CLIs, "main.CLIs not defined"
2593 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002594
2595 # printing colors to terminal
2596 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2597 'blue': '\033[94m', 'green': '\033[92m',
2598 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2599 main.case( "Test Cleanup" )
2600 main.step( "Killing tcpdumps" )
2601 main.Mininet2.stopTcpdump()
2602
2603 testname = main.TEST
2604 if main.params[ 'BACKUP' ] == "True":
2605 main.step( "Copying MN pcap and ONOS log files to test station" )
2606 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2607 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
2608 # NOTE: MN Pcap file is being saved to ~/packet_captures
2609 # scp this file as MN and TestON aren't necessarily the same vm
2610 # FIXME: scp
2611 # mn files
2612 # TODO: Load these from params
2613 # NOTE: must end in /
2614 logFolder = "/opt/onos/log/"
2615 logFiles = [ "karaf.log", "karaf.log.1" ]
2616 # NOTE: must end in /
2617 dstDir = "~/packet_captures/"
2618 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002619 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002620 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2621 ":" + logFolder + f + " " +
2622 teststationUser + "@" +
2623 teststationIP + ":" +
2624 dstDir + str( testname ) +
2625 "-" + node.name + "-" + f )
2626 main.ONOSbench.handle.expect( "\$" )
2627
2628 # std*.log's
2629 # NOTE: must end in /
2630 logFolder = "/opt/onos/var/"
2631 logFiles = [ "stderr.log", "stdout.log" ]
2632 # NOTE: must end in /
2633 dstDir = "~/packet_captures/"
2634 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002635 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002636 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2637 ":" + logFolder + f + " " +
2638 teststationUser + "@" +
2639 teststationIP + ":" +
2640 dstDir + str( testname ) +
2641 "-" + node.name + "-" + f )
2642 main.ONOSbench.handle.expect( "\$" )
2643 # sleep so scp can finish
2644 time.sleep( 10 )
2645 main.step( "Packing and rotating pcap archives" )
2646 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
2647
2648 main.step( "Stopping Mininet" )
2649 mnResult = main.Mininet1.stopNet()
2650 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2651 onpass="Mininet stopped",
2652 onfail="MN cleanup NOT successful" )
2653
2654 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002655 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002656 print colors[ 'purple' ] + "Checking logs for errors on " + \
2657 node.name + ":" + colors[ 'end' ]
2658 print main.ONOSbench.checkLogs( node.ip_address )
2659
2660 try:
2661 timerLog = open( main.logdir + "/Timers.csv", 'w')
2662 # Overwrite with empty line and close
2663 labels = "Gossip Intents"
2664 data = str( gossipTime )
2665 timerLog.write( labels + "\n" + data )
2666 timerLog.close()
2667 except NameError, e:
2668 main.log.exception(e)
2669
2670 def CASE14( self, main ):
2671 """
2672 start election app on all onos nodes
2673 """
Jon Halle1a3b752015-07-22 13:02:46 -07002674 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002675 assert main, "main not defined"
2676 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002677 assert main.CLIs, "main.CLIs not defined"
2678 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002679
2680 main.case("Start Leadership Election app")
2681 main.step( "Install leadership election app" )
2682 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2683 utilities.assert_equals(
2684 expect=main.TRUE,
2685 actual=appResult,
2686 onpass="Election app installed",
2687 onfail="Something went wrong with installing Leadership election" )
2688
2689 main.step( "Run for election on each node" )
2690 leaderResult = main.TRUE
2691 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002692 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002693 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002694 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002695 leader = cli.electionTestLeader()
2696 if leader is None or leader == main.FALSE:
2697 main.log.error( cli.name + ": Leader for the election app " +
2698 "should be an ONOS node, instead got '" +
2699 str( leader ) + "'" )
2700 leaderResult = main.FALSE
2701 leaders.append( leader )
2702 utilities.assert_equals(
2703 expect=main.TRUE,
2704 actual=leaderResult,
2705 onpass="Successfully ran for leadership",
2706 onfail="Failed to run for leadership" )
2707
2708 main.step( "Check that each node shows the same leader" )
2709 sameLeader = main.TRUE
2710 if len( set( leaders ) ) != 1:
2711 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002712 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002713 str( leaders ) )
2714 utilities.assert_equals(
2715 expect=main.TRUE,
2716 actual=sameLeader,
2717 onpass="Leadership is consistent for the election topic",
2718 onfail="Nodes have different leaders" )
2719
2720 def CASE15( self, main ):
2721 """
2722 Check that Leadership Election is still functional
2723 """
2724 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002725 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002726 assert main, "main not defined"
2727 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002728 assert main.CLIs, "main.CLIs not defined"
2729 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002730
2731 leaderResult = main.TRUE
2732 description = "Check that Leadership Election is still functional"
2733 main.case( description )
2734
2735 main.step( "Check that each node shows the same leader" )
2736 sameLeader = main.TRUE
2737 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002738 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002739 leader = cli.electionTestLeader()
2740 leaders.append( leader )
2741 if len( set( leaders ) ) != 1:
2742 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002743 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002744 str( leaders ) )
2745 utilities.assert_equals(
2746 expect=main.TRUE,
2747 actual=sameLeader,
2748 onpass="Leadership is consistent for the election topic",
2749 onfail="Nodes have different leaders" )
2750
2751 main.step( "Find current leader and withdraw" )
2752 leader = main.ONOScli1.electionTestLeader()
2753 # do some sanity checking on leader before using it
2754 withdrawResult = main.FALSE
2755 if leader is None or leader == main.FALSE:
2756 main.log.error(
2757 "Leader for the election app should be an ONOS node," +
2758 "instead got '" + str( leader ) + "'" )
2759 leaderResult = main.FALSE
2760 oldLeader = None
Jon Halle1a3b752015-07-22 13:02:46 -07002761 for i in range( len( main.CLIs ) ):
2762 if leader == main.nodes[ i ].ip_address:
2763 oldLeader = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002764 break
2765 else: # FOR/ELSE statement
2766 main.log.error( "Leader election, could not find current leader" )
2767 if oldLeader:
2768 withdrawResult = oldLeader.electionTestWithdraw()
2769 utilities.assert_equals(
2770 expect=main.TRUE,
2771 actual=withdrawResult,
2772 onpass="Node was withdrawn from election",
2773 onfail="Node was not withdrawn from election" )
2774
2775 main.step( "Make sure new leader is elected" )
2776 # FIXME: use threads
2777 leaderList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002778 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002779 leaderN = cli.electionTestLeader()
2780 leaderList.append( leaderN )
2781 if leaderN == leader:
2782 main.log.error( cli.name + " still sees " + str( leader ) +
2783 " as leader after they withdrew" )
2784 leaderResult = main.FALSE
2785 elif leaderN == main.FALSE:
2786 # error in response
2787 # TODO: add check for "Command not found:" in the driver, this
2788 # means the app isn't loaded
2789 main.log.error( "Something is wrong with " +
2790 "electionTestLeader function, " +
2791 "check the error logs" )
2792 leaderResult = main.FALSE
2793 elif leaderN is None:
2794 # node may not have recieved the event yet
2795 time.sleep(7)
2796 leaderN = cli.electionTestLeader()
2797 leaderList.pop()
2798 leaderList.append( leaderN )
2799 consistentLeader = main.FALSE
2800 if len( set( leaderList ) ) == 1:
2801 main.log.info( "Each Election-app sees '" +
2802 str( leaderList[ 0 ] ) +
2803 "' as the leader" )
2804 consistentLeader = main.TRUE
2805 else:
2806 main.log.error(
2807 "Inconsistent responses for leader of Election-app:" )
2808 for n in range( len( leaderList ) ):
2809 main.log.error( "ONOS" + str( n + 1 ) + " response: " +
2810 str( leaderList[ n ] ) )
2811 leaderResult = leaderResult and consistentLeader
2812 utilities.assert_equals(
2813 expect=main.TRUE,
2814 actual=leaderResult,
2815 onpass="Leadership election passed",
2816 onfail="Something went wrong with Leadership election" )
2817
2818 main.step( "Run for election on old leader( just so everyone " +
2819 "is in the hat )" )
2820 if oldLeader:
2821 runResult = oldLeader.electionTestRun()
2822 else:
2823 runResult = main.FALSE
2824 utilities.assert_equals(
2825 expect=main.TRUE,
2826 actual=runResult,
2827 onpass="App re-ran for election",
2828 onfail="App failed to run for election" )
2829
2830 main.step( "Leader did not change when old leader re-ran" )
2831 afterRun = main.ONOScli1.electionTestLeader()
2832 # verify leader didn't just change
2833 if afterRun == leaderList[ 0 ]:
2834 afterResult = main.TRUE
2835 else:
2836 afterResult = main.FALSE
2837
2838 utilities.assert_equals(
2839 expect=main.TRUE,
2840 actual=afterResult,
2841 onpass="Old leader successfully re-ran for election",
2842 onfail="Something went wrong with Leadership election after " +
2843 "the old leader re-ran for election" )
2844
2845 def CASE16( self, main ):
2846 """
2847 Install Distributed Primitives app
2848 """
2849 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002850 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002851 assert main, "main not defined"
2852 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002853 assert main.CLIs, "main.CLIs not defined"
2854 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002855
2856 # Variables for the distributed primitives tests
2857 global pCounterName
2858 global iCounterName
2859 global pCounterValue
2860 global iCounterValue
2861 global onosSet
2862 global onosSetName
2863 pCounterName = "TestON-Partitions"
2864 iCounterName = "TestON-inMemory"
2865 pCounterValue = 0
2866 iCounterValue = 0
2867 onosSet = set([])
2868 onosSetName = "TestON-set"
2869
2870 description = "Install Primitives app"
2871 main.case( description )
2872 main.step( "Install Primitives app" )
2873 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002874 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002875 utilities.assert_equals( expect=main.TRUE,
2876 actual=appResults,
2877 onpass="Primitives app activated",
2878 onfail="Primitives app not activated" )
2879 time.sleep( 5 ) # To allow all nodes to activate
2880
2881 def CASE17( self, main ):
2882 """
2883 Check for basic functionality with distributed primitives
2884 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002885 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002886 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002887 assert main, "main not defined"
2888 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002889 assert main.CLIs, "main.CLIs not defined"
2890 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002891 assert pCounterName, "pCounterName not defined"
2892 assert iCounterName, "iCounterName not defined"
2893 assert onosSetName, "onosSetName not defined"
2894 # NOTE: assert fails if value is 0/None/Empty/False
2895 try:
2896 pCounterValue
2897 except NameError:
2898 main.log.error( "pCounterValue not defined, setting to 0" )
2899 pCounterValue = 0
2900 try:
2901 iCounterValue
2902 except NameError:
2903 main.log.error( "iCounterValue not defined, setting to 0" )
2904 iCounterValue = 0
2905 try:
2906 onosSet
2907 except NameError:
2908 main.log.error( "onosSet not defined, setting to empty Set" )
2909 onosSet = set([])
2910 # Variables for the distributed primitives tests. These are local only
2911 addValue = "a"
2912 addAllValue = "a b c d e f"
2913 retainValue = "c d e f"
2914
2915 description = "Check for basic functionality with distributed " +\
2916 "primitives"
2917 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07002918 main.caseExplanation = "Test the methods of the distributed " +\
2919 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07002920 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07002921 # Partitioned counters
2922 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002923 pCounters = []
2924 threads = []
2925 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07002926 for i in range( main.numCtrls ):
2927 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2928 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07002929 args=[ pCounterName ] )
2930 pCounterValue += 1
2931 addedPValues.append( pCounterValue )
2932 threads.append( t )
2933 t.start()
2934
2935 for t in threads:
2936 t.join()
2937 pCounters.append( t.result )
2938 # Check that counter incremented numController times
2939 pCounterResults = True
2940 for i in addedPValues:
2941 tmpResult = i in pCounters
2942 pCounterResults = pCounterResults and tmpResult
2943 if not tmpResult:
2944 main.log.error( str( i ) + " is not in partitioned "
2945 "counter incremented results" )
2946 utilities.assert_equals( expect=True,
2947 actual=pCounterResults,
2948 onpass="Default counter incremented",
2949 onfail="Error incrementing default" +
2950 " counter" )
2951
Jon Halle1a3b752015-07-22 13:02:46 -07002952 main.step( "Get then Increment a default counter on each node" )
2953 pCounters = []
2954 threads = []
2955 addedPValues = []
2956 for i in range( main.numCtrls ):
2957 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2958 name="counterGetAndAdd-" + str( i ),
2959 args=[ pCounterName ] )
2960 addedPValues.append( pCounterValue )
2961 pCounterValue += 1
2962 threads.append( t )
2963 t.start()
2964
2965 for t in threads:
2966 t.join()
2967 pCounters.append( t.result )
2968 # Check that counter incremented numController times
2969 pCounterResults = True
2970 for i in addedPValues:
2971 tmpResult = i in pCounters
2972 pCounterResults = pCounterResults and tmpResult
2973 if not tmpResult:
2974 main.log.error( str( i ) + " is not in partitioned "
2975 "counter incremented results" )
2976 utilities.assert_equals( expect=True,
2977 actual=pCounterResults,
2978 onpass="Default counter incremented",
2979 onfail="Error incrementing default" +
2980 " counter" )
2981
2982 main.step( "Counters we added have the correct values" )
2983 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2984 utilities.assert_equals( expect=main.TRUE,
2985 actual=incrementCheck,
2986 onpass="Added counters are correct",
2987 onfail="Added counters are incorrect" )
2988
2989 main.step( "Add -8 to then get a default counter on each node" )
2990 pCounters = []
2991 threads = []
2992 addedPValues = []
2993 for i in range( main.numCtrls ):
2994 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2995 name="counterIncrement-" + str( i ),
2996 args=[ pCounterName ],
2997 kwargs={ "delta": -8 } )
2998 pCounterValue += -8
2999 addedPValues.append( pCounterValue )
3000 threads.append( t )
3001 t.start()
3002
3003 for t in threads:
3004 t.join()
3005 pCounters.append( t.result )
3006 # Check that counter incremented numController times
3007 pCounterResults = True
3008 for i in addedPValues:
3009 tmpResult = i in pCounters
3010 pCounterResults = pCounterResults and tmpResult
3011 if not tmpResult:
3012 main.log.error( str( i ) + " is not in partitioned "
3013 "counter incremented results" )
3014 utilities.assert_equals( expect=True,
3015 actual=pCounterResults,
3016 onpass="Default counter incremented",
3017 onfail="Error incrementing default" +
3018 " counter" )
3019
3020 main.step( "Add 5 to then get a default counter on each node" )
3021 pCounters = []
3022 threads = []
3023 addedPValues = []
3024 for i in range( main.numCtrls ):
3025 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3026 name="counterIncrement-" + str( i ),
3027 args=[ pCounterName ],
3028 kwargs={ "delta": 5 } )
3029 pCounterValue += 5
3030 addedPValues.append( pCounterValue )
3031 threads.append( t )
3032 t.start()
3033
3034 for t in threads:
3035 t.join()
3036 pCounters.append( t.result )
3037 # Check that counter incremented numController times
3038 pCounterResults = True
3039 for i in addedPValues:
3040 tmpResult = i in pCounters
3041 pCounterResults = pCounterResults and tmpResult
3042 if not tmpResult:
3043 main.log.error( str( i ) + " is not in partitioned "
3044 "counter incremented results" )
3045 utilities.assert_equals( expect=True,
3046 actual=pCounterResults,
3047 onpass="Default counter incremented",
3048 onfail="Error incrementing default" +
3049 " counter" )
3050
3051 main.step( "Get then add 5 to a default counter on each node" )
3052 pCounters = []
3053 threads = []
3054 addedPValues = []
3055 for i in range( main.numCtrls ):
3056 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3057 name="counterIncrement-" + str( i ),
3058 args=[ pCounterName ],
3059 kwargs={ "delta": 5 } )
3060 addedPValues.append( pCounterValue )
3061 pCounterValue += 5
3062 threads.append( t )
3063 t.start()
3064
3065 for t in threads:
3066 t.join()
3067 pCounters.append( t.result )
3068 # Check that counter incremented numController times
3069 pCounterResults = True
3070 for i in addedPValues:
3071 tmpResult = i in pCounters
3072 pCounterResults = pCounterResults and tmpResult
3073 if not tmpResult:
3074 main.log.error( str( i ) + " is not in partitioned "
3075 "counter incremented results" )
3076 utilities.assert_equals( expect=True,
3077 actual=pCounterResults,
3078 onpass="Default counter incremented",
3079 onfail="Error incrementing default" +
3080 " counter" )
3081
3082 main.step( "Counters we added have the correct values" )
3083 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3084 utilities.assert_equals( expect=main.TRUE,
3085 actual=incrementCheck,
3086 onpass="Added counters are correct",
3087 onfail="Added counters are incorrect" )
3088
3089 # In-Memory counters
3090 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003091 iCounters = []
3092 addedIValues = []
3093 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003094 for i in range( main.numCtrls ):
3095 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003096 name="icounterIncrement-" + str( i ),
3097 args=[ iCounterName ],
3098 kwargs={ "inMemory": True } )
3099 iCounterValue += 1
3100 addedIValues.append( iCounterValue )
3101 threads.append( t )
3102 t.start()
3103
3104 for t in threads:
3105 t.join()
3106 iCounters.append( t.result )
3107 # Check that counter incremented numController times
3108 iCounterResults = True
3109 for i in addedIValues:
3110 tmpResult = i in iCounters
3111 iCounterResults = iCounterResults and tmpResult
3112 if not tmpResult:
3113 main.log.error( str( i ) + " is not in the in-memory "
3114 "counter incremented results" )
3115 utilities.assert_equals( expect=True,
3116 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003117 onpass="In-memory counter incremented",
3118 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003119 " counter" )
3120
Jon Halle1a3b752015-07-22 13:02:46 -07003121 main.step( "Get then Increment a in-memory counter on each node" )
3122 iCounters = []
3123 threads = []
3124 addedIValues = []
3125 for i in range( main.numCtrls ):
3126 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3127 name="counterGetAndAdd-" + str( i ),
3128 args=[ iCounterName ],
3129 kwargs={ "inMemory": True } )
3130 addedIValues.append( iCounterValue )
3131 iCounterValue += 1
3132 threads.append( t )
3133 t.start()
3134
3135 for t in threads:
3136 t.join()
3137 iCounters.append( t.result )
3138 # Check that counter incremented numController times
3139 iCounterResults = True
3140 for i in addedIValues:
3141 tmpResult = i in iCounters
3142 iCounterResults = iCounterResults and tmpResult
3143 if not tmpResult:
3144 main.log.error( str( i ) + " is not in in-memory "
3145 "counter incremented results" )
3146 utilities.assert_equals( expect=True,
3147 actual=iCounterResults,
3148 onpass="In-memory counter incremented",
3149 onfail="Error incrementing in-memory" +
3150 " counter" )
3151
3152 main.step( "Counters we added have the correct values" )
3153 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3154 utilities.assert_equals( expect=main.TRUE,
3155 actual=incrementCheck,
3156 onpass="Added counters are correct",
3157 onfail="Added counters are incorrect" )
3158
3159 main.step( "Add -8 to then get a in-memory counter on each node" )
3160 iCounters = []
3161 threads = []
3162 addedIValues = []
3163 for i in range( main.numCtrls ):
3164 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3165 name="counterIncrement-" + str( i ),
3166 args=[ iCounterName ],
3167 kwargs={ "delta": -8, "inMemory": True } )
3168 iCounterValue += -8
3169 addedIValues.append( iCounterValue )
3170 threads.append( t )
3171 t.start()
3172
3173 for t in threads:
3174 t.join()
3175 iCounters.append( t.result )
3176 # Check that counter incremented numController times
3177 iCounterResults = True
3178 for i in addedIValues:
3179 tmpResult = i in iCounters
3180 iCounterResults = iCounterResults and tmpResult
3181 if not tmpResult:
3182 main.log.error( str( i ) + " is not in in-memory "
3183 "counter incremented results" )
3184 utilities.assert_equals( expect=True,
3185 actual=pCounterResults,
3186 onpass="In-memory counter incremented",
3187 onfail="Error incrementing in-memory" +
3188 " counter" )
3189
3190 main.step( "Add 5 to then get a in-memory counter on each node" )
3191 iCounters = []
3192 threads = []
3193 addedIValues = []
3194 for i in range( main.numCtrls ):
3195 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3196 name="counterIncrement-" + str( i ),
3197 args=[ iCounterName ],
3198 kwargs={ "delta": 5, "inMemory": True } )
3199 iCounterValue += 5
3200 addedIValues.append( iCounterValue )
3201 threads.append( t )
3202 t.start()
3203
3204 for t in threads:
3205 t.join()
3206 iCounters.append( t.result )
3207 # Check that counter incremented numController times
3208 iCounterResults = True
3209 for i in addedIValues:
3210 tmpResult = i in iCounters
3211 iCounterResults = iCounterResults and tmpResult
3212 if not tmpResult:
3213 main.log.error( str( i ) + " is not in in-memory "
3214 "counter incremented results" )
3215 utilities.assert_equals( expect=True,
3216 actual=pCounterResults,
3217 onpass="In-memory counter incremented",
3218 onfail="Error incrementing in-memory" +
3219 " counter" )
3220
3221 main.step( "Get then add 5 to a in-memory counter on each node" )
3222 iCounters = []
3223 threads = []
3224 addedIValues = []
3225 for i in range( main.numCtrls ):
3226 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3227 name="counterIncrement-" + str( i ),
3228 args=[ iCounterName ],
3229 kwargs={ "delta": 5, "inMemory": True } )
3230 addedIValues.append( iCounterValue )
3231 iCounterValue += 5
3232 threads.append( t )
3233 t.start()
3234
3235 for t in threads:
3236 t.join()
3237 iCounters.append( t.result )
3238 # Check that counter incremented numController times
3239 iCounterResults = True
3240 for i in addedIValues:
3241 tmpResult = i in iCounters
3242 iCounterResults = iCounterResults and tmpResult
3243 if not tmpResult:
3244 main.log.error( str( i ) + " is not in in-memory "
3245 "counter incremented results" )
3246 utilities.assert_equals( expect=True,
3247 actual=iCounterResults,
3248 onpass="In-memory counter incremented",
3249 onfail="Error incrementing in-memory" +
3250 " counter" )
3251
3252 main.step( "Counters we added have the correct values" )
3253 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3254 utilities.assert_equals( expect=main.TRUE,
3255 actual=incrementCheck,
3256 onpass="Added counters are correct",
3257 onfail="Added counters are incorrect" )
3258
Jon Hall5cf14d52015-07-16 12:15:19 -07003259 main.step( "Check counters are consistant across nodes" )
3260 onosCounters = []
3261 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003262 for i in range( main.numCtrls ):
3263 t = main.Thread( target=main.CLIs[i].counters,
Jon Hall5cf14d52015-07-16 12:15:19 -07003264 name="counters-" + str( i ) )
3265 threads.append( t )
3266 t.start()
3267 for t in threads:
3268 t.join()
3269 onosCounters.append( t.result )
3270 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3271 if all( tmp ):
3272 main.log.info( "Counters are consistent across all nodes" )
3273 consistentCounterResults = main.TRUE
3274 else:
3275 main.log.error( "Counters are not consistent across all nodes" )
3276 consistentCounterResults = main.FALSE
3277 utilities.assert_equals( expect=main.TRUE,
3278 actual=consistentCounterResults,
3279 onpass="ONOS counters are consistent " +
3280 "across nodes",
3281 onfail="ONOS Counters are inconsistent " +
3282 "across nodes" )
3283
3284 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003285 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3286 incrementCheck = incrementCheck and \
3287 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003288 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003289 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003290 onpass="Added counters are correct",
3291 onfail="Added counters are incorrect" )
Jon Halle1a3b752015-07-22 13:02:46 -07003292
Jon Hall5cf14d52015-07-16 12:15:19 -07003293 # DISTRIBUTED SETS
3294 main.step( "Distributed Set get" )
3295 size = len( onosSet )
3296 getResponses = []
3297 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003298 for i in range( main.numCtrls ):
3299 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003300 name="setTestGet-" + str( i ),
3301 args=[ onosSetName ] )
3302 threads.append( t )
3303 t.start()
3304 for t in threads:
3305 t.join()
3306 getResponses.append( t.result )
3307
3308 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003309 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003310 if isinstance( getResponses[ i ], list):
3311 current = set( getResponses[ i ] )
3312 if len( current ) == len( getResponses[ i ] ):
3313 # no repeats
3314 if onosSet != current:
3315 main.log.error( "ONOS" + str( i + 1 ) +
3316 " has incorrect view" +
3317 " of set " + onosSetName + ":\n" +
3318 str( getResponses[ i ] ) )
3319 main.log.debug( "Expected: " + str( onosSet ) )
3320 main.log.debug( "Actual: " + str( current ) )
3321 getResults = main.FALSE
3322 else:
3323 # error, set is not a set
3324 main.log.error( "ONOS" + str( i + 1 ) +
3325 " has repeat elements in" +
3326 " set " + onosSetName + ":\n" +
3327 str( getResponses[ i ] ) )
3328 getResults = main.FALSE
3329 elif getResponses[ i ] == main.ERROR:
3330 getResults = main.FALSE
3331 utilities.assert_equals( expect=main.TRUE,
3332 actual=getResults,
3333 onpass="Set elements are correct",
3334 onfail="Set elements are incorrect" )
3335
3336 main.step( "Distributed Set size" )
3337 sizeResponses = []
3338 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003339 for i in range( main.numCtrls ):
3340 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003341 name="setTestSize-" + str( i ),
3342 args=[ onosSetName ] )
3343 threads.append( t )
3344 t.start()
3345 for t in threads:
3346 t.join()
3347 sizeResponses.append( t.result )
3348
3349 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003350 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003351 if size != sizeResponses[ i ]:
3352 sizeResults = main.FALSE
3353 main.log.error( "ONOS" + str( i + 1 ) +
3354 " expected a size of " + str( size ) +
3355 " for set " + onosSetName +
3356 " but got " + str( sizeResponses[ i ] ) )
3357 utilities.assert_equals( expect=main.TRUE,
3358 actual=sizeResults,
3359 onpass="Set sizes are correct",
3360 onfail="Set sizes are incorrect" )
3361
3362 main.step( "Distributed Set add()" )
3363 onosSet.add( addValue )
3364 addResponses = []
3365 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003366 for i in range( main.numCtrls ):
3367 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003368 name="setTestAdd-" + str( i ),
3369 args=[ onosSetName, addValue ] )
3370 threads.append( t )
3371 t.start()
3372 for t in threads:
3373 t.join()
3374 addResponses.append( t.result )
3375
3376 # main.TRUE = successfully changed the set
3377 # main.FALSE = action resulted in no change in set
3378 # main.ERROR - Some error in executing the function
3379 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003380 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003381 if addResponses[ i ] == main.TRUE:
3382 # All is well
3383 pass
3384 elif addResponses[ i ] == main.FALSE:
3385 # Already in set, probably fine
3386 pass
3387 elif addResponses[ i ] == main.ERROR:
3388 # Error in execution
3389 addResults = main.FALSE
3390 else:
3391 # unexpected result
3392 addResults = main.FALSE
3393 if addResults != main.TRUE:
3394 main.log.error( "Error executing set add" )
3395
3396 # Check if set is still correct
3397 size = len( onosSet )
3398 getResponses = []
3399 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003400 for i in range( main.numCtrls ):
3401 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003402 name="setTestGet-" + str( i ),
3403 args=[ onosSetName ] )
3404 threads.append( t )
3405 t.start()
3406 for t in threads:
3407 t.join()
3408 getResponses.append( t.result )
3409 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003410 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003411 if isinstance( getResponses[ i ], list):
3412 current = set( getResponses[ i ] )
3413 if len( current ) == len( getResponses[ i ] ):
3414 # no repeats
3415 if onosSet != current:
3416 main.log.error( "ONOS" + str( i + 1 ) +
3417 " has incorrect view" +
3418 " of set " + onosSetName + ":\n" +
3419 str( getResponses[ i ] ) )
3420 main.log.debug( "Expected: " + str( onosSet ) )
3421 main.log.debug( "Actual: " + str( current ) )
3422 getResults = main.FALSE
3423 else:
3424 # error, set is not a set
3425 main.log.error( "ONOS" + str( i + 1 ) +
3426 " has repeat elements in" +
3427 " set " + onosSetName + ":\n" +
3428 str( getResponses[ i ] ) )
3429 getResults = main.FALSE
3430 elif getResponses[ i ] == main.ERROR:
3431 getResults = main.FALSE
3432 sizeResponses = []
3433 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003434 for i in range( main.numCtrls ):
3435 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003436 name="setTestSize-" + str( i ),
3437 args=[ onosSetName ] )
3438 threads.append( t )
3439 t.start()
3440 for t in threads:
3441 t.join()
3442 sizeResponses.append( t.result )
3443 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003444 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003445 if size != sizeResponses[ i ]:
3446 sizeResults = main.FALSE
3447 main.log.error( "ONOS" + str( i + 1 ) +
3448 " expected a size of " + str( size ) +
3449 " for set " + onosSetName +
3450 " but got " + str( sizeResponses[ i ] ) )
3451 addResults = addResults and getResults and sizeResults
3452 utilities.assert_equals( expect=main.TRUE,
3453 actual=addResults,
3454 onpass="Set add correct",
3455 onfail="Set add was incorrect" )
3456
3457 main.step( "Distributed Set addAll()" )
3458 onosSet.update( addAllValue.split() )
3459 addResponses = []
3460 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003461 for i in range( main.numCtrls ):
3462 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003463 name="setTestAddAll-" + str( i ),
3464 args=[ onosSetName, addAllValue ] )
3465 threads.append( t )
3466 t.start()
3467 for t in threads:
3468 t.join()
3469 addResponses.append( t.result )
3470
3471 # main.TRUE = successfully changed the set
3472 # main.FALSE = action resulted in no change in set
3473 # main.ERROR - Some error in executing the function
3474 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003475 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003476 if addResponses[ i ] == main.TRUE:
3477 # All is well
3478 pass
3479 elif addResponses[ i ] == main.FALSE:
3480 # Already in set, probably fine
3481 pass
3482 elif addResponses[ i ] == main.ERROR:
3483 # Error in execution
3484 addAllResults = main.FALSE
3485 else:
3486 # unexpected result
3487 addAllResults = main.FALSE
3488 if addAllResults != main.TRUE:
3489 main.log.error( "Error executing set addAll" )
3490
3491 # Check if set is still correct
3492 size = len( onosSet )
3493 getResponses = []
3494 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003495 for i in range( main.numCtrls ):
3496 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003497 name="setTestGet-" + str( i ),
3498 args=[ onosSetName ] )
3499 threads.append( t )
3500 t.start()
3501 for t in threads:
3502 t.join()
3503 getResponses.append( t.result )
3504 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003505 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003506 if isinstance( getResponses[ i ], list):
3507 current = set( getResponses[ i ] )
3508 if len( current ) == len( getResponses[ i ] ):
3509 # no repeats
3510 if onosSet != current:
3511 main.log.error( "ONOS" + str( i + 1 ) +
3512 " has incorrect view" +
3513 " of set " + onosSetName + ":\n" +
3514 str( getResponses[ i ] ) )
3515 main.log.debug( "Expected: " + str( onosSet ) )
3516 main.log.debug( "Actual: " + str( current ) )
3517 getResults = main.FALSE
3518 else:
3519 # error, set is not a set
3520 main.log.error( "ONOS" + str( i + 1 ) +
3521 " has repeat elements in" +
3522 " set " + onosSetName + ":\n" +
3523 str( getResponses[ i ] ) )
3524 getResults = main.FALSE
3525 elif getResponses[ i ] == main.ERROR:
3526 getResults = main.FALSE
3527 sizeResponses = []
3528 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003529 for i in range( main.numCtrls ):
3530 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003531 name="setTestSize-" + str( i ),
3532 args=[ onosSetName ] )
3533 threads.append( t )
3534 t.start()
3535 for t in threads:
3536 t.join()
3537 sizeResponses.append( t.result )
3538 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003539 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003540 if size != sizeResponses[ i ]:
3541 sizeResults = main.FALSE
3542 main.log.error( "ONOS" + str( i + 1 ) +
3543 " expected a size of " + str( size ) +
3544 " for set " + onosSetName +
3545 " but got " + str( sizeResponses[ i ] ) )
3546 addAllResults = addAllResults and getResults and sizeResults
3547 utilities.assert_equals( expect=main.TRUE,
3548 actual=addAllResults,
3549 onpass="Set addAll correct",
3550 onfail="Set addAll was incorrect" )
3551
3552 main.step( "Distributed Set contains()" )
3553 containsResponses = []
3554 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003555 for i in range( main.numCtrls ):
3556 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003557 name="setContains-" + str( i ),
3558 args=[ onosSetName ],
3559 kwargs={ "values": addValue } )
3560 threads.append( t )
3561 t.start()
3562 for t in threads:
3563 t.join()
3564 # NOTE: This is the tuple
3565 containsResponses.append( t.result )
3566
3567 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003568 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003569 if containsResponses[ i ] == main.ERROR:
3570 containsResults = main.FALSE
3571 else:
3572 containsResults = containsResults and\
3573 containsResponses[ i ][ 1 ]
3574 utilities.assert_equals( expect=main.TRUE,
3575 actual=containsResults,
3576 onpass="Set contains is functional",
3577 onfail="Set contains failed" )
3578
3579 main.step( "Distributed Set containsAll()" )
3580 containsAllResponses = []
3581 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003582 for i in range( main.numCtrls ):
3583 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003584 name="setContainsAll-" + str( i ),
3585 args=[ onosSetName ],
3586 kwargs={ "values": addAllValue } )
3587 threads.append( t )
3588 t.start()
3589 for t in threads:
3590 t.join()
3591 # NOTE: This is the tuple
3592 containsAllResponses.append( t.result )
3593
3594 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003595 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003596 if containsResponses[ i ] == main.ERROR:
3597 containsResults = main.FALSE
3598 else:
3599 containsResults = containsResults and\
3600 containsResponses[ i ][ 1 ]
3601 utilities.assert_equals( expect=main.TRUE,
3602 actual=containsAllResults,
3603 onpass="Set containsAll is functional",
3604 onfail="Set containsAll failed" )
3605
3606 main.step( "Distributed Set remove()" )
3607 onosSet.remove( addValue )
3608 removeResponses = []
3609 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003610 for i in range( main.numCtrls ):
3611 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003612 name="setTestRemove-" + str( i ),
3613 args=[ onosSetName, addValue ] )
3614 threads.append( t )
3615 t.start()
3616 for t in threads:
3617 t.join()
3618 removeResponses.append( t.result )
3619
3620 # main.TRUE = successfully changed the set
3621 # main.FALSE = action resulted in no change in set
3622 # main.ERROR - Some error in executing the function
3623 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003624 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003625 if removeResponses[ i ] == main.TRUE:
3626 # All is well
3627 pass
3628 elif removeResponses[ i ] == main.FALSE:
3629 # not in set, probably fine
3630 pass
3631 elif removeResponses[ i ] == main.ERROR:
3632 # Error in execution
3633 removeResults = main.FALSE
3634 else:
3635 # unexpected result
3636 removeResults = main.FALSE
3637 if removeResults != main.TRUE:
3638 main.log.error( "Error executing set remove" )
3639
3640 # Check if set is still correct
3641 size = len( onosSet )
3642 getResponses = []
3643 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003644 for i in range( main.numCtrls ):
3645 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003646 name="setTestGet-" + str( i ),
3647 args=[ onosSetName ] )
3648 threads.append( t )
3649 t.start()
3650 for t in threads:
3651 t.join()
3652 getResponses.append( t.result )
3653 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003654 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003655 if isinstance( getResponses[ i ], list):
3656 current = set( getResponses[ i ] )
3657 if len( current ) == len( getResponses[ i ] ):
3658 # no repeats
3659 if onosSet != current:
3660 main.log.error( "ONOS" + str( i + 1 ) +
3661 " has incorrect view" +
3662 " of set " + onosSetName + ":\n" +
3663 str( getResponses[ i ] ) )
3664 main.log.debug( "Expected: " + str( onosSet ) )
3665 main.log.debug( "Actual: " + str( current ) )
3666 getResults = main.FALSE
3667 else:
3668 # error, set is not a set
3669 main.log.error( "ONOS" + str( i + 1 ) +
3670 " has repeat elements in" +
3671 " set " + onosSetName + ":\n" +
3672 str( getResponses[ i ] ) )
3673 getResults = main.FALSE
3674 elif getResponses[ i ] == main.ERROR:
3675 getResults = main.FALSE
3676 sizeResponses = []
3677 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003678 for i in range( main.numCtrls ):
3679 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003680 name="setTestSize-" + str( i ),
3681 args=[ onosSetName ] )
3682 threads.append( t )
3683 t.start()
3684 for t in threads:
3685 t.join()
3686 sizeResponses.append( t.result )
3687 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003688 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003689 if size != sizeResponses[ i ]:
3690 sizeResults = main.FALSE
3691 main.log.error( "ONOS" + str( i + 1 ) +
3692 " expected a size of " + str( size ) +
3693 " for set " + onosSetName +
3694 " but got " + str( sizeResponses[ i ] ) )
3695 removeResults = removeResults and getResults and sizeResults
3696 utilities.assert_equals( expect=main.TRUE,
3697 actual=removeResults,
3698 onpass="Set remove correct",
3699 onfail="Set remove was incorrect" )
3700
3701 main.step( "Distributed Set removeAll()" )
3702 onosSet.difference_update( addAllValue.split() )
3703 removeAllResponses = []
3704 threads = []
3705 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003706 for i in range( main.numCtrls ):
3707 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003708 name="setTestRemoveAll-" + str( i ),
3709 args=[ onosSetName, addAllValue ] )
3710 threads.append( t )
3711 t.start()
3712 for t in threads:
3713 t.join()
3714 removeAllResponses.append( t.result )
3715 except Exception, e:
3716 main.log.exception(e)
3717
3718 # main.TRUE = successfully changed the set
3719 # main.FALSE = action resulted in no change in set
3720 # main.ERROR - Some error in executing the function
3721 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003722 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003723 if removeAllResponses[ i ] == main.TRUE:
3724 # All is well
3725 pass
3726 elif removeAllResponses[ i ] == main.FALSE:
3727 # not in set, probably fine
3728 pass
3729 elif removeAllResponses[ i ] == main.ERROR:
3730 # Error in execution
3731 removeAllResults = main.FALSE
3732 else:
3733 # unexpected result
3734 removeAllResults = main.FALSE
3735 if removeAllResults != main.TRUE:
3736 main.log.error( "Error executing set removeAll" )
3737
3738 # Check if set is still correct
3739 size = len( onosSet )
3740 getResponses = []
3741 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003742 for i in range( main.numCtrls ):
3743 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003744 name="setTestGet-" + str( i ),
3745 args=[ onosSetName ] )
3746 threads.append( t )
3747 t.start()
3748 for t in threads:
3749 t.join()
3750 getResponses.append( t.result )
3751 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003752 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003753 if isinstance( getResponses[ i ], list):
3754 current = set( getResponses[ i ] )
3755 if len( current ) == len( getResponses[ i ] ):
3756 # no repeats
3757 if onosSet != current:
3758 main.log.error( "ONOS" + str( i + 1 ) +
3759 " has incorrect view" +
3760 " of set " + onosSetName + ":\n" +
3761 str( getResponses[ i ] ) )
3762 main.log.debug( "Expected: " + str( onosSet ) )
3763 main.log.debug( "Actual: " + str( current ) )
3764 getResults = main.FALSE
3765 else:
3766 # error, set is not a set
3767 main.log.error( "ONOS" + str( i + 1 ) +
3768 " has repeat elements in" +
3769 " set " + onosSetName + ":\n" +
3770 str( getResponses[ i ] ) )
3771 getResults = main.FALSE
3772 elif getResponses[ i ] == main.ERROR:
3773 getResults = main.FALSE
3774 sizeResponses = []
3775 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003776 for i in range( main.numCtrls ):
3777 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003778 name="setTestSize-" + str( i ),
3779 args=[ onosSetName ] )
3780 threads.append( t )
3781 t.start()
3782 for t in threads:
3783 t.join()
3784 sizeResponses.append( t.result )
3785 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003786 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003787 if size != sizeResponses[ i ]:
3788 sizeResults = main.FALSE
3789 main.log.error( "ONOS" + str( i + 1 ) +
3790 " expected a size of " + str( size ) +
3791 " for set " + onosSetName +
3792 " but got " + str( sizeResponses[ i ] ) )
3793 removeAllResults = removeAllResults and getResults and sizeResults
3794 utilities.assert_equals( expect=main.TRUE,
3795 actual=removeAllResults,
3796 onpass="Set removeAll correct",
3797 onfail="Set removeAll was incorrect" )
3798
3799 main.step( "Distributed Set addAll()" )
3800 onosSet.update( addAllValue.split() )
3801 addResponses = []
3802 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003803 for i in range( main.numCtrls ):
3804 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003805 name="setTestAddAll-" + str( i ),
3806 args=[ onosSetName, addAllValue ] )
3807 threads.append( t )
3808 t.start()
3809 for t in threads:
3810 t.join()
3811 addResponses.append( t.result )
3812
3813 # main.TRUE = successfully changed the set
3814 # main.FALSE = action resulted in no change in set
3815 # main.ERROR - Some error in executing the function
3816 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003817 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003818 if addResponses[ i ] == main.TRUE:
3819 # All is well
3820 pass
3821 elif addResponses[ i ] == main.FALSE:
3822 # Already in set, probably fine
3823 pass
3824 elif addResponses[ i ] == main.ERROR:
3825 # Error in execution
3826 addAllResults = main.FALSE
3827 else:
3828 # unexpected result
3829 addAllResults = main.FALSE
3830 if addAllResults != main.TRUE:
3831 main.log.error( "Error executing set addAll" )
3832
3833 # Check if set is still correct
3834 size = len( onosSet )
3835 getResponses = []
3836 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003837 for i in range( main.numCtrls ):
3838 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003839 name="setTestGet-" + str( i ),
3840 args=[ onosSetName ] )
3841 threads.append( t )
3842 t.start()
3843 for t in threads:
3844 t.join()
3845 getResponses.append( t.result )
3846 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003847 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003848 if isinstance( getResponses[ i ], list):
3849 current = set( getResponses[ i ] )
3850 if len( current ) == len( getResponses[ i ] ):
3851 # no repeats
3852 if onosSet != current:
3853 main.log.error( "ONOS" + str( i + 1 ) +
3854 " has incorrect view" +
3855 " of set " + onosSetName + ":\n" +
3856 str( getResponses[ i ] ) )
3857 main.log.debug( "Expected: " + str( onosSet ) )
3858 main.log.debug( "Actual: " + str( current ) )
3859 getResults = main.FALSE
3860 else:
3861 # error, set is not a set
3862 main.log.error( "ONOS" + str( i + 1 ) +
3863 " has repeat elements in" +
3864 " set " + onosSetName + ":\n" +
3865 str( getResponses[ i ] ) )
3866 getResults = main.FALSE
3867 elif getResponses[ i ] == main.ERROR:
3868 getResults = main.FALSE
3869 sizeResponses = []
3870 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003871 for i in range( main.numCtrls ):
3872 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003873 name="setTestSize-" + str( i ),
3874 args=[ onosSetName ] )
3875 threads.append( t )
3876 t.start()
3877 for t in threads:
3878 t.join()
3879 sizeResponses.append( t.result )
3880 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003881 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003882 if size != sizeResponses[ i ]:
3883 sizeResults = main.FALSE
3884 main.log.error( "ONOS" + str( i + 1 ) +
3885 " expected a size of " + str( size ) +
3886 " for set " + onosSetName +
3887 " but got " + str( sizeResponses[ i ] ) )
3888 addAllResults = addAllResults and getResults and sizeResults
3889 utilities.assert_equals( expect=main.TRUE,
3890 actual=addAllResults,
3891 onpass="Set addAll correct",
3892 onfail="Set addAll was incorrect" )
3893
3894 main.step( "Distributed Set clear()" )
3895 onosSet.clear()
3896 clearResponses = []
3897 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003898 for i in range( main.numCtrls ):
3899 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003900 name="setTestClear-" + str( i ),
3901 args=[ onosSetName, " "], # Values doesn't matter
3902 kwargs={ "clear": True } )
3903 threads.append( t )
3904 t.start()
3905 for t in threads:
3906 t.join()
3907 clearResponses.append( t.result )
3908
3909 # main.TRUE = successfully changed the set
3910 # main.FALSE = action resulted in no change in set
3911 # main.ERROR - Some error in executing the function
3912 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003913 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003914 if clearResponses[ i ] == main.TRUE:
3915 # All is well
3916 pass
3917 elif clearResponses[ i ] == main.FALSE:
3918 # Nothing set, probably fine
3919 pass
3920 elif clearResponses[ i ] == main.ERROR:
3921 # Error in execution
3922 clearResults = main.FALSE
3923 else:
3924 # unexpected result
3925 clearResults = main.FALSE
3926 if clearResults != main.TRUE:
3927 main.log.error( "Error executing set clear" )
3928
3929 # Check if set is still correct
3930 size = len( onosSet )
3931 getResponses = []
3932 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003933 for i in range( main.numCtrls ):
3934 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003935 name="setTestGet-" + str( i ),
3936 args=[ onosSetName ] )
3937 threads.append( t )
3938 t.start()
3939 for t in threads:
3940 t.join()
3941 getResponses.append( t.result )
3942 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003943 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003944 if isinstance( getResponses[ i ], list):
3945 current = set( getResponses[ i ] )
3946 if len( current ) == len( getResponses[ i ] ):
3947 # no repeats
3948 if onosSet != current:
3949 main.log.error( "ONOS" + str( i + 1 ) +
3950 " has incorrect view" +
3951 " of set " + onosSetName + ":\n" +
3952 str( getResponses[ i ] ) )
3953 main.log.debug( "Expected: " + str( onosSet ) )
3954 main.log.debug( "Actual: " + str( current ) )
3955 getResults = main.FALSE
3956 else:
3957 # error, set is not a set
3958 main.log.error( "ONOS" + str( i + 1 ) +
3959 " has repeat elements in" +
3960 " set " + onosSetName + ":\n" +
3961 str( getResponses[ i ] ) )
3962 getResults = main.FALSE
3963 elif getResponses[ i ] == main.ERROR:
3964 getResults = main.FALSE
3965 sizeResponses = []
3966 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003967 for i in range( main.numCtrls ):
3968 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003969 name="setTestSize-" + str( i ),
3970 args=[ onosSetName ] )
3971 threads.append( t )
3972 t.start()
3973 for t in threads:
3974 t.join()
3975 sizeResponses.append( t.result )
3976 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003977 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003978 if size != sizeResponses[ i ]:
3979 sizeResults = main.FALSE
3980 main.log.error( "ONOS" + str( i + 1 ) +
3981 " expected a size of " + str( size ) +
3982 " for set " + onosSetName +
3983 " but got " + str( sizeResponses[ i ] ) )
3984 clearResults = clearResults and getResults and sizeResults
3985 utilities.assert_equals( expect=main.TRUE,
3986 actual=clearResults,
3987 onpass="Set clear correct",
3988 onfail="Set clear was incorrect" )
3989
3990 main.step( "Distributed Set addAll()" )
3991 onosSet.update( addAllValue.split() )
3992 addResponses = []
3993 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003994 for i in range( main.numCtrls ):
3995 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003996 name="setTestAddAll-" + str( i ),
3997 args=[ onosSetName, addAllValue ] )
3998 threads.append( t )
3999 t.start()
4000 for t in threads:
4001 t.join()
4002 addResponses.append( t.result )
4003
4004 # main.TRUE = successfully changed the set
4005 # main.FALSE = action resulted in no change in set
4006 # main.ERROR - Some error in executing the function
4007 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004008 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004009 if addResponses[ i ] == main.TRUE:
4010 # All is well
4011 pass
4012 elif addResponses[ i ] == main.FALSE:
4013 # Already in set, probably fine
4014 pass
4015 elif addResponses[ i ] == main.ERROR:
4016 # Error in execution
4017 addAllResults = main.FALSE
4018 else:
4019 # unexpected result
4020 addAllResults = main.FALSE
4021 if addAllResults != main.TRUE:
4022 main.log.error( "Error executing set addAll" )
4023
4024 # Check if set is still correct
4025 size = len( onosSet )
4026 getResponses = []
4027 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004028 for i in range( main.numCtrls ):
4029 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004030 name="setTestGet-" + str( i ),
4031 args=[ onosSetName ] )
4032 threads.append( t )
4033 t.start()
4034 for t in threads:
4035 t.join()
4036 getResponses.append( t.result )
4037 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004038 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004039 if isinstance( getResponses[ i ], list):
4040 current = set( getResponses[ i ] )
4041 if len( current ) == len( getResponses[ i ] ):
4042 # no repeats
4043 if onosSet != current:
4044 main.log.error( "ONOS" + str( i + 1 ) +
4045 " has incorrect view" +
4046 " of set " + onosSetName + ":\n" +
4047 str( getResponses[ i ] ) )
4048 main.log.debug( "Expected: " + str( onosSet ) )
4049 main.log.debug( "Actual: " + str( current ) )
4050 getResults = main.FALSE
4051 else:
4052 # error, set is not a set
4053 main.log.error( "ONOS" + str( i + 1 ) +
4054 " has repeat elements in" +
4055 " set " + onosSetName + ":\n" +
4056 str( getResponses[ i ] ) )
4057 getResults = main.FALSE
4058 elif getResponses[ i ] == main.ERROR:
4059 getResults = main.FALSE
4060 sizeResponses = []
4061 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004062 for i in range( main.numCtrls ):
4063 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004064 name="setTestSize-" + str( i ),
4065 args=[ onosSetName ] )
4066 threads.append( t )
4067 t.start()
4068 for t in threads:
4069 t.join()
4070 sizeResponses.append( t.result )
4071 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004072 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004073 if size != sizeResponses[ i ]:
4074 sizeResults = main.FALSE
4075 main.log.error( "ONOS" + str( i + 1 ) +
4076 " expected a size of " + str( size ) +
4077 " for set " + onosSetName +
4078 " but got " + str( sizeResponses[ i ] ) )
4079 addAllResults = addAllResults and getResults and sizeResults
4080 utilities.assert_equals( expect=main.TRUE,
4081 actual=addAllResults,
4082 onpass="Set addAll correct",
4083 onfail="Set addAll was incorrect" )
4084
4085 main.step( "Distributed Set retain()" )
4086 onosSet.intersection_update( retainValue.split() )
4087 retainResponses = []
4088 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004089 for i in range( main.numCtrls ):
4090 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004091 name="setTestRetain-" + str( i ),
4092 args=[ onosSetName, retainValue ],
4093 kwargs={ "retain": True } )
4094 threads.append( t )
4095 t.start()
4096 for t in threads:
4097 t.join()
4098 retainResponses.append( t.result )
4099
4100 # main.TRUE = successfully changed the set
4101 # main.FALSE = action resulted in no change in set
4102 # main.ERROR - Some error in executing the function
4103 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004104 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004105 if retainResponses[ i ] == main.TRUE:
4106 # All is well
4107 pass
4108 elif retainResponses[ i ] == main.FALSE:
4109 # Already in set, probably fine
4110 pass
4111 elif retainResponses[ i ] == main.ERROR:
4112 # Error in execution
4113 retainResults = main.FALSE
4114 else:
4115 # unexpected result
4116 retainResults = main.FALSE
4117 if retainResults != main.TRUE:
4118 main.log.error( "Error executing set retain" )
4119
4120 # Check if set is still correct
4121 size = len( onosSet )
4122 getResponses = []
4123 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004124 for i in range( main.numCtrls ):
4125 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004126 name="setTestGet-" + str( i ),
4127 args=[ onosSetName ] )
4128 threads.append( t )
4129 t.start()
4130 for t in threads:
4131 t.join()
4132 getResponses.append( t.result )
4133 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004134 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004135 if isinstance( getResponses[ i ], list):
4136 current = set( getResponses[ i ] )
4137 if len( current ) == len( getResponses[ i ] ):
4138 # no repeats
4139 if onosSet != current:
4140 main.log.error( "ONOS" + str( i + 1 ) +
4141 " has incorrect view" +
4142 " of set " + onosSetName + ":\n" +
4143 str( getResponses[ i ] ) )
4144 main.log.debug( "Expected: " + str( onosSet ) )
4145 main.log.debug( "Actual: " + str( current ) )
4146 getResults = main.FALSE
4147 else:
4148 # error, set is not a set
4149 main.log.error( "ONOS" + str( i + 1 ) +
4150 " has repeat elements in" +
4151 " set " + onosSetName + ":\n" +
4152 str( getResponses[ i ] ) )
4153 getResults = main.FALSE
4154 elif getResponses[ i ] == main.ERROR:
4155 getResults = main.FALSE
4156 sizeResponses = []
4157 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004158 for i in range( main.numCtrls ):
4159 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004160 name="setTestSize-" + str( i ),
4161 args=[ onosSetName ] )
4162 threads.append( t )
4163 t.start()
4164 for t in threads:
4165 t.join()
4166 sizeResponses.append( t.result )
4167 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004168 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004169 if size != sizeResponses[ i ]:
4170 sizeResults = main.FALSE
4171 main.log.error( "ONOS" + str( i + 1 ) +
4172 " expected a size of " +
4173 str( size ) + " for set " + onosSetName +
4174 " but got " + str( sizeResponses[ i ] ) )
4175 retainResults = retainResults and getResults and sizeResults
4176 utilities.assert_equals( expect=main.TRUE,
4177 actual=retainResults,
4178 onpass="Set retain correct",
4179 onfail="Set retain was incorrect" )
4180
Jon Hall2a5002c2015-08-21 16:49:11 -07004181 # Transactional maps
4182 main.step( "Partitioned Transactional maps put" )
4183 tMapValue = "Testing"
4184 numKeys = 100
4185 putResult = True
4186 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4187 if len( putResponses ) == 100:
4188 for i in putResponses:
4189 if putResponses[ i ][ 'value' ] != tMapValue:
4190 putResult = False
4191 else:
4192 putResult = False
4193 if not putResult:
4194 main.log.debug( "Put response values: " + str( putResponses ) )
4195 utilities.assert_equals( expect=True,
4196 actual=putResult,
4197 onpass="Partitioned Transactional Map put successful",
4198 onfail="Partitioned Transactional Map put values are incorrect" )
4199
4200 main.step( "Partitioned Transactional maps get" )
4201 getCheck = True
4202 for n in range( 1, numKeys + 1 ):
4203 getResponses = []
4204 threads = []
4205 valueCheck = True
4206 for i in range( main.numCtrls ):
4207 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4208 name="TMap-get-" + str( i ),
4209 args=[ "Key" + str ( n ) ] )
4210 threads.append( t )
4211 t.start()
4212 for t in threads:
4213 t.join()
4214 getResponses.append( t.result )
4215 for node in getResponses:
4216 if node != tMapValue:
4217 valueCheck = False
4218 if not valueCheck:
4219 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4220 main.log.warn( getResponses )
4221 getCheck = getCheck and valueCheck
4222 utilities.assert_equals( expect=True,
4223 actual=getCheck,
4224 onpass="Partitioned Transactional Map get values were correct",
4225 onfail="Partitioned Transactional Map values incorrect" )
4226
4227 main.step( "In-memory Transactional maps put" )
4228 tMapValue = "Testing"
4229 numKeys = 100
4230 putResult = True
4231 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4232 if len( putResponses ) == 100:
4233 for i in putResponses:
4234 if putResponses[ i ][ 'value' ] != tMapValue:
4235 putResult = False
4236 else:
4237 putResult = False
4238 if not putResult:
4239 main.log.debug( "Put response values: " + str( putResponses ) )
4240 utilities.assert_equals( expect=True,
4241 actual=putResult,
4242 onpass="In-Memory Transactional Map put successful",
4243 onfail="In-Memory Transactional Map put values are incorrect" )
4244
4245 main.step( "In-Memory Transactional maps get" )
4246 getCheck = True
4247 for n in range( 1, numKeys + 1 ):
4248 getResponses = []
4249 threads = []
4250 valueCheck = True
4251 for i in range( main.numCtrls ):
4252 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4253 name="TMap-get-" + str( i ),
4254 args=[ "Key" + str ( n ) ],
4255 kwargs={ "inMemory": True } )
4256 threads.append( t )
4257 t.start()
4258 for t in threads:
4259 t.join()
4260 getResponses.append( t.result )
4261 for node in getResponses:
4262 if node != tMapValue:
4263 valueCheck = False
4264 if not valueCheck:
4265 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4266 main.log.warn( getResponses )
4267 getCheck = getCheck and valueCheck
4268 utilities.assert_equals( expect=True,
4269 actual=getCheck,
4270 onpass="In-Memory Transactional Map get values were correct",
4271 onfail="In-Memory Transactional Map values incorrect" )