blob: fe1557398634c8844b83bd7c91d0b3e0c487878f [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAclusterRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hall5cf14d52015-07-16 12:15:19 -070051 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
52 "initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
70 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070071 global ONOS1Port
72 global ONOS2Port
73 global ONOS3Port
74 global ONOS4Port
75 global ONOS5Port
76 global ONOS6Port
77 global ONOS7Port
78 # These are for csv plotting in jenkins
79 global labels
80 global data
81 labels = []
82 data = []
83
84 # FIXME: just get controller port from params?
85 # TODO: do we really need all these?
86 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
87 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
88 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
89 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
90 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
91 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
92 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
93
Jon Halle1a3b752015-07-22 13:02:46 -070094 try:
95 fileName = "Counters"
96 path = main.params[ 'imports' ][ 'path' ]
97 main.Counters = imp.load_source( fileName,
98 path + fileName + ".py" )
99 except Exception as e:
100 main.log.exception( e )
101 main.cleanup()
102 main.exit()
103
104 main.CLIs = []
105 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700106 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700107 for i in range( 1, main.numCtrls + 1 ):
108 try:
109 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
110 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
111 ipList.append( main.nodes[ -1 ].ip_address )
112 except AttributeError:
113 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700114
115 main.step( "Create cell file" )
116 cellAppString = main.params[ 'ENV' ][ 'appString' ]
117 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
118 main.Mininet1.ip_address,
119 cellAppString, ipList )
120 main.step( "Applying cell variable to environment" )
121 cellResult = main.ONOSbench.setCell( cellName )
122 verifyResult = main.ONOSbench.verifyCell()
123
124 # FIXME:this is short term fix
125 main.log.info( "Removing raft logs" )
126 main.ONOSbench.onosRemoveRaftLogs()
127
128 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700129 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700130 main.ONOSbench.onosUninstall( node.ip_address )
131
132 # Make sure ONOS is DEAD
133 main.log.info( "Killing any ONOS processes" )
134 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700135 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700136 killed = main.ONOSbench.onosKill( node.ip_address )
137 killResults = killResults and killed
138
139 cleanInstallResult = main.TRUE
140 gitPullResult = main.TRUE
141
142 main.step( "Starting Mininet" )
143 # scp topo file to mininet
144 # TODO: move to params?
145 topoName = "obelisk.py"
146 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700147 main.ONOSbench.scp( main.Mininet1,
148 filePath + topoName,
149 main.Mininet1.home,
150 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700151 mnResult = main.Mininet1.startNet( )
152 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
153 onpass="Mininet Started",
154 onfail="Error starting Mininet" )
155
156 main.step( "Git checkout and pull " + gitBranch )
157 if PULLCODE:
158 main.ONOSbench.gitCheckout( gitBranch )
159 gitPullResult = main.ONOSbench.gitPull()
160 # values of 1 or 3 are good
161 utilities.assert_lesser( expect=0, actual=gitPullResult,
162 onpass="Git pull successful",
163 onfail="Git pull failed" )
164 main.ONOSbench.getVersion( report=True )
165
166 main.step( "Using mvn clean install" )
167 cleanInstallResult = main.TRUE
168 if PULLCODE and gitPullResult == main.TRUE:
169 cleanInstallResult = main.ONOSbench.cleanInstall()
170 else:
171 main.log.warn( "Did not pull new code so skipping mvn " +
172 "clean install" )
173 utilities.assert_equals( expect=main.TRUE,
174 actual=cleanInstallResult,
175 onpass="MCI successful",
176 onfail="MCI failed" )
177 # GRAPHS
178 # NOTE: important params here:
179 # job = name of Jenkins job
180 # Plot Name = Plot-HA, only can be used if multiple plots
181 # index = The number of the graph under plot name
182 job = "HAclusterRestart"
183 plotName = "Plot-HA"
184 graphs = '<ac:structured-macro ac:name="html">\n'
185 graphs += '<ac:plain-text-body><![CDATA[\n'
186 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
187 '/plot/' + plotName + '/getPlot?index=0' +\
188 '&width=500&height=300"' +\
189 'noborder="0" width="500" height="300" scrolling="yes" ' +\
190 'seamless="seamless"></iframe>\n'
191 graphs += ']]></ac:plain-text-body>\n'
192 graphs += '</ac:structured-macro>\n'
193 main.log.wiki(graphs)
194
195 main.step( "Creating ONOS package" )
196 packageResult = main.ONOSbench.onosPackage()
197 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
198 onpass="ONOS package successful",
199 onfail="ONOS package failed" )
200
201 main.step( "Installing ONOS package" )
202 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700203 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700204 tmpResult = main.ONOSbench.onosInstall( options="-f",
205 node=node.ip_address )
206 onosInstallResult = onosInstallResult and tmpResult
207 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
208 onpass="ONOS install successful",
209 onfail="ONOS install failed" )
210
211 main.step( "Checking if ONOS is up yet" )
212 for i in range( 2 ):
213 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700214 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700215 started = main.ONOSbench.isup( node.ip_address )
216 if not started:
217 main.log.error( node.name + " didn't start!" )
218 main.ONOSbench.onosStop( node.ip_address )
219 main.ONOSbench.onosStart( node.ip_address )
220 onosIsupResult = onosIsupResult and started
221 if onosIsupResult == main.TRUE:
222 break
223 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
224 onpass="ONOS startup successful",
225 onfail="ONOS startup failed" )
226
227 main.log.step( "Starting ONOS CLI sessions" )
228 cliResults = main.TRUE
229 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700230 for i in range( main.numCtrls ):
231 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700232 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700233 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700234 threads.append( t )
235 t.start()
236
237 for t in threads:
238 t.join()
239 cliResults = cliResults and t.result
240 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
241 onpass="ONOS cli startup successful",
242 onfail="ONOS cli startup failed" )
243
244 if main.params[ 'tcpdump' ].lower() == "true":
245 main.step( "Start Packet Capture MN" )
246 main.Mininet2.startTcpdump(
247 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
248 + "-MN.pcap",
249 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
250 port=main.params[ 'MNtcpdump' ][ 'port' ] )
251
252 main.step( "App Ids check" )
253 appCheck = main.TRUE
254 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700255 for i in range( main.numCtrls ):
256 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700257 name="appToIDCheck-" + str( i ),
258 args=[] )
259 threads.append( t )
260 t.start()
261
262 for t in threads:
263 t.join()
264 appCheck = appCheck and t.result
265 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700266 main.log.warn( main.CLIs[0].apps() )
267 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700268 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
269 onpass="App Ids seem to be correct",
270 onfail="Something is wrong with app Ids" )
271
272 if cliResults == main.FALSE:
273 main.log.error( "Failed to start ONOS, stopping test" )
274 main.cleanup()
275 main.exit()
276
277 def CASE2( self, main ):
278 """
279 Assign devices to controllers
280 """
281 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700282 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700283 assert main, "main not defined"
284 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700285 assert main.CLIs, "main.CLIs not defined"
286 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700287 assert ONOS1Port, "ONOS1Port not defined"
288 assert ONOS2Port, "ONOS2Port not defined"
289 assert ONOS3Port, "ONOS3Port not defined"
290 assert ONOS4Port, "ONOS4Port not defined"
291 assert ONOS5Port, "ONOS5Port not defined"
292 assert ONOS6Port, "ONOS6Port not defined"
293 assert ONOS7Port, "ONOS7Port not defined"
294
295 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700296 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700297 "and check that an ONOS node becomes the " +\
298 "master of the device."
299 main.step( "Assign switches to controllers" )
300
301 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700302 for i in range( main.numCtrls ):
303 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700304 swList = []
305 for i in range( 1, 29 ):
306 swList.append( "s" + str( i ) )
307 main.Mininet1.assignSwController( sw=swList, ip=ipList )
308
309 mastershipCheck = main.TRUE
310 for i in range( 1, 29 ):
311 response = main.Mininet1.getSwController( "s" + str( i ) )
312 try:
313 main.log.info( str( response ) )
314 except Exception:
315 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700316 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700317 if re.search( "tcp:" + node.ip_address, response ):
318 mastershipCheck = mastershipCheck and main.TRUE
319 else:
320 main.log.error( "Error, node " + node.ip_address + " is " +
321 "not in the list of controllers s" +
322 str( i ) + " is connecting to." )
323 mastershipCheck = main.FALSE
324 utilities.assert_equals(
325 expect=main.TRUE,
326 actual=mastershipCheck,
327 onpass="Switch mastership assigned correctly",
328 onfail="Switches not assigned correctly to controllers" )
329
330 def CASE21( self, main ):
331 """
332 Assign mastership to controllers
333 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700334 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700335 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700336 assert main, "main not defined"
337 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700338 assert main.CLIs, "main.CLIs not defined"
339 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700340 assert ONOS1Port, "ONOS1Port not defined"
341 assert ONOS2Port, "ONOS2Port not defined"
342 assert ONOS3Port, "ONOS3Port not defined"
343 assert ONOS4Port, "ONOS4Port not defined"
344 assert ONOS5Port, "ONOS5Port not defined"
345 assert ONOS6Port, "ONOS6Port not defined"
346 assert ONOS7Port, "ONOS7Port not defined"
347
348 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700349 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700350 "device. Then manually assign" +\
351 " mastership to specific ONOS nodes using" +\
352 " 'device-role'"
353 main.step( "Assign mastership of switches to specific controllers" )
354 # Manually assign mastership to the controller we want
355 roleCall = main.TRUE
356
357 ipList = [ ]
358 deviceList = []
359 try:
360 # Assign mastership to specific controllers. This assignment was
361 # determined for a 7 node cluser, but will work with any sized
362 # cluster
363 for i in range( 1, 29 ): # switches 1 through 28
364 # set up correct variables:
365 if i == 1:
366 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700367 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
369 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700370 c = 1 % main.numCtrls
371 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
373 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700374 c = 1 % main.numCtrls
375 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700376 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
377 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700378 c = 3 % main.numCtrls
379 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700380 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
381 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700382 c = 2 % main.numCtrls
383 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700384 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
385 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700386 c = 2 % main.numCtrls
387 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700388 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
389 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700390 c = 5 % main.numCtrls
391 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700392 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
393 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700394 c = 4 % main.numCtrls
395 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700396 dpid = '3' + str( i ).zfill( 3 )
397 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
398 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700399 c = 6 % main.numCtrls
400 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700401 dpid = '6' + str( i ).zfill( 3 )
402 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
403 elif i == 28:
404 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700405 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700406 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
407 else:
408 main.log.error( "You didn't write an else statement for " +
409 "switch s" + str( i ) )
410 roleCall = main.FALSE
411 # Assign switch
412 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
413 # TODO: make this controller dynamic
414 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
415 ip )
416 ipList.append( ip )
417 deviceList.append( deviceId )
418 except ( AttributeError, AssertionError ):
419 main.log.exception( "Something is wrong with ONOS device view" )
420 main.log.info( main.ONOScli1.devices() )
421 utilities.assert_equals(
422 expect=main.TRUE,
423 actual=roleCall,
424 onpass="Re-assigned switch mastership to designated controller",
425 onfail="Something wrong with deviceRole calls" )
426
427 main.step( "Check mastership was correctly assigned" )
428 roleCheck = main.TRUE
429 # NOTE: This is due to the fact that device mastership change is not
430 # atomic and is actually a multi step process
431 time.sleep( 5 )
432 for i in range( len( ipList ) ):
433 ip = ipList[i]
434 deviceId = deviceList[i]
435 # Check assignment
436 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
437 if ip in master:
438 roleCheck = roleCheck and main.TRUE
439 else:
440 roleCheck = roleCheck and main.FALSE
441 main.log.error( "Error, controller " + ip + " is not" +
442 " master " + "of device " +
443 str( deviceId ) + ". Master is " +
444 repr( master ) + "." )
445 utilities.assert_equals(
446 expect=main.TRUE,
447 actual=roleCheck,
448 onpass="Switches were successfully reassigned to designated " +
449 "controller",
450 onfail="Switches were not successfully reassigned" )
451
452 def CASE3( self, main ):
453 """
454 Assign intents
455 """
456 import time
457 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700458 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 assert main, "main not defined"
460 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700461 assert main.CLIs, "main.CLIs not defined"
462 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 try:
464 labels
465 except NameError:
466 main.log.error( "labels not defined, setting to []" )
467 labels = []
468 try:
469 data
470 except NameError:
471 main.log.error( "data not defined, setting to []" )
472 data = []
473 # NOTE: we must reinstall intents until we have a persistant intent
474 # datastore!
475 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700476 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700477 "assign predetermined host-to-host intents." +\
478 " After installation, check that the intent" +\
479 " is distributed to all nodes and the state" +\
480 " is INSTALLED"
481
482 # install onos-app-fwd
483 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700484 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700485 utilities.assert_equals( expect=main.TRUE, actual=installResults,
486 onpass="Install fwd successful",
487 onfail="Install fwd failed" )
488
489 main.step( "Check app ids" )
490 appCheck = main.TRUE
491 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700492 for i in range( main.numCtrls ):
493 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700494 name="appToIDCheck-" + str( i ),
495 args=[] )
496 threads.append( t )
497 t.start()
498
499 for t in threads:
500 t.join()
501 appCheck = appCheck and t.result
502 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700503 main.log.warn( main.CLIs[0].apps() )
504 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700505 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
506 onpass="App Ids seem to be correct",
507 onfail="Something is wrong with app Ids" )
508
509 main.step( "Discovering Hosts( Via pingall for now )" )
510 # FIXME: Once we have a host discovery mechanism, use that instead
511 # REACTIVE FWD test
512 pingResult = main.FALSE
513 for i in range(2): # Retry if pingall fails first time
514 time1 = time.time()
515 pingResult = main.Mininet1.pingall()
516 if i == 0:
517 utilities.assert_equals(
518 expect=main.TRUE,
519 actual=pingResult,
520 onpass="Reactive Pingall test passed",
521 onfail="Reactive Pingall failed, " +
522 "one or more ping pairs failed" )
523 time2 = time.time()
524 main.log.info( "Time for pingall: %2f seconds" %
525 ( time2 - time1 ) )
526 # timeout for fwd flows
527 time.sleep( 11 )
528 # uninstall onos-app-fwd
529 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700530 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700531 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
532 onpass="Uninstall fwd successful",
533 onfail="Uninstall fwd failed" )
534
535 main.step( "Check app ids" )
536 threads = []
537 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700538 for i in range( main.numCtrls ):
539 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700540 name="appToIDCheck-" + str( i ),
541 args=[] )
542 threads.append( t )
543 t.start()
544
545 for t in threads:
546 t.join()
547 appCheck2 = appCheck2 and t.result
548 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700549 main.log.warn( main.CLIs[0].apps() )
550 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700551 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
552 onpass="App Ids seem to be correct",
553 onfail="Something is wrong with app Ids" )
554
555 main.step( "Add host intents via cli" )
556 intentIds = []
557 # TODO: move the host numbers to params
558 # Maybe look at all the paths we ping?
559 intentAddResult = True
560 hostResult = main.TRUE
561 for i in range( 8, 18 ):
562 main.log.info( "Adding host intent between h" + str( i ) +
563 " and h" + str( i + 10 ) )
564 host1 = "00:00:00:00:00:" + \
565 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
566 host2 = "00:00:00:00:00:" + \
567 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
568 # NOTE: getHost can return None
569 host1Dict = main.ONOScli1.getHost( host1 )
570 host2Dict = main.ONOScli1.getHost( host2 )
571 host1Id = None
572 host2Id = None
573 if host1Dict and host2Dict:
574 host1Id = host1Dict.get( 'id', None )
575 host2Id = host2Dict.get( 'id', None )
576 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700577 nodeNum = ( i % main.numCtrls )
578 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700579 if tmpId:
580 main.log.info( "Added intent with id: " + tmpId )
581 intentIds.append( tmpId )
582 else:
583 main.log.error( "addHostIntent returned: " +
584 repr( tmpId ) )
585 else:
586 main.log.error( "Error, getHost() failed for h" + str( i ) +
587 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700588 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700589 main.log.warn( "Hosts output: " )
590 try:
591 main.log.warn( json.dumps( json.loads( hosts ),
592 sort_keys=True,
593 indent=4,
594 separators=( ',', ': ' ) ) )
595 except ( ValueError, TypeError ):
596 main.log.warn( repr( hosts ) )
597 hostResult = main.FALSE
598 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
599 onpass="Found a host id for each host",
600 onfail="Error looking up host ids" )
601
602 intentStart = time.time()
603 onosIds = main.ONOScli1.getAllIntentsId()
604 main.log.info( "Submitted intents: " + str( intentIds ) )
605 main.log.info( "Intents in ONOS: " + str( onosIds ) )
606 for intent in intentIds:
607 if intent in onosIds:
608 pass # intent submitted is in onos
609 else:
610 intentAddResult = False
611 if intentAddResult:
612 intentStop = time.time()
613 else:
614 intentStop = None
615 # Print the intent states
616 intents = main.ONOScli1.intents()
617 intentStates = []
618 installedCheck = True
619 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
620 count = 0
621 try:
622 for intent in json.loads( intents ):
623 state = intent.get( 'state', None )
624 if "INSTALLED" not in state:
625 installedCheck = False
626 intentId = intent.get( 'id', None )
627 intentStates.append( ( intentId, state ) )
628 except ( ValueError, TypeError ):
629 main.log.exception( "Error parsing intents" )
630 # add submitted intents not in the store
631 tmplist = [ i for i, s in intentStates ]
632 missingIntents = False
633 for i in intentIds:
634 if i not in tmplist:
635 intentStates.append( ( i, " - " ) )
636 missingIntents = True
637 intentStates.sort()
638 for i, s in intentStates:
639 count += 1
640 main.log.info( "%-6s%-15s%-15s" %
641 ( str( count ), str( i ), str( s ) ) )
642 leaders = main.ONOScli1.leaders()
643 try:
644 missing = False
645 if leaders:
646 parsedLeaders = json.loads( leaders )
647 main.log.warn( json.dumps( parsedLeaders,
648 sort_keys=True,
649 indent=4,
650 separators=( ',', ': ' ) ) )
651 # check for all intent partitions
652 topics = []
653 for i in range( 14 ):
654 topics.append( "intent-partition-" + str( i ) )
655 main.log.debug( topics )
656 ONOStopics = [ j['topic'] for j in parsedLeaders ]
657 for topic in topics:
658 if topic not in ONOStopics:
659 main.log.error( "Error: " + topic +
660 " not in leaders" )
661 missing = True
662 else:
663 main.log.error( "leaders() returned None" )
664 except ( ValueError, TypeError ):
665 main.log.exception( "Error parsing leaders" )
666 main.log.error( repr( leaders ) )
667 # Check all nodes
668 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700669 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700670 response = node.leaders( jsonFormat=False)
671 main.log.warn( str( node.name ) + " leaders output: \n" +
672 str( response ) )
673
674 partitions = main.ONOScli1.partitions()
675 try:
676 if partitions :
677 parsedPartitions = json.loads( partitions )
678 main.log.warn( json.dumps( parsedPartitions,
679 sort_keys=True,
680 indent=4,
681 separators=( ',', ': ' ) ) )
682 # TODO check for a leader in all paritions
683 # TODO check for consistency among nodes
684 else:
685 main.log.error( "partitions() returned None" )
686 except ( ValueError, TypeError ):
687 main.log.exception( "Error parsing partitions" )
688 main.log.error( repr( partitions ) )
689 pendingMap = main.ONOScli1.pendingMap()
690 try:
691 if pendingMap :
692 parsedPending = json.loads( pendingMap )
693 main.log.warn( json.dumps( parsedPending,
694 sort_keys=True,
695 indent=4,
696 separators=( ',', ': ' ) ) )
697 # TODO check something here?
698 else:
699 main.log.error( "pendingMap() returned None" )
700 except ( ValueError, TypeError ):
701 main.log.exception( "Error parsing pending map" )
702 main.log.error( repr( pendingMap ) )
703
704 intentAddResult = bool( intentAddResult and not missingIntents and
705 installedCheck )
706 if not intentAddResult:
707 main.log.error( "Error in pushing host intents to ONOS" )
708
709 main.step( "Intent Anti-Entropy dispersion" )
710 for i in range(100):
711 correct = True
712 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700713 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700714 onosIds = []
715 ids = cli.getAllIntentsId()
716 onosIds.append( ids )
717 main.log.debug( "Intents in " + cli.name + ": " +
718 str( sorted( onosIds ) ) )
719 if sorted( ids ) != sorted( intentIds ):
720 main.log.warn( "Set of intent IDs doesn't match" )
721 correct = False
722 break
723 else:
724 intents = json.loads( cli.intents() )
725 for intent in intents:
726 if intent[ 'state' ] != "INSTALLED":
727 main.log.warn( "Intent " + intent[ 'id' ] +
728 " is " + intent[ 'state' ] )
729 correct = False
730 break
731 if correct:
732 break
733 else:
734 time.sleep(1)
735 if not intentStop:
736 intentStop = time.time()
737 global gossipTime
738 gossipTime = intentStop - intentStart
739 main.log.info( "It took about " + str( gossipTime ) +
740 " seconds for all intents to appear in each node" )
741 append = False
742 title = "Gossip Intents"
743 count = 1
744 while append is False:
745 curTitle = title + str( count )
746 if curTitle not in labels:
747 labels.append( curTitle )
748 data.append( str( gossipTime ) )
749 append = True
750 else:
751 count += 1
752 # FIXME: make this time configurable/calculate based off of number of
753 # nodes and gossip rounds
754 utilities.assert_greater_equals(
755 expect=40, actual=gossipTime,
756 onpass="ECM anti-entropy for intents worked within " +
757 "expected time",
758 onfail="Intent ECM anti-entropy took too long" )
759 if gossipTime <= 40:
760 intentAddResult = True
761
762 if not intentAddResult or "key" in pendingMap:
763 import time
764 installedCheck = True
765 main.log.info( "Sleeping 60 seconds to see if intents are found" )
766 time.sleep( 60 )
767 onosIds = main.ONOScli1.getAllIntentsId()
768 main.log.info( "Submitted intents: " + str( intentIds ) )
769 main.log.info( "Intents in ONOS: " + str( onosIds ) )
770 # Print the intent states
771 intents = main.ONOScli1.intents()
772 intentStates = []
773 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
774 count = 0
775 try:
776 for intent in json.loads( intents ):
777 # Iter through intents of a node
778 state = intent.get( 'state', None )
779 if "INSTALLED" not in state:
780 installedCheck = False
781 intentId = intent.get( 'id', None )
782 intentStates.append( ( intentId, state ) )
783 except ( ValueError, TypeError ):
784 main.log.exception( "Error parsing intents" )
785 # add submitted intents not in the store
786 tmplist = [ i for i, s in intentStates ]
787 for i in intentIds:
788 if i not in tmplist:
789 intentStates.append( ( i, " - " ) )
790 intentStates.sort()
791 for i, s in intentStates:
792 count += 1
793 main.log.info( "%-6s%-15s%-15s" %
794 ( str( count ), str( i ), str( s ) ) )
795 leaders = main.ONOScli1.leaders()
796 try:
797 missing = False
798 if leaders:
799 parsedLeaders = json.loads( leaders )
800 main.log.warn( json.dumps( parsedLeaders,
801 sort_keys=True,
802 indent=4,
803 separators=( ',', ': ' ) ) )
804 # check for all intent partitions
805 # check for election
806 topics = []
807 for i in range( 14 ):
808 topics.append( "intent-partition-" + str( i ) )
809 # FIXME: this should only be after we start the app
810 topics.append( "org.onosproject.election" )
811 main.log.debug( topics )
812 ONOStopics = [ j['topic'] for j in parsedLeaders ]
813 for topic in topics:
814 if topic not in ONOStopics:
815 main.log.error( "Error: " + topic +
816 " not in leaders" )
817 missing = True
818 else:
819 main.log.error( "leaders() returned None" )
820 except ( ValueError, TypeError ):
821 main.log.exception( "Error parsing leaders" )
822 main.log.error( repr( leaders ) )
823 # Check all nodes
824 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700825 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700826 response = node.leaders( jsonFormat=False)
827 main.log.warn( str( node.name ) + " leaders output: \n" +
828 str( response ) )
829
830 partitions = main.ONOScli1.partitions()
831 try:
832 if partitions :
833 parsedPartitions = json.loads( partitions )
834 main.log.warn( json.dumps( parsedPartitions,
835 sort_keys=True,
836 indent=4,
837 separators=( ',', ': ' ) ) )
838 # TODO check for a leader in all paritions
839 # TODO check for consistency among nodes
840 else:
841 main.log.error( "partitions() returned None" )
842 except ( ValueError, TypeError ):
843 main.log.exception( "Error parsing partitions" )
844 main.log.error( repr( partitions ) )
845 pendingMap = main.ONOScli1.pendingMap()
846 try:
847 if pendingMap :
848 parsedPending = json.loads( pendingMap )
849 main.log.warn( json.dumps( parsedPending,
850 sort_keys=True,
851 indent=4,
852 separators=( ',', ': ' ) ) )
853 # TODO check something here?
854 else:
855 main.log.error( "pendingMap() returned None" )
856 except ( ValueError, TypeError ):
857 main.log.exception( "Error parsing pending map" )
858 main.log.error( repr( pendingMap ) )
859
860 def CASE4( self, main ):
861 """
862 Ping across added host intents
863 """
864 import json
865 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700866 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700867 assert main, "main not defined"
868 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700869 assert main.CLIs, "main.CLIs not defined"
870 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700871 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700872 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700873 "functionality and check the state of " +\
874 "the intent"
875 main.step( "Ping across added host intents" )
876 PingResult = main.TRUE
877 for i in range( 8, 18 ):
878 ping = main.Mininet1.pingHost( src="h" + str( i ),
879 target="h" + str( i + 10 ) )
880 PingResult = PingResult and ping
881 if ping == main.FALSE:
882 main.log.warn( "Ping failed between h" + str( i ) +
883 " and h" + str( i + 10 ) )
884 elif ping == main.TRUE:
885 main.log.info( "Ping test passed!" )
886 # Don't set PingResult or you'd override failures
887 if PingResult == main.FALSE:
888 main.log.error(
889 "Intents have not been installed correctly, pings failed." )
890 # TODO: pretty print
891 main.log.warn( "ONOS1 intents: " )
892 try:
893 tmpIntents = main.ONOScli1.intents()
894 main.log.warn( json.dumps( json.loads( tmpIntents ),
895 sort_keys=True,
896 indent=4,
897 separators=( ',', ': ' ) ) )
898 except ( ValueError, TypeError ):
899 main.log.warn( repr( tmpIntents ) )
900 utilities.assert_equals(
901 expect=main.TRUE,
902 actual=PingResult,
903 onpass="Intents have been installed correctly and pings work",
904 onfail="Intents have not been installed correctly, pings failed." )
905
906 main.step( "Check Intent state" )
907 installedCheck = False
908 loopCount = 0
909 while not installedCheck and loopCount < 40:
910 installedCheck = True
911 # Print the intent states
912 intents = main.ONOScli1.intents()
913 intentStates = []
914 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700915 count = 0
Jon Hall5cf14d52015-07-16 12:15:19 -0700916 # Iter through intents of a node
917 try:
918 for intent in json.loads( intents ):
919 state = intent.get( 'state', None )
920 if "INSTALLED" not in state:
921 installedCheck = False
922 intentId = intent.get( 'id', None )
923 intentStates.append( ( intentId, state ) )
924 except ( ValueError, TypeError ):
925 main.log.exception( "Error parsing intents." )
926 # Print states
927 intentStates.sort()
928 for i, s in intentStates:
929 count += 1
930 main.log.info( "%-6s%-15s%-15s" %
931 ( str( count ), str( i ), str( s ) ) )
932 if not installedCheck:
933 time.sleep( 1 )
934 loopCount += 1
935 utilities.assert_equals( expect=True, actual=installedCheck,
936 onpass="Intents are all INSTALLED",
937 onfail="Intents are not all in " +
938 "INSTALLED state" )
939
940 main.step( "Check leadership of topics" )
941 leaders = main.ONOScli1.leaders()
942 topicCheck = main.TRUE
943 try:
944 if leaders:
945 parsedLeaders = json.loads( leaders )
946 main.log.warn( json.dumps( parsedLeaders,
947 sort_keys=True,
948 indent=4,
949 separators=( ',', ': ' ) ) )
950 # check for all intent partitions
951 # check for election
952 # TODO: Look at Devices as topics now that it uses this system
953 topics = []
954 for i in range( 14 ):
955 topics.append( "intent-partition-" + str( i ) )
956 # FIXME: this should only be after we start the app
957 # FIXME: topics.append( "org.onosproject.election" )
958 # Print leaders output
959 main.log.debug( topics )
960 ONOStopics = [ j['topic'] for j in parsedLeaders ]
961 for topic in topics:
962 if topic not in ONOStopics:
963 main.log.error( "Error: " + topic +
964 " not in leaders" )
965 topicCheck = main.FALSE
966 else:
967 main.log.error( "leaders() returned None" )
968 topicCheck = main.FALSE
969 except ( ValueError, TypeError ):
970 topicCheck = main.FALSE
971 main.log.exception( "Error parsing leaders" )
972 main.log.error( repr( leaders ) )
973 # TODO: Check for a leader of these topics
974 # Check all nodes
975 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700976 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700977 response = node.leaders( jsonFormat=False)
978 main.log.warn( str( node.name ) + " leaders output: \n" +
979 str( response ) )
980
981 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
982 onpass="intent Partitions is in leaders",
983 onfail="Some topics were lost " )
984 # Print partitions
985 partitions = main.ONOScli1.partitions()
986 try:
987 if partitions :
988 parsedPartitions = json.loads( partitions )
989 main.log.warn( json.dumps( parsedPartitions,
990 sort_keys=True,
991 indent=4,
992 separators=( ',', ': ' ) ) )
993 # TODO check for a leader in all paritions
994 # TODO check for consistency among nodes
995 else:
996 main.log.error( "partitions() returned None" )
997 except ( ValueError, TypeError ):
998 main.log.exception( "Error parsing partitions" )
999 main.log.error( repr( partitions ) )
1000 # Print Pending Map
1001 pendingMap = main.ONOScli1.pendingMap()
1002 try:
1003 if pendingMap :
1004 parsedPending = json.loads( pendingMap )
1005 main.log.warn( json.dumps( parsedPending,
1006 sort_keys=True,
1007 indent=4,
1008 separators=( ',', ': ' ) ) )
1009 # TODO check something here?
1010 else:
1011 main.log.error( "pendingMap() returned None" )
1012 except ( ValueError, TypeError ):
1013 main.log.exception( "Error parsing pending map" )
1014 main.log.error( repr( pendingMap ) )
1015
1016 if not installedCheck:
1017 main.log.info( "Waiting 60 seconds to see if the state of " +
1018 "intents change" )
1019 time.sleep( 60 )
1020 # Print the intent states
1021 intents = main.ONOScli1.intents()
1022 intentStates = []
1023 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1024 count = 0
1025 # Iter through intents of a node
1026 try:
1027 for intent in json.loads( intents ):
1028 state = intent.get( 'state', None )
1029 if "INSTALLED" not in state:
1030 installedCheck = False
1031 intentId = intent.get( 'id', None )
1032 intentStates.append( ( intentId, state ) )
1033 except ( ValueError, TypeError ):
1034 main.log.exception( "Error parsing intents." )
1035 intentStates.sort()
1036 for i, s in intentStates:
1037 count += 1
1038 main.log.info( "%-6s%-15s%-15s" %
1039 ( str( count ), str( i ), str( s ) ) )
1040 leaders = main.ONOScli1.leaders()
1041 try:
1042 missing = False
1043 if leaders:
1044 parsedLeaders = json.loads( leaders )
1045 main.log.warn( json.dumps( parsedLeaders,
1046 sort_keys=True,
1047 indent=4,
1048 separators=( ',', ': ' ) ) )
1049 # check for all intent partitions
1050 # check for election
1051 topics = []
1052 for i in range( 14 ):
1053 topics.append( "intent-partition-" + str( i ) )
1054 # FIXME: this should only be after we start the app
1055 topics.append( "org.onosproject.election" )
1056 main.log.debug( topics )
1057 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1058 for topic in topics:
1059 if topic not in ONOStopics:
1060 main.log.error( "Error: " + topic +
1061 " not in leaders" )
1062 missing = True
1063 else:
1064 main.log.error( "leaders() returned None" )
1065 except ( ValueError, TypeError ):
1066 main.log.exception( "Error parsing leaders" )
1067 main.log.error( repr( leaders ) )
1068 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001069 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001070 response = node.leaders( jsonFormat=False)
1071 main.log.warn( str( node.name ) + " leaders output: \n" +
1072 str( response ) )
1073
1074 partitions = main.ONOScli1.partitions()
1075 try:
1076 if partitions :
1077 parsedPartitions = json.loads( partitions )
1078 main.log.warn( json.dumps( parsedPartitions,
1079 sort_keys=True,
1080 indent=4,
1081 separators=( ',', ': ' ) ) )
1082 # TODO check for a leader in all paritions
1083 # TODO check for consistency among nodes
1084 else:
1085 main.log.error( "partitions() returned None" )
1086 except ( ValueError, TypeError ):
1087 main.log.exception( "Error parsing partitions" )
1088 main.log.error( repr( partitions ) )
1089 pendingMap = main.ONOScli1.pendingMap()
1090 try:
1091 if pendingMap :
1092 parsedPending = json.loads( pendingMap )
1093 main.log.warn( json.dumps( parsedPending,
1094 sort_keys=True,
1095 indent=4,
1096 separators=( ',', ': ' ) ) )
1097 # TODO check something here?
1098 else:
1099 main.log.error( "pendingMap() returned None" )
1100 except ( ValueError, TypeError ):
1101 main.log.exception( "Error parsing pending map" )
1102 main.log.error( repr( pendingMap ) )
1103 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001104 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001105 main.step( "Wait a minute then ping again" )
1106 # the wait is above
1107 PingResult = main.TRUE
1108 for i in range( 8, 18 ):
1109 ping = main.Mininet1.pingHost( src="h" + str( i ),
1110 target="h" + str( i + 10 ) )
1111 PingResult = PingResult and ping
1112 if ping == main.FALSE:
1113 main.log.warn( "Ping failed between h" + str( i ) +
1114 " and h" + str( i + 10 ) )
1115 elif ping == main.TRUE:
1116 main.log.info( "Ping test passed!" )
1117 # Don't set PingResult or you'd override failures
1118 if PingResult == main.FALSE:
1119 main.log.error(
1120 "Intents have not been installed correctly, pings failed." )
1121 # TODO: pretty print
1122 main.log.warn( "ONOS1 intents: " )
1123 try:
1124 tmpIntents = main.ONOScli1.intents()
1125 main.log.warn( json.dumps( json.loads( tmpIntents ),
1126 sort_keys=True,
1127 indent=4,
1128 separators=( ',', ': ' ) ) )
1129 except ( ValueError, TypeError ):
1130 main.log.warn( repr( tmpIntents ) )
1131 utilities.assert_equals(
1132 expect=main.TRUE,
1133 actual=PingResult,
1134 onpass="Intents have been installed correctly and pings work",
1135 onfail="Intents have not been installed correctly, pings failed." )
1136
1137 def CASE5( self, main ):
1138 """
1139 Reading state of ONOS
1140 """
1141 import json
1142 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001143 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001144 assert main, "main not defined"
1145 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001146 assert main.CLIs, "main.CLIs not defined"
1147 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001148
1149 main.case( "Setting up and gathering data for current state" )
1150 # The general idea for this test case is to pull the state of
1151 # ( intents,flows, topology,... ) from each ONOS node
1152 # We can then compare them with each other and also with past states
1153
1154 main.step( "Check that each switch has a master" )
1155 global mastershipState
1156 mastershipState = '[]'
1157
1158 # Assert that each device has a master
1159 rolesNotNull = main.TRUE
1160 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001161 for i in range( main.numCtrls ):
1162 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001163 name="rolesNotNull-" + str( i ),
1164 args=[] )
1165 threads.append( t )
1166 t.start()
1167
1168 for t in threads:
1169 t.join()
1170 rolesNotNull = rolesNotNull and t.result
1171 utilities.assert_equals(
1172 expect=main.TRUE,
1173 actual=rolesNotNull,
1174 onpass="Each device has a master",
1175 onfail="Some devices don't have a master assigned" )
1176
1177 main.step( "Get the Mastership of each switch from each controller" )
1178 ONOSMastership = []
1179 mastershipCheck = main.FALSE
1180 consistentMastership = True
1181 rolesResults = True
1182 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001183 for i in range( main.numCtrls ):
1184 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001185 name="roles-" + str( i ),
1186 args=[] )
1187 threads.append( t )
1188 t.start()
1189
1190 for t in threads:
1191 t.join()
1192 ONOSMastership.append( t.result )
1193
Jon Halle1a3b752015-07-22 13:02:46 -07001194 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001195 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1196 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1197 " roles" )
1198 main.log.warn(
1199 "ONOS" + str( i + 1 ) + " mastership response: " +
1200 repr( ONOSMastership[i] ) )
1201 rolesResults = False
1202 utilities.assert_equals(
1203 expect=True,
1204 actual=rolesResults,
1205 onpass="No error in reading roles output",
1206 onfail="Error in reading roles from ONOS" )
1207
1208 main.step( "Check for consistency in roles from each controller" )
1209 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1210 main.log.info(
1211 "Switch roles are consistent across all ONOS nodes" )
1212 else:
1213 consistentMastership = False
1214 utilities.assert_equals(
1215 expect=True,
1216 actual=consistentMastership,
1217 onpass="Switch roles are consistent across all ONOS nodes",
1218 onfail="ONOS nodes have different views of switch roles" )
1219
1220 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001221 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001222 try:
1223 main.log.warn(
1224 "ONOS" + str( i + 1 ) + " roles: ",
1225 json.dumps(
1226 json.loads( ONOSMastership[ i ] ),
1227 sort_keys=True,
1228 indent=4,
1229 separators=( ',', ': ' ) ) )
1230 except ( ValueError, TypeError ):
1231 main.log.warn( repr( ONOSMastership[ i ] ) )
1232 elif rolesResults and consistentMastership:
1233 mastershipCheck = main.TRUE
1234 mastershipState = ONOSMastership[ 0 ]
1235
1236 main.step( "Get the intents from each controller" )
1237 global intentState
1238 intentState = []
1239 ONOSIntents = []
1240 intentCheck = main.FALSE
1241 consistentIntents = True
1242 intentsResults = True
1243 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001244 for i in range( main.numCtrls ):
1245 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001246 name="intents-" + str( i ),
1247 args=[],
1248 kwargs={ 'jsonFormat': True } )
1249 threads.append( t )
1250 t.start()
1251
1252 for t in threads:
1253 t.join()
1254 ONOSIntents.append( t.result )
1255
Jon Halle1a3b752015-07-22 13:02:46 -07001256 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001257 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1258 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1259 " intents" )
1260 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1261 repr( ONOSIntents[ i ] ) )
1262 intentsResults = False
1263 utilities.assert_equals(
1264 expect=True,
1265 actual=intentsResults,
1266 onpass="No error in reading intents output",
1267 onfail="Error in reading intents from ONOS" )
1268
1269 main.step( "Check for consistency in Intents from each controller" )
1270 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1271 main.log.info( "Intents are consistent across all ONOS " +
1272 "nodes" )
1273 else:
1274 consistentIntents = False
1275 main.log.error( "Intents not consistent" )
1276 utilities.assert_equals(
1277 expect=True,
1278 actual=consistentIntents,
1279 onpass="Intents are consistent across all ONOS nodes",
1280 onfail="ONOS nodes have different views of intents" )
1281
1282 if intentsResults:
1283 # Try to make it easy to figure out what is happening
1284 #
1285 # Intent ONOS1 ONOS2 ...
1286 # 0x01 INSTALLED INSTALLING
1287 # ... ... ...
1288 # ... ... ...
1289 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001290 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001291 title += " " * 10 + "ONOS" + str( n + 1 )
1292 main.log.warn( title )
1293 # get all intent keys in the cluster
1294 keys = []
1295 for nodeStr in ONOSIntents:
1296 node = json.loads( nodeStr )
1297 for intent in node:
1298 keys.append( intent.get( 'id' ) )
1299 keys = set( keys )
1300 for key in keys:
1301 row = "%-13s" % key
1302 for nodeStr in ONOSIntents:
1303 node = json.loads( nodeStr )
1304 for intent in node:
1305 if intent.get( 'id', "Error" ) == key:
1306 row += "%-15s" % intent.get( 'state' )
1307 main.log.warn( row )
1308 # End table view
1309
1310 if intentsResults and not consistentIntents:
1311 # print the json objects
1312 n = len(ONOSIntents)
1313 main.log.debug( "ONOS" + str( n ) + " intents: " )
1314 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1315 sort_keys=True,
1316 indent=4,
1317 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001318 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001319 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1320 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1321 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1322 sort_keys=True,
1323 indent=4,
1324 separators=( ',', ': ' ) ) )
1325 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001326 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001327 str( n ) + " intents" )
1328 elif intentsResults and consistentIntents:
1329 intentCheck = main.TRUE
1330 intentState = ONOSIntents[ 0 ]
1331
1332 main.step( "Get the flows from each controller" )
1333 global flowState
1334 flowState = []
1335 ONOSFlows = []
1336 ONOSFlowsJson = []
1337 flowCheck = main.FALSE
1338 consistentFlows = True
1339 flowsResults = True
1340 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001341 for i in range( main.numCtrls ):
1342 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001343 name="flows-" + str( i ),
1344 args=[],
1345 kwargs={ 'jsonFormat': True } )
1346 threads.append( t )
1347 t.start()
1348
1349 # NOTE: Flows command can take some time to run
1350 time.sleep(30)
1351 for t in threads:
1352 t.join()
1353 result = t.result
1354 ONOSFlows.append( result )
1355
Jon Halle1a3b752015-07-22 13:02:46 -07001356 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001357 num = str( i + 1 )
1358 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1359 main.log.error( "Error in getting ONOS" + num + " flows" )
1360 main.log.warn( "ONOS" + num + " flows response: " +
1361 repr( ONOSFlows[ i ] ) )
1362 flowsResults = False
1363 ONOSFlowsJson.append( None )
1364 else:
1365 try:
1366 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1367 except ( ValueError, TypeError ):
1368 # FIXME: change this to log.error?
1369 main.log.exception( "Error in parsing ONOS" + num +
1370 " response as json." )
1371 main.log.error( repr( ONOSFlows[ i ] ) )
1372 ONOSFlowsJson.append( None )
1373 flowsResults = False
1374 utilities.assert_equals(
1375 expect=True,
1376 actual=flowsResults,
1377 onpass="No error in reading flows output",
1378 onfail="Error in reading flows from ONOS" )
1379
1380 main.step( "Check for consistency in Flows from each controller" )
1381 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1382 if all( tmp ):
1383 main.log.info( "Flow count is consistent across all ONOS nodes" )
1384 else:
1385 consistentFlows = False
1386 utilities.assert_equals(
1387 expect=True,
1388 actual=consistentFlows,
1389 onpass="The flow count is consistent across all ONOS nodes",
1390 onfail="ONOS nodes have different flow counts" )
1391
1392 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001393 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001394 try:
1395 main.log.warn(
1396 "ONOS" + str( i + 1 ) + " flows: " +
1397 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1398 indent=4, separators=( ',', ': ' ) ) )
1399 except ( ValueError, TypeError ):
1400 main.log.warn(
1401 "ONOS" + str( i + 1 ) + " flows: " +
1402 repr( ONOSFlows[ i ] ) )
1403 elif flowsResults and consistentFlows:
1404 flowCheck = main.TRUE
1405 flowState = ONOSFlows[ 0 ]
1406
1407 main.step( "Get the OF Table entries" )
1408 global flows
1409 flows = []
1410 for i in range( 1, 29 ):
Jon Hall9043c902015-07-30 14:23:44 -07001411 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001412 if flowCheck == main.FALSE:
1413 for table in flows:
1414 main.log.warn( table )
1415 # TODO: Compare switch flow tables with ONOS flow tables
1416
1417 main.step( "Start continuous pings" )
1418 main.Mininet2.pingLong(
1419 src=main.params[ 'PING' ][ 'source1' ],
1420 target=main.params[ 'PING' ][ 'target1' ],
1421 pingTime=500 )
1422 main.Mininet2.pingLong(
1423 src=main.params[ 'PING' ][ 'source2' ],
1424 target=main.params[ 'PING' ][ 'target2' ],
1425 pingTime=500 )
1426 main.Mininet2.pingLong(
1427 src=main.params[ 'PING' ][ 'source3' ],
1428 target=main.params[ 'PING' ][ 'target3' ],
1429 pingTime=500 )
1430 main.Mininet2.pingLong(
1431 src=main.params[ 'PING' ][ 'source4' ],
1432 target=main.params[ 'PING' ][ 'target4' ],
1433 pingTime=500 )
1434 main.Mininet2.pingLong(
1435 src=main.params[ 'PING' ][ 'source5' ],
1436 target=main.params[ 'PING' ][ 'target5' ],
1437 pingTime=500 )
1438 main.Mininet2.pingLong(
1439 src=main.params[ 'PING' ][ 'source6' ],
1440 target=main.params[ 'PING' ][ 'target6' ],
1441 pingTime=500 )
1442 main.Mininet2.pingLong(
1443 src=main.params[ 'PING' ][ 'source7' ],
1444 target=main.params[ 'PING' ][ 'target7' ],
1445 pingTime=500 )
1446 main.Mininet2.pingLong(
1447 src=main.params[ 'PING' ][ 'source8' ],
1448 target=main.params[ 'PING' ][ 'target8' ],
1449 pingTime=500 )
1450 main.Mininet2.pingLong(
1451 src=main.params[ 'PING' ][ 'source9' ],
1452 target=main.params[ 'PING' ][ 'target9' ],
1453 pingTime=500 )
1454 main.Mininet2.pingLong(
1455 src=main.params[ 'PING' ][ 'source10' ],
1456 target=main.params[ 'PING' ][ 'target10' ],
1457 pingTime=500 )
1458
1459 main.step( "Collecting topology information from ONOS" )
1460 devices = []
1461 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001462 for i in range( main.numCtrls ):
1463 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001464 name="devices-" + str( i ),
1465 args=[ ] )
1466 threads.append( t )
1467 t.start()
1468
1469 for t in threads:
1470 t.join()
1471 devices.append( t.result )
1472 hosts = []
1473 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001474 for i in range( main.numCtrls ):
1475 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001476 name="hosts-" + str( i ),
1477 args=[ ] )
1478 threads.append( t )
1479 t.start()
1480
1481 for t in threads:
1482 t.join()
1483 try:
1484 hosts.append( json.loads( t.result ) )
1485 except ( ValueError, TypeError ):
1486 # FIXME: better handling of this, print which node
1487 # Maybe use thread name?
1488 main.log.exception( "Error parsing json output of hosts" )
1489 # FIXME: should this be an empty json object instead?
1490 hosts.append( None )
1491
1492 ports = []
1493 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001494 for i in range( main.numCtrls ):
1495 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001496 name="ports-" + str( i ),
1497 args=[ ] )
1498 threads.append( t )
1499 t.start()
1500
1501 for t in threads:
1502 t.join()
1503 ports.append( t.result )
1504 links = []
1505 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001506 for i in range( main.numCtrls ):
1507 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001508 name="links-" + str( i ),
1509 args=[ ] )
1510 threads.append( t )
1511 t.start()
1512
1513 for t in threads:
1514 t.join()
1515 links.append( t.result )
1516 clusters = []
1517 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001518 for i in range( main.numCtrls ):
1519 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001520 name="clusters-" + str( i ),
1521 args=[ ] )
1522 threads.append( t )
1523 t.start()
1524
1525 for t in threads:
1526 t.join()
1527 clusters.append( t.result )
1528 # Compare json objects for hosts and dataplane clusters
1529
1530 # hosts
1531 main.step( "Host view is consistent across ONOS nodes" )
1532 consistentHostsResult = main.TRUE
1533 for controller in range( len( hosts ) ):
1534 controllerStr = str( controller + 1 )
1535 if "Error" not in hosts[ controller ]:
1536 if hosts[ controller ] == hosts[ 0 ]:
1537 continue
1538 else: # hosts not consistent
1539 main.log.error( "hosts from ONOS" +
1540 controllerStr +
1541 " is inconsistent with ONOS1" )
1542 main.log.warn( repr( hosts[ controller ] ) )
1543 consistentHostsResult = main.FALSE
1544
1545 else:
1546 main.log.error( "Error in getting ONOS hosts from ONOS" +
1547 controllerStr )
1548 consistentHostsResult = main.FALSE
1549 main.log.warn( "ONOS" + controllerStr +
1550 " hosts response: " +
1551 repr( hosts[ controller ] ) )
1552 utilities.assert_equals(
1553 expect=main.TRUE,
1554 actual=consistentHostsResult,
1555 onpass="Hosts view is consistent across all ONOS nodes",
1556 onfail="ONOS nodes have different views of hosts" )
1557
1558 main.step( "Each host has an IP address" )
1559 ipResult = main.TRUE
1560 for controller in range( 0, len( hosts ) ):
1561 controllerStr = str( controller + 1 )
1562 for host in hosts[ controller ]:
1563 if not host.get( 'ipAddresses', [ ] ):
1564 main.log.error( "DEBUG:Error with host ips on controller" +
1565 controllerStr + ": " + str( host ) )
1566 ipResult = main.FALSE
1567 utilities.assert_equals(
1568 expect=main.TRUE,
1569 actual=ipResult,
1570 onpass="The ips of the hosts aren't empty",
1571 onfail="The ip of at least one host is missing" )
1572
1573 # Strongly connected clusters of devices
1574 main.step( "Cluster view is consistent across ONOS nodes" )
1575 consistentClustersResult = main.TRUE
1576 for controller in range( len( clusters ) ):
1577 controllerStr = str( controller + 1 )
1578 if "Error" not in clusters[ controller ]:
1579 if clusters[ controller ] == clusters[ 0 ]:
1580 continue
1581 else: # clusters not consistent
1582 main.log.error( "clusters from ONOS" + controllerStr +
1583 " is inconsistent with ONOS1" )
1584 consistentClustersResult = main.FALSE
1585
1586 else:
1587 main.log.error( "Error in getting dataplane clusters " +
1588 "from ONOS" + controllerStr )
1589 consistentClustersResult = main.FALSE
1590 main.log.warn( "ONOS" + controllerStr +
1591 " clusters response: " +
1592 repr( clusters[ controller ] ) )
1593 utilities.assert_equals(
1594 expect=main.TRUE,
1595 actual=consistentClustersResult,
1596 onpass="Clusters view is consistent across all ONOS nodes",
1597 onfail="ONOS nodes have different views of clusters" )
1598 # there should always only be one cluster
1599 main.step( "Cluster view correct across ONOS nodes" )
1600 try:
1601 numClusters = len( json.loads( clusters[ 0 ] ) )
1602 except ( ValueError, TypeError ):
1603 main.log.exception( "Error parsing clusters[0]: " +
1604 repr( clusters[ 0 ] ) )
1605 clusterResults = main.FALSE
1606 if numClusters == 1:
1607 clusterResults = main.TRUE
1608 utilities.assert_equals(
1609 expect=1,
1610 actual=numClusters,
1611 onpass="ONOS shows 1 SCC",
1612 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1613
1614 main.step( "Comparing ONOS topology to MN" )
1615 devicesResults = main.TRUE
1616 linksResults = main.TRUE
1617 hostsResults = main.TRUE
1618 mnSwitches = main.Mininet1.getSwitches()
1619 mnLinks = main.Mininet1.getLinks()
1620 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001621 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001622 controllerStr = str( controller + 1 )
1623 if devices[ controller ] and ports[ controller ] and\
1624 "Error" not in devices[ controller ] and\
1625 "Error" not in ports[ controller ]:
Jon Halle1a3b752015-07-22 13:02:46 -07001626 currentDevicesResult = main.Mininet1.compareSwitches(
1627 mnSwitches,
1628 json.loads( devices[ controller ] ),
1629 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001630 else:
1631 currentDevicesResult = main.FALSE
1632 utilities.assert_equals( expect=main.TRUE,
1633 actual=currentDevicesResult,
1634 onpass="ONOS" + controllerStr +
1635 " Switches view is correct",
1636 onfail="ONOS" + controllerStr +
1637 " Switches view is incorrect" )
1638 if links[ controller ] and "Error" not in links[ controller ]:
1639 currentLinksResult = main.Mininet1.compareLinks(
1640 mnSwitches, mnLinks,
1641 json.loads( links[ controller ] ) )
1642 else:
1643 currentLinksResult = main.FALSE
1644 utilities.assert_equals( expect=main.TRUE,
1645 actual=currentLinksResult,
1646 onpass="ONOS" + controllerStr +
1647 " links view is correct",
1648 onfail="ONOS" + controllerStr +
1649 " links view is incorrect" )
1650
1651 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1652 currentHostsResult = main.Mininet1.compareHosts(
1653 mnHosts,
1654 hosts[ controller ] )
1655 else:
1656 currentHostsResult = main.FALSE
1657 utilities.assert_equals( expect=main.TRUE,
1658 actual=currentHostsResult,
1659 onpass="ONOS" + controllerStr +
1660 " hosts exist in Mininet",
1661 onfail="ONOS" + controllerStr +
1662 " hosts don't match Mininet" )
1663
1664 devicesResults = devicesResults and currentDevicesResult
1665 linksResults = linksResults and currentLinksResult
1666 hostsResults = hostsResults and currentHostsResult
1667
1668 main.step( "Device information is correct" )
1669 utilities.assert_equals(
1670 expect=main.TRUE,
1671 actual=devicesResults,
1672 onpass="Device information is correct",
1673 onfail="Device information is incorrect" )
1674
1675 main.step( "Links are correct" )
1676 utilities.assert_equals(
1677 expect=main.TRUE,
1678 actual=linksResults,
1679 onpass="Link are correct",
1680 onfail="Links are incorrect" )
1681
1682 main.step( "Hosts are correct" )
1683 utilities.assert_equals(
1684 expect=main.TRUE,
1685 actual=hostsResults,
1686 onpass="Hosts are correct",
1687 onfail="Hosts are incorrect" )
1688
1689 def CASE6( self, main ):
1690 """
1691 The Failure case.
1692 """
1693 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001694 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001695 assert main, "main not defined"
1696 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001697 assert main.CLIs, "main.CLIs not defined"
1698 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001699 try:
1700 labels
1701 except NameError:
1702 main.log.error( "labels not defined, setting to []" )
1703 global labels
1704 labels = []
1705 try:
1706 data
1707 except NameError:
1708 main.log.error( "data not defined, setting to []" )
1709 global data
1710 data = []
1711 # Reset non-persistent variables
1712 try:
1713 iCounterValue = 0
1714 except NameError:
1715 main.log.error( "iCounterValue not defined, setting to 0" )
1716 iCounterValue = 0
1717
1718 main.case( "Restart entire ONOS cluster" )
1719
1720 main.step( "Killing ONOS nodes" )
1721 killResults = main.TRUE
1722 killTime = time.time()
Jon Halle1a3b752015-07-22 13:02:46 -07001723 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001724 killed = main.ONOSbench.onosKill( node.ip_address )
1725 killResults = killResults and killed
1726 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1727 onpass="ONOS nodes killed",
1728 onfail="ONOS kill unsuccessful" )
1729
1730 main.step( "Checking if ONOS is up yet" )
1731 for i in range( 2 ):
1732 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001733 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001734 started = main.ONOSbench.isup( node.ip_address )
1735 if not started:
1736 main.log.error( node.name + " didn't start!" )
1737 onosIsupResult = onosIsupResult and started
1738 if onosIsupResult == main.TRUE:
1739 break
1740 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1741 onpass="ONOS restarted",
1742 onfail="ONOS restart NOT successful" )
1743
1744 main.log.step( "Starting ONOS CLI sessions" )
1745 cliResults = main.TRUE
1746 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001747 for i in range( main.numCtrls ):
1748 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -07001749 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -07001750 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001751 threads.append( t )
1752 t.start()
1753
1754 for t in threads:
1755 t.join()
1756 cliResults = cliResults and t.result
1757 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1758 onpass="ONOS cli started",
1759 onfail="ONOS clis did not restart" )
1760
1761 # Grab the time of restart so we chan check how long the gossip
1762 # protocol has had time to work
1763 main.restartTime = time.time() - killTime
1764 main.log.debug( "Restart time: " + str( main.restartTime ) )
1765 labels.append( "Restart" )
1766 data.append( str( main.restartTime ) )
1767
1768 # FIXME: revisit test plan for election with madan
1769 # Rerun for election on restarted nodes
1770 runResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001771 for cli in main.CLIs:
1772 run = main.CLIs[0].electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001773 if run != main.TRUE:
1774 main.log.error( "Error running for election on " + cli.name )
1775 runResults = runResults and run
1776 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1777 onpass="Reran for election",
1778 onfail="Failed to rerun for election" )
1779
1780 # TODO: Make this configurable
1781 time.sleep( 60 )
Jon Halle1a3b752015-07-22 13:02:46 -07001782 main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) )
1783 main.log.debug( main.CLIs[0].leaders( jsonFormat=False ) )
1784 main.log.debug( main.CLIs[0].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001785
1786 def CASE7( self, main ):
1787 """
1788 Check state after ONOS failure
1789 """
1790 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001791 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001792 assert main, "main not defined"
1793 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001794 assert main.CLIs, "main.CLIs not defined"
1795 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001796 main.case( "Running ONOS Constant State Tests" )
1797
1798 main.step( "Check that each switch has a master" )
1799 # Assert that each device has a master
1800 rolesNotNull = main.TRUE
1801 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001802 for i in range( main.numCtrls ):
1803 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001804 name="rolesNotNull-" + str( i ),
1805 args=[ ] )
1806 threads.append( t )
1807 t.start()
1808
1809 for t in threads:
1810 t.join()
1811 rolesNotNull = rolesNotNull and t.result
1812 utilities.assert_equals(
1813 expect=main.TRUE,
1814 actual=rolesNotNull,
1815 onpass="Each device has a master",
1816 onfail="Some devices don't have a master assigned" )
1817
1818 main.step( "Read device roles from ONOS" )
1819 ONOSMastership = []
1820 mastershipCheck = main.FALSE
1821 consistentMastership = True
1822 rolesResults = True
1823 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001824 for i in range( main.numCtrls ):
1825 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001826 name="roles-" + str( i ),
1827 args=[] )
1828 threads.append( t )
1829 t.start()
1830
1831 for t in threads:
1832 t.join()
1833 ONOSMastership.append( t.result )
1834
Jon Halle1a3b752015-07-22 13:02:46 -07001835 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001836 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1837 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1838 " roles" )
1839 main.log.warn(
1840 "ONOS" + str( i + 1 ) + " mastership response: " +
1841 repr( ONOSMastership[i] ) )
1842 rolesResults = False
1843 utilities.assert_equals(
1844 expect=True,
1845 actual=rolesResults,
1846 onpass="No error in reading roles output",
1847 onfail="Error in reading roles from ONOS" )
1848
1849 main.step( "Check for consistency in roles from each controller" )
1850 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1851 main.log.info(
1852 "Switch roles are consistent across all ONOS nodes" )
1853 else:
1854 consistentMastership = False
1855 utilities.assert_equals(
1856 expect=True,
1857 actual=consistentMastership,
1858 onpass="Switch roles are consistent across all ONOS nodes",
1859 onfail="ONOS nodes have different views of switch roles" )
1860
1861 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001862 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001863 main.log.warn(
1864 "ONOS" + str( i + 1 ) + " roles: ",
1865 json.dumps(
1866 json.loads( ONOSMastership[ i ] ),
1867 sort_keys=True,
1868 indent=4,
1869 separators=( ',', ': ' ) ) )
1870 elif rolesResults and not consistentMastership:
1871 mastershipCheck = main.TRUE
1872
1873 '''
1874 description2 = "Compare switch roles from before failure"
1875 main.step( description2 )
1876 try:
1877 currentJson = json.loads( ONOSMastership[0] )
1878 oldJson = json.loads( mastershipState )
1879 except ( ValueError, TypeError ):
1880 main.log.exception( "Something is wrong with parsing " +
1881 "ONOSMastership[0] or mastershipState" )
1882 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1883 main.log.error( "mastershipState" + repr( mastershipState ) )
1884 main.cleanup()
1885 main.exit()
1886 mastershipCheck = main.TRUE
1887 for i in range( 1, 29 ):
1888 switchDPID = str(
1889 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1890 current = [ switch[ 'master' ] for switch in currentJson
1891 if switchDPID in switch[ 'id' ] ]
1892 old = [ switch[ 'master' ] for switch in oldJson
1893 if switchDPID in switch[ 'id' ] ]
1894 if current == old:
1895 mastershipCheck = mastershipCheck and main.TRUE
1896 else:
1897 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1898 mastershipCheck = main.FALSE
1899 utilities.assert_equals(
1900 expect=main.TRUE,
1901 actual=mastershipCheck,
1902 onpass="Mastership of Switches was not changed",
1903 onfail="Mastership of some switches changed" )
1904 '''
1905 # NOTE: we expect mastership to change on controller failure
1906
1907 main.step( "Get the intents and compare across all nodes" )
1908 ONOSIntents = []
1909 intentCheck = main.FALSE
1910 consistentIntents = True
1911 intentsResults = True
1912 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001913 for i in range( main.numCtrls ):
1914 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001915 name="intents-" + str( i ),
1916 args=[],
1917 kwargs={ 'jsonFormat': True } )
1918 threads.append( t )
1919 t.start()
1920
1921 for t in threads:
1922 t.join()
1923 ONOSIntents.append( t.result )
1924
Jon Halle1a3b752015-07-22 13:02:46 -07001925 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001926 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1927 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1928 " intents" )
1929 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1930 repr( ONOSIntents[ i ] ) )
1931 intentsResults = False
1932 utilities.assert_equals(
1933 expect=True,
1934 actual=intentsResults,
1935 onpass="No error in reading intents output",
1936 onfail="Error in reading intents from ONOS" )
1937
1938 main.step( "Check for consistency in Intents from each controller" )
1939 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1940 main.log.info( "Intents are consistent across all ONOS " +
1941 "nodes" )
1942 else:
1943 consistentIntents = False
1944
1945 # Try to make it easy to figure out what is happening
1946 #
1947 # Intent ONOS1 ONOS2 ...
1948 # 0x01 INSTALLED INSTALLING
1949 # ... ... ...
1950 # ... ... ...
1951 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001952 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001953 title += " " * 10 + "ONOS" + str( n + 1 )
1954 main.log.warn( title )
1955 # get all intent keys in the cluster
1956 keys = []
1957 for nodeStr in ONOSIntents:
1958 node = json.loads( nodeStr )
1959 for intent in node:
1960 keys.append( intent.get( 'id' ) )
1961 keys = set( keys )
1962 for key in keys:
1963 row = "%-13s" % key
1964 for nodeStr in ONOSIntents:
1965 node = json.loads( nodeStr )
1966 for intent in node:
1967 if intent.get( 'id' ) == key:
1968 row += "%-15s" % intent.get( 'state' )
1969 main.log.warn( row )
1970 # End table view
1971
1972 utilities.assert_equals(
1973 expect=True,
1974 actual=consistentIntents,
1975 onpass="Intents are consistent across all ONOS nodes",
1976 onfail="ONOS nodes have different views of intents" )
1977 intentStates = []
1978 for node in ONOSIntents: # Iter through ONOS nodes
1979 nodeStates = []
1980 # Iter through intents of a node
1981 try:
1982 for intent in json.loads( node ):
1983 nodeStates.append( intent[ 'state' ] )
1984 except ( ValueError, TypeError ):
1985 main.log.exception( "Error in parsing intents" )
1986 main.log.error( repr( node ) )
1987 intentStates.append( nodeStates )
1988 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1989 main.log.info( dict( out ) )
1990
1991 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001992 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001993 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1994 main.log.warn( json.dumps(
1995 json.loads( ONOSIntents[ i ] ),
1996 sort_keys=True,
1997 indent=4,
1998 separators=( ',', ': ' ) ) )
1999 elif intentsResults and consistentIntents:
2000 intentCheck = main.TRUE
2001
2002 # NOTE: Store has no durability, so intents are lost across system
2003 # restarts
2004 """
2005 main.step( "Compare current intents with intents before the failure" )
2006 # NOTE: this requires case 5 to pass for intentState to be set.
2007 # maybe we should stop the test if that fails?
2008 sameIntents = main.FALSE
2009 if intentState and intentState == ONOSIntents[ 0 ]:
2010 sameIntents = main.TRUE
2011 main.log.info( "Intents are consistent with before failure" )
2012 # TODO: possibly the states have changed? we may need to figure out
2013 # what the acceptable states are
2014 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2015 sameIntents = main.TRUE
2016 try:
2017 before = json.loads( intentState )
2018 after = json.loads( ONOSIntents[ 0 ] )
2019 for intent in before:
2020 if intent not in after:
2021 sameIntents = main.FALSE
2022 main.log.debug( "Intent is not currently in ONOS " +
2023 "(at least in the same form):" )
2024 main.log.debug( json.dumps( intent ) )
2025 except ( ValueError, TypeError ):
2026 main.log.exception( "Exception printing intents" )
2027 main.log.debug( repr( ONOSIntents[0] ) )
2028 main.log.debug( repr( intentState ) )
2029 if sameIntents == main.FALSE:
2030 try:
2031 main.log.debug( "ONOS intents before: " )
2032 main.log.debug( json.dumps( json.loads( intentState ),
2033 sort_keys=True, indent=4,
2034 separators=( ',', ': ' ) ) )
2035 main.log.debug( "Current ONOS intents: " )
2036 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2037 sort_keys=True, indent=4,
2038 separators=( ',', ': ' ) ) )
2039 except ( ValueError, TypeError ):
2040 main.log.exception( "Exception printing intents" )
2041 main.log.debug( repr( ONOSIntents[0] ) )
2042 main.log.debug( repr( intentState ) )
2043 utilities.assert_equals(
2044 expect=main.TRUE,
2045 actual=sameIntents,
2046 onpass="Intents are consistent with before failure",
2047 onfail="The Intents changed during failure" )
2048 intentCheck = intentCheck and sameIntents
2049 """
2050 main.step( "Get the OF Table entries and compare to before " +
2051 "component failure" )
2052 FlowTables = main.TRUE
2053 flows2 = []
2054 for i in range( 28 ):
2055 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall9043c902015-07-30 14:23:44 -07002056 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002057 flows2.append( tmpFlows )
Jon Hall9043c902015-07-30 14:23:44 -07002058 tempResult = main.Mininet1.flowComp(
Jon Hall5cf14d52015-07-16 12:15:19 -07002059 flow1=flows[ i ],
2060 flow2=tmpFlows )
2061 FlowTables = FlowTables and tempResult
2062 if FlowTables == main.FALSE:
2063 main.log.info( "Differences in flow table for switch: s" +
2064 str( i + 1 ) )
2065 utilities.assert_equals(
2066 expect=main.TRUE,
2067 actual=FlowTables,
2068 onpass="No changes were found in the flow tables",
2069 onfail="Changes were found in the flow tables" )
2070
2071 main.Mininet2.pingLongKill()
2072 '''
2073 # main.step( "Check the continuous pings to ensure that no packets " +
2074 # "were dropped during component failure" )
2075 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2076 main.params[ 'TESTONIP' ] )
2077 LossInPings = main.FALSE
2078 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2079 for i in range( 8, 18 ):
2080 main.log.info(
2081 "Checking for a loss in pings along flow from s" +
2082 str( i ) )
2083 LossInPings = main.Mininet2.checkForLoss(
2084 "/tmp/ping.h" +
2085 str( i ) ) or LossInPings
2086 if LossInPings == main.TRUE:
2087 main.log.info( "Loss in ping detected" )
2088 elif LossInPings == main.ERROR:
2089 main.log.info( "There are multiple mininet process running" )
2090 elif LossInPings == main.FALSE:
2091 main.log.info( "No Loss in the pings" )
2092 main.log.info( "No loss of dataplane connectivity" )
2093 # utilities.assert_equals(
2094 # expect=main.FALSE,
2095 # actual=LossInPings,
2096 # onpass="No Loss of connectivity",
2097 # onfail="Loss of dataplane connectivity detected" )
2098
2099 # NOTE: Since intents are not persisted with IntnentStore,
2100 # we expect loss in dataplane connectivity
2101 LossInPings = main.FALSE
2102 '''
2103
2104 main.step( "Leadership Election is still functional" )
2105 # Test of LeadershipElection
2106 leaderList = []
2107 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002108 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002109 leaderN = cli.electionTestLeader()
2110 leaderList.append( leaderN )
2111 if leaderN == main.FALSE:
2112 # error in response
2113 main.log.error( "Something is wrong with " +
2114 "electionTestLeader function, check the" +
2115 " error logs" )
2116 leaderResult = main.FALSE
2117 elif leaderN is None:
2118 main.log.error( cli.name +
2119 " shows no leader for the election-app." )
2120 leaderResult = main.FALSE
2121 if len( set( leaderList ) ) != 1:
2122 leaderResult = main.FALSE
2123 main.log.error(
2124 "Inconsistent view of leader for the election test app" )
2125 # TODO: print the list
2126 utilities.assert_equals(
2127 expect=main.TRUE,
2128 actual=leaderResult,
2129 onpass="Leadership election passed",
2130 onfail="Something went wrong with Leadership election" )
2131
2132 def CASE8( self, main ):
2133 """
2134 Compare topo
2135 """
2136 import json
2137 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002138 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002139 assert main, "main not defined"
2140 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002141 assert main.CLIs, "main.CLIs not defined"
2142 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002143
2144 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002145 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002146 " and ONOS"
2147
2148 main.step( "Comparing ONOS topology to MN" )
2149 devicesResults = main.TRUE
2150 linksResults = main.TRUE
2151 hostsResults = main.TRUE
2152 hostAttachmentResults = True
2153 topoResult = main.FALSE
2154 elapsed = 0
2155 count = 0
2156 main.step( "Collecting topology information from ONOS" )
2157 startTime = time.time()
2158 # Give time for Gossip to work
2159 while topoResult == main.FALSE and elapsed < 60:
2160 count += 1
2161 cliStart = time.time()
2162 devices = []
2163 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002164 for i in range( main.numCtrls ):
2165 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002166 name="devices-" + str( i ),
2167 args=[ ] )
2168 threads.append( t )
2169 t.start()
2170
2171 for t in threads:
2172 t.join()
2173 devices.append( t.result )
2174 hosts = []
2175 ipResult = main.TRUE
2176 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002177 for i in range( main.numCtrls ):
2178 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07002179 name="hosts-" + str( i ),
2180 args=[ ] )
2181 threads.append( t )
2182 t.start()
2183
2184 for t in threads:
2185 t.join()
2186 try:
2187 hosts.append( json.loads( t.result ) )
2188 except ( ValueError, TypeError ):
2189 main.log.exception( "Error parsing hosts results" )
2190 main.log.error( repr( t.result ) )
2191 for controller in range( 0, len( hosts ) ):
2192 controllerStr = str( controller + 1 )
2193 for host in hosts[ controller ]:
2194 if host is None or host.get( 'ipAddresses', [] ) == []:
2195 main.log.error(
2196 "DEBUG:Error with host ipAddresses on controller" +
2197 controllerStr + ": " + str( host ) )
2198 ipResult = main.FALSE
2199 ports = []
2200 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002201 for i in range( main.numCtrls ):
2202 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002203 name="ports-" + str( i ),
2204 args=[ ] )
2205 threads.append( t )
2206 t.start()
2207
2208 for t in threads:
2209 t.join()
2210 ports.append( t.result )
2211 links = []
2212 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002213 for i in range( main.numCtrls ):
2214 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002215 name="links-" + str( i ),
2216 args=[ ] )
2217 threads.append( t )
2218 t.start()
2219
2220 for t in threads:
2221 t.join()
2222 links.append( t.result )
2223 clusters = []
2224 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002225 for i in range( main.numCtrls ):
2226 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002227 name="clusters-" + str( i ),
2228 args=[ ] )
2229 threads.append( t )
2230 t.start()
2231
2232 for t in threads:
2233 t.join()
2234 clusters.append( t.result )
2235
2236 elapsed = time.time() - startTime
2237 cliTime = time.time() - cliStart
2238 print "Elapsed time: " + str( elapsed )
2239 print "CLI time: " + str( cliTime )
2240
2241 mnSwitches = main.Mininet1.getSwitches()
2242 mnLinks = main.Mininet1.getLinks()
2243 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002244 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002245 controllerStr = str( controller + 1 )
2246 if devices[ controller ] and ports[ controller ] and\
2247 "Error" not in devices[ controller ] and\
2248 "Error" not in ports[ controller ]:
2249
2250 currentDevicesResult = main.Mininet1.compareSwitches(
2251 mnSwitches,
2252 json.loads( devices[ controller ] ),
2253 json.loads( ports[ controller ] ) )
2254 else:
2255 currentDevicesResult = main.FALSE
2256 utilities.assert_equals( expect=main.TRUE,
2257 actual=currentDevicesResult,
2258 onpass="ONOS" + controllerStr +
2259 " Switches view is correct",
2260 onfail="ONOS" + controllerStr +
2261 " Switches view is incorrect" )
2262
2263 if links[ controller ] and "Error" not in links[ controller ]:
2264 currentLinksResult = main.Mininet1.compareLinks(
2265 mnSwitches, mnLinks,
2266 json.loads( links[ controller ] ) )
2267 else:
2268 currentLinksResult = main.FALSE
2269 utilities.assert_equals( expect=main.TRUE,
2270 actual=currentLinksResult,
2271 onpass="ONOS" + controllerStr +
2272 " links view is correct",
2273 onfail="ONOS" + controllerStr +
2274 " links view is incorrect" )
2275
2276 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2277 currentHostsResult = main.Mininet1.compareHosts(
2278 mnHosts,
2279 hosts[ controller ] )
2280 else:
2281 currentHostsResult = main.FALSE
2282 utilities.assert_equals( expect=main.TRUE,
2283 actual=currentHostsResult,
2284 onpass="ONOS" + controllerStr +
2285 " hosts exist in Mininet",
2286 onfail="ONOS" + controllerStr +
2287 " hosts don't match Mininet" )
2288 # CHECKING HOST ATTACHMENT POINTS
2289 hostAttachment = True
2290 noHosts = False
2291 # FIXME: topo-HA/obelisk specific mappings:
2292 # key is mac and value is dpid
2293 mappings = {}
2294 for i in range( 1, 29 ): # hosts 1 through 28
2295 # set up correct variables:
2296 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2297 if i == 1:
2298 deviceId = "1000".zfill(16)
2299 elif i == 2:
2300 deviceId = "2000".zfill(16)
2301 elif i == 3:
2302 deviceId = "3000".zfill(16)
2303 elif i == 4:
2304 deviceId = "3004".zfill(16)
2305 elif i == 5:
2306 deviceId = "5000".zfill(16)
2307 elif i == 6:
2308 deviceId = "6000".zfill(16)
2309 elif i == 7:
2310 deviceId = "6007".zfill(16)
2311 elif i >= 8 and i <= 17:
2312 dpid = '3' + str( i ).zfill( 3 )
2313 deviceId = dpid.zfill(16)
2314 elif i >= 18 and i <= 27:
2315 dpid = '6' + str( i ).zfill( 3 )
2316 deviceId = dpid.zfill(16)
2317 elif i == 28:
2318 deviceId = "2800".zfill(16)
2319 mappings[ macId ] = deviceId
2320 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2321 if hosts[ controller ] == []:
2322 main.log.warn( "There are no hosts discovered" )
2323 noHosts = True
2324 else:
2325 for host in hosts[ controller ]:
2326 mac = None
2327 location = None
2328 device = None
2329 port = None
2330 try:
2331 mac = host.get( 'mac' )
2332 assert mac, "mac field could not be found for this host object"
2333
2334 location = host.get( 'location' )
2335 assert location, "location field could not be found for this host object"
2336
2337 # Trim the protocol identifier off deviceId
2338 device = str( location.get( 'elementId' ) ).split(':')[1]
2339 assert device, "elementId field could not be found for this host location object"
2340
2341 port = location.get( 'port' )
2342 assert port, "port field could not be found for this host location object"
2343
2344 # Now check if this matches where they should be
2345 if mac and device and port:
2346 if str( port ) != "1":
2347 main.log.error( "The attachment port is incorrect for " +
2348 "host " + str( mac ) +
2349 ". Expected: 1 Actual: " + str( port) )
2350 hostAttachment = False
2351 if device != mappings[ str( mac ) ]:
2352 main.log.error( "The attachment device is incorrect for " +
2353 "host " + str( mac ) +
2354 ". Expected: " + mappings[ str( mac ) ] +
2355 " Actual: " + device )
2356 hostAttachment = False
2357 else:
2358 hostAttachment = False
2359 except AssertionError:
2360 main.log.exception( "Json object not as expected" )
2361 main.log.error( repr( host ) )
2362 hostAttachment = False
2363 else:
2364 main.log.error( "No hosts json output or \"Error\"" +
2365 " in output. hosts = " +
2366 repr( hosts[ controller ] ) )
2367 if noHosts is False:
2368 # TODO: Find a way to know if there should be hosts in a
2369 # given point of the test
2370 hostAttachment = True
2371
2372 # END CHECKING HOST ATTACHMENT POINTS
2373 devicesResults = devicesResults and currentDevicesResult
2374 linksResults = linksResults and currentLinksResult
2375 hostsResults = hostsResults and currentHostsResult
2376 hostAttachmentResults = hostAttachmentResults and\
2377 hostAttachment
2378 topoResult = ( devicesResults and linksResults
2379 and hostsResults and ipResult and
2380 hostAttachmentResults )
2381
2382 # Compare json objects for hosts and dataplane clusters
2383
2384 # hosts
2385 main.step( "Hosts view is consistent across all ONOS nodes" )
2386 consistentHostsResult = main.TRUE
2387 for controller in range( len( hosts ) ):
2388 controllerStr = str( controller + 1 )
2389 if "Error" not in hosts[ controller ]:
2390 if hosts[ controller ] == hosts[ 0 ]:
2391 continue
2392 else: # hosts not consistent
2393 main.log.error( "hosts from ONOS" + controllerStr +
2394 " is inconsistent with ONOS1" )
2395 main.log.warn( repr( hosts[ controller ] ) )
2396 consistentHostsResult = main.FALSE
2397
2398 else:
2399 main.log.error( "Error in getting ONOS hosts from ONOS" +
2400 controllerStr )
2401 consistentHostsResult = main.FALSE
2402 main.log.warn( "ONOS" + controllerStr +
2403 " hosts response: " +
2404 repr( hosts[ controller ] ) )
2405 utilities.assert_equals(
2406 expect=main.TRUE,
2407 actual=consistentHostsResult,
2408 onpass="Hosts view is consistent across all ONOS nodes",
2409 onfail="ONOS nodes have different views of hosts" )
2410
2411 main.step( "Hosts information is correct" )
2412 hostsResults = hostsResults and ipResult
2413 utilities.assert_equals(
2414 expect=main.TRUE,
2415 actual=hostsResults,
2416 onpass="Host information is correct",
2417 onfail="Host information is incorrect" )
2418
2419 main.step( "Host attachment points to the network" )
2420 utilities.assert_equals(
2421 expect=True,
2422 actual=hostAttachmentResults,
2423 onpass="Hosts are correctly attached to the network",
2424 onfail="ONOS did not correctly attach hosts to the network" )
2425
2426 # Strongly connected clusters of devices
2427 main.step( "Clusters view is consistent across all ONOS nodes" )
2428 consistentClustersResult = main.TRUE
2429 for controller in range( len( clusters ) ):
2430 controllerStr = str( controller + 1 )
2431 if "Error" not in clusters[ controller ]:
2432 if clusters[ controller ] == clusters[ 0 ]:
2433 continue
2434 else: # clusters not consistent
2435 main.log.error( "clusters from ONOS" +
2436 controllerStr +
2437 " is inconsistent with ONOS1" )
2438 consistentClustersResult = main.FALSE
2439
2440 else:
2441 main.log.error( "Error in getting dataplane clusters " +
2442 "from ONOS" + controllerStr )
2443 consistentClustersResult = main.FALSE
2444 main.log.warn( "ONOS" + controllerStr +
2445 " clusters response: " +
2446 repr( clusters[ controller ] ) )
2447 utilities.assert_equals(
2448 expect=main.TRUE,
2449 actual=consistentClustersResult,
2450 onpass="Clusters view is consistent across all ONOS nodes",
2451 onfail="ONOS nodes have different views of clusters" )
2452
2453 main.step( "There is only one SCC" )
2454 # there should always only be one cluster
2455 try:
2456 numClusters = len( json.loads( clusters[ 0 ] ) )
2457 except ( ValueError, TypeError ):
2458 main.log.exception( "Error parsing clusters[0]: " +
2459 repr( clusters[0] ) )
2460 clusterResults = main.FALSE
2461 if numClusters == 1:
2462 clusterResults = main.TRUE
2463 utilities.assert_equals(
2464 expect=1,
2465 actual=numClusters,
2466 onpass="ONOS shows 1 SCC",
2467 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2468
2469 topoResult = ( devicesResults and linksResults
2470 and hostsResults and consistentHostsResult
2471 and consistentClustersResult and clusterResults
2472 and ipResult and hostAttachmentResults )
2473
2474 topoResult = topoResult and int( count <= 2 )
2475 note = "note it takes about " + str( int( cliTime ) ) + \
2476 " seconds for the test to make all the cli calls to fetch " +\
2477 "the topology from each ONOS instance"
2478 main.log.info(
2479 "Very crass estimate for topology discovery/convergence( " +
2480 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2481 str( count ) + " tries" )
2482
2483 main.step( "Device information is correct" )
2484 utilities.assert_equals(
2485 expect=main.TRUE,
2486 actual=devicesResults,
2487 onpass="Device information is correct",
2488 onfail="Device information is incorrect" )
2489
2490 main.step( "Links are correct" )
2491 utilities.assert_equals(
2492 expect=main.TRUE,
2493 actual=linksResults,
2494 onpass="Link are correct",
2495 onfail="Links are incorrect" )
2496
2497 # FIXME: move this to an ONOS state case
2498 main.step( "Checking ONOS nodes" )
2499 nodesOutput = []
2500 nodeResults = main.TRUE
2501 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002502 for i in range( main.numCtrls ):
2503 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002504 name="nodes-" + str( i ),
2505 args=[ ] )
2506 threads.append( t )
2507 t.start()
2508
2509 for t in threads:
2510 t.join()
2511 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002512 ips = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002513 for i in nodesOutput:
2514 try:
2515 current = json.loads( i )
2516 for node in current:
2517 currentResult = main.FALSE
2518 if node['ip'] in ips: # node in nodes() output is in cell
2519 if node['state'] == 'ACTIVE':
2520 currentResult = main.TRUE
2521 else:
2522 main.log.error( "Error in ONOS node availability" )
2523 main.log.error(
2524 json.dumps( current,
2525 sort_keys=True,
2526 indent=4,
2527 separators=( ',', ': ' ) ) )
2528 break
2529 nodeResults = nodeResults and currentResult
2530 except ( ValueError, TypeError ):
2531 main.log.error( "Error parsing nodes output" )
2532 main.log.warn( repr( i ) )
2533 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2534 onpass="Nodes check successful",
2535 onfail="Nodes check NOT successful" )
2536
2537 def CASE9( self, main ):
2538 """
2539 Link s3-s28 down
2540 """
2541 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002542 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002543 assert main, "main not defined"
2544 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002545 assert main.CLIs, "main.CLIs not defined"
2546 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002547 # NOTE: You should probably run a topology check after this
2548
2549 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2550
2551 description = "Turn off a link to ensure that Link Discovery " +\
2552 "is working properly"
2553 main.case( description )
2554
2555 main.step( "Kill Link between s3 and s28" )
2556 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2557 main.log.info( "Waiting " + str( linkSleep ) +
2558 " seconds for link down to be discovered" )
2559 time.sleep( linkSleep )
2560 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2561 onpass="Link down successful",
2562 onfail="Failed to bring link down" )
2563 # TODO do some sort of check here
2564
2565 def CASE10( self, main ):
2566 """
2567 Link s3-s28 up
2568 """
2569 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002570 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002571 assert main, "main not defined"
2572 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002573 assert main.CLIs, "main.CLIs not defined"
2574 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002575 # NOTE: You should probably run a topology check after this
2576
2577 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2578
2579 description = "Restore a link to ensure that Link Discovery is " + \
2580 "working properly"
2581 main.case( description )
2582
2583 main.step( "Bring link between s3 and s28 back up" )
2584 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2585 main.log.info( "Waiting " + str( linkSleep ) +
2586 " seconds for link up to be discovered" )
2587 time.sleep( linkSleep )
2588 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2589 onpass="Link up successful",
2590 onfail="Failed to bring link up" )
2591 # TODO do some sort of check here
2592
2593 def CASE11( self, main ):
2594 """
2595 Switch Down
2596 """
2597 # NOTE: You should probably run a topology check after this
2598 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002599 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002600 assert main, "main not defined"
2601 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002602 assert main.CLIs, "main.CLIs not defined"
2603 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002604
2605 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2606
2607 description = "Killing a switch to ensure it is discovered correctly"
2608 main.case( description )
2609 switch = main.params[ 'kill' ][ 'switch' ]
2610 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2611
2612 # TODO: Make this switch parameterizable
2613 main.step( "Kill " + switch )
2614 main.log.info( "Deleting " + switch )
2615 main.Mininet1.delSwitch( switch )
2616 main.log.info( "Waiting " + str( switchSleep ) +
2617 " seconds for switch down to be discovered" )
2618 time.sleep( switchSleep )
2619 device = main.ONOScli1.getDevice( dpid=switchDPID )
2620 # Peek at the deleted switch
2621 main.log.warn( str( device ) )
2622 result = main.FALSE
2623 if device and device[ 'available' ] is False:
2624 result = main.TRUE
2625 utilities.assert_equals( expect=main.TRUE, actual=result,
2626 onpass="Kill switch successful",
2627 onfail="Failed to kill switch?" )
2628
2629 def CASE12( self, main ):
2630 """
2631 Switch Up
2632 """
2633 # NOTE: You should probably run a topology check after this
2634 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002635 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002636 assert main, "main not defined"
2637 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002638 assert main.CLIs, "main.CLIs not defined"
2639 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002640 assert ONOS1Port, "ONOS1Port not defined"
2641 assert ONOS2Port, "ONOS2Port not defined"
2642 assert ONOS3Port, "ONOS3Port not defined"
2643 assert ONOS4Port, "ONOS4Port not defined"
2644 assert ONOS5Port, "ONOS5Port not defined"
2645 assert ONOS6Port, "ONOS6Port not defined"
2646 assert ONOS7Port, "ONOS7Port not defined"
2647
2648 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2649 switch = main.params[ 'kill' ][ 'switch' ]
2650 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2651 links = main.params[ 'kill' ][ 'links' ].split()
2652 description = "Adding a switch to ensure it is discovered correctly"
2653 main.case( description )
2654
2655 main.step( "Add back " + switch )
2656 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2657 for peer in links:
2658 main.Mininet1.addLink( switch, peer )
2659 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002660 for i in range( main.numCtrls ):
2661 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002662 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2663 main.log.info( "Waiting " + str( switchSleep ) +
2664 " seconds for switch up to be discovered" )
2665 time.sleep( switchSleep )
2666 device = main.ONOScli1.getDevice( dpid=switchDPID )
2667 # Peek at the deleted switch
2668 main.log.warn( str( device ) )
2669 result = main.FALSE
2670 if device and device[ 'available' ]:
2671 result = main.TRUE
2672 utilities.assert_equals( expect=main.TRUE, actual=result,
2673 onpass="add switch successful",
2674 onfail="Failed to add switch?" )
2675
2676 def CASE13( self, main ):
2677 """
2678 Clean up
2679 """
2680 import os
2681 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002682 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002683 assert main, "main not defined"
2684 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002685 assert main.CLIs, "main.CLIs not defined"
2686 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002687
2688 # printing colors to terminal
2689 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2690 'blue': '\033[94m', 'green': '\033[92m',
2691 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2692 main.case( "Test Cleanup" )
2693 main.step( "Killing tcpdumps" )
2694 main.Mininet2.stopTcpdump()
2695
2696 testname = main.TEST
2697 if main.params[ 'BACKUP' ] == "True":
2698 main.step( "Copying MN pcap and ONOS log files to test station" )
2699 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2700 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
2701 # NOTE: MN Pcap file is being saved to ~/packet_captures
2702 # scp this file as MN and TestON aren't necessarily the same vm
2703 # FIXME: scp
2704 # mn files
2705 # TODO: Load these from params
2706 # NOTE: must end in /
2707 logFolder = "/opt/onos/log/"
2708 logFiles = [ "karaf.log", "karaf.log.1" ]
2709 # NOTE: must end in /
2710 dstDir = "~/packet_captures/"
2711 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002712 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002713 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2714 ":" + logFolder + f + " " +
2715 teststationUser + "@" +
2716 teststationIP + ":" +
2717 dstDir + str( testname ) +
2718 "-" + node.name + "-" + f )
2719 main.ONOSbench.handle.expect( "\$" )
2720
2721 # std*.log's
2722 # NOTE: must end in /
2723 logFolder = "/opt/onos/var/"
2724 logFiles = [ "stderr.log", "stdout.log" ]
2725 # NOTE: must end in /
2726 dstDir = "~/packet_captures/"
2727 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002728 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002729 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2730 ":" + logFolder + f + " " +
2731 teststationUser + "@" +
2732 teststationIP + ":" +
2733 dstDir + str( testname ) +
2734 "-" + node.name + "-" + f )
2735 main.ONOSbench.handle.expect( "\$" )
2736 # sleep so scp can finish
2737 time.sleep( 10 )
2738 main.step( "Packing and rotating pcap archives" )
2739 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
2740
2741 main.step( "Stopping Mininet" )
2742 mnResult = main.Mininet1.stopNet()
2743 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2744 onpass="Mininet stopped",
2745 onfail="MN cleanup NOT successful" )
2746
2747 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002748 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002749 print colors[ 'purple' ] + "Checking logs for errors on " + \
2750 node.name + ":" + colors[ 'end' ]
2751 print main.ONOSbench.checkLogs( node.ip_address, restart=True )
2752
2753 try:
2754 timerLog = open( main.logdir + "/Timers.csv", 'w')
2755 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2756 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2757 timerLog.close()
2758 except NameError, e:
2759 main.log.exception(e)
2760
2761 def CASE14( self, main ):
2762 """
2763 start election app on all onos nodes
2764 """
Jon Halle1a3b752015-07-22 13:02:46 -07002765 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002766 assert main, "main not defined"
2767 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002768 assert main.CLIs, "main.CLIs not defined"
2769 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002770
2771 main.case("Start Leadership Election app")
2772 main.step( "Install leadership election app" )
2773 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2774 utilities.assert_equals(
2775 expect=main.TRUE,
2776 actual=appResult,
2777 onpass="Election app installed",
2778 onfail="Something went wrong with installing Leadership election" )
2779
2780 main.step( "Run for election on each node" )
2781 leaderResult = main.TRUE
2782 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002783 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002784 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002785 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002786 leader = cli.electionTestLeader()
2787 if leader is None or leader == main.FALSE:
2788 main.log.error( cli.name + ": Leader for the election app " +
2789 "should be an ONOS node, instead got '" +
2790 str( leader ) + "'" )
2791 leaderResult = main.FALSE
2792 leaders.append( leader )
2793 utilities.assert_equals(
2794 expect=main.TRUE,
2795 actual=leaderResult,
2796 onpass="Successfully ran for leadership",
2797 onfail="Failed to run for leadership" )
2798
2799 main.step( "Check that each node shows the same leader" )
2800 sameLeader = main.TRUE
2801 if len( set( leaders ) ) != 1:
2802 sameLeader = main.FALSE
2803 main.log.error( "Results of electionTestLeader is order of CLIs:" +
2804 str( leaders ) )
2805 utilities.assert_equals(
2806 expect=main.TRUE,
2807 actual=sameLeader,
2808 onpass="Leadership is consistent for the election topic",
2809 onfail="Nodes have different leaders" )
2810
2811 def CASE15( self, main ):
2812 """
2813 Check that Leadership Election is still functional
2814 """
2815 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002816 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002817 assert main, "main not defined"
2818 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002819 assert main.CLIs, "main.CLIs not defined"
2820 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002821
2822 leaderResult = main.TRUE
2823 description = "Check that Leadership Election is still functional"
2824 main.case( description )
2825 # NOTE: Need to re-run since being a canidate is not persistant
2826 main.step( "Run for election on each node" )
2827 leaderResult = main.TRUE
2828 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002829 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002830 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002831 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002832 leader = cli.electionTestLeader()
2833 if leader is None or leader == main.FALSE:
2834 main.log.error( cli.name + ": Leader for the election app " +
2835 "should be an ONOS node, instead got '" +
2836 str( leader ) + "'" )
2837 leaderResult = main.FALSE
2838 leaders.append( leader )
2839 utilities.assert_equals(
2840 expect=main.TRUE,
2841 actual=leaderResult,
2842 onpass="Successfully ran for leadership",
2843 onfail="Failed to run for leadership" )
2844
2845 main.step( "Check that each node shows the same leader" )
2846 sameLeader = main.TRUE
2847 if len( set( leaders ) ) != 1:
2848 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002849 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002850 str( leaders ) )
2851 utilities.assert_equals(
2852 expect=main.TRUE,
2853 actual=sameLeader,
2854 onpass="Leadership is consistent for the election topic",
2855 onfail="Nodes have different leaders" )
2856
2857 main.step( "Find current leader and withdraw" )
2858 leader = main.ONOScli1.electionTestLeader()
2859 # do some sanity checking on leader before using it
2860 withdrawResult = main.FALSE
2861 if leader is None or leader == main.FALSE:
2862 main.log.error(
2863 "Leader for the election app should be an ONOS node," +
2864 "instead got '" + str( leader ) + "'" )
2865 leaderResult = main.FALSE
2866 oldLeader = None
Jon Halle1a3b752015-07-22 13:02:46 -07002867 for i in range( len( main.CLIs ) ):
2868 if leader == main.nodes[ i ].ip_address:
2869 oldLeader = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002870 break
2871 else: # FOR/ELSE statement
2872 main.log.error( "Leader election, could not find current leader" )
2873 if oldLeader:
2874 withdrawResult = oldLeader.electionTestWithdraw()
2875 utilities.assert_equals(
2876 expect=main.TRUE,
2877 actual=withdrawResult,
2878 onpass="Node was withdrawn from election",
2879 onfail="Node was not withdrawn from election" )
2880
2881 main.step( "Make sure new leader is elected" )
2882 # FIXME: use threads
2883 leaderList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002884 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002885 leaderN = cli.electionTestLeader()
2886 leaderList.append( leaderN )
2887 if leaderN == leader:
2888 main.log.error( cli.name + " still sees " + str( leader ) +
2889 " as leader after they withdrew" )
2890 leaderResult = main.FALSE
2891 elif leaderN == main.FALSE:
2892 # error in response
2893 # TODO: add check for "Command not found:" in the driver, this
2894 # means the app isn't loaded
2895 main.log.error( "Something is wrong with " +
2896 "electionTestLeader function, " +
2897 "check the error logs" )
2898 leaderResult = main.FALSE
2899 elif leaderN is None:
2900 # node may not have recieved the event yet
2901 time.sleep(7)
2902 leaderN = cli.electionTestLeader()
2903 leaderList.pop()
2904 leaderList.append( leaderN )
2905 consistentLeader = main.FALSE
2906 if len( set( leaderList ) ) == 1:
2907 main.log.info( "Each Election-app sees '" +
2908 str( leaderList[ 0 ] ) +
2909 "' as the leader" )
2910 consistentLeader = main.TRUE
2911 else:
2912 main.log.error(
2913 "Inconsistent responses for leader of Election-app:" )
2914 for n in range( len( leaderList ) ):
2915 main.log.error( "ONOS" + str( n + 1 ) + " response: " +
2916 str( leaderList[ n ] ) )
2917 leaderResult = leaderResult and consistentLeader
2918 utilities.assert_equals(
2919 expect=main.TRUE,
2920 actual=leaderResult,
2921 onpass="Leadership election passed",
2922 onfail="Something went wrong with Leadership election" )
2923
2924 main.step( "Run for election on old leader( just so everyone " +
2925 "is in the hat )" )
2926 if oldLeader:
2927 runResult = oldLeader.electionTestRun()
2928 else:
2929 runResult = main.FALSE
2930 utilities.assert_equals(
2931 expect=main.TRUE,
2932 actual=runResult,
2933 onpass="App re-ran for election",
2934 onfail="App failed to run for election" )
2935
2936 main.step( "Leader did not change when old leader re-ran" )
2937 afterRun = main.ONOScli1.electionTestLeader()
2938 # verify leader didn't just change
2939 if afterRun == leaderList[ 0 ]:
2940 afterResult = main.TRUE
2941 else:
2942 afterResult = main.FALSE
2943
2944 utilities.assert_equals(
2945 expect=main.TRUE,
2946 actual=afterResult,
2947 onpass="Old leader successfully re-ran for election",
2948 onfail="Something went wrong with Leadership election after " +
2949 "the old leader re-ran for election" )
2950
2951 def CASE16( self, main ):
2952 """
2953 Install Distributed Primitives app
2954 """
2955 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002956 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002957 assert main, "main not defined"
2958 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002959 assert main.CLIs, "main.CLIs not defined"
2960 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002961
2962 # Variables for the distributed primitives tests
2963 global pCounterName
2964 global iCounterName
2965 global pCounterValue
2966 global iCounterValue
2967 global onosSet
2968 global onosSetName
2969 pCounterName = "TestON-Partitions"
2970 iCounterName = "TestON-inMemory"
2971 pCounterValue = 0
2972 iCounterValue = 0
2973 onosSet = set([])
2974 onosSetName = "TestON-set"
2975
2976 description = "Install Primitives app"
2977 main.case( description )
2978 main.step( "Install Primitives app" )
2979 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002980 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002981 utilities.assert_equals( expect=main.TRUE,
2982 actual=appResults,
2983 onpass="Primitives app activated",
2984 onfail="Primitives app not activated" )
2985 time.sleep( 5 ) # To allow all nodes to activate
2986
2987 def CASE17( self, main ):
2988 """
2989 Check for basic functionality with distributed primitives
2990 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002991 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002992 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002993 assert main, "main not defined"
2994 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002995 assert main.CLIs, "main.CLIs not defined"
2996 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002997 assert pCounterName, "pCounterName not defined"
2998 assert iCounterName, "iCounterName not defined"
2999 assert onosSetName, "onosSetName not defined"
3000 # NOTE: assert fails if value is 0/None/Empty/False
3001 try:
3002 pCounterValue
3003 except NameError:
3004 main.log.error( "pCounterValue not defined, setting to 0" )
3005 pCounterValue = 0
3006 try:
3007 iCounterValue
3008 except NameError:
3009 main.log.error( "iCounterValue not defined, setting to 0" )
3010 iCounterValue = 0
3011 try:
3012 onosSet
3013 except NameError:
3014 main.log.error( "onosSet not defined, setting to empty Set" )
3015 onosSet = set([])
3016 # Variables for the distributed primitives tests. These are local only
3017 addValue = "a"
3018 addAllValue = "a b c d e f"
3019 retainValue = "c d e f"
3020
3021 description = "Check for basic functionality with distributed " +\
3022 "primitives"
3023 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003024 main.caseExplanation = "Test the methods of the distributed " +\
3025 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003026 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003027 # Partitioned counters
3028 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003029 pCounters = []
3030 threads = []
3031 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003032 for i in range( main.numCtrls ):
3033 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3034 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003035 args=[ pCounterName ] )
3036 pCounterValue += 1
3037 addedPValues.append( pCounterValue )
3038 threads.append( t )
3039 t.start()
3040
3041 for t in threads:
3042 t.join()
3043 pCounters.append( t.result )
3044 # Check that counter incremented numController times
3045 pCounterResults = True
3046 for i in addedPValues:
3047 tmpResult = i in pCounters
3048 pCounterResults = pCounterResults and tmpResult
3049 if not tmpResult:
3050 main.log.error( str( i ) + " is not in partitioned "
3051 "counter incremented results" )
3052 utilities.assert_equals( expect=True,
3053 actual=pCounterResults,
3054 onpass="Default counter incremented",
3055 onfail="Error incrementing default" +
3056 " counter" )
3057
Jon Halle1a3b752015-07-22 13:02:46 -07003058 main.step( "Get then Increment a default counter on each node" )
3059 pCounters = []
3060 threads = []
3061 addedPValues = []
3062 for i in range( main.numCtrls ):
3063 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3064 name="counterGetAndAdd-" + str( i ),
3065 args=[ pCounterName ] )
3066 addedPValues.append( pCounterValue )
3067 pCounterValue += 1
3068 threads.append( t )
3069 t.start()
3070
3071 for t in threads:
3072 t.join()
3073 pCounters.append( t.result )
3074 # Check that counter incremented numController times
3075 pCounterResults = True
3076 for i in addedPValues:
3077 tmpResult = i in pCounters
3078 pCounterResults = pCounterResults and tmpResult
3079 if not tmpResult:
3080 main.log.error( str( i ) + " is not in partitioned "
3081 "counter incremented results" )
3082 utilities.assert_equals( expect=True,
3083 actual=pCounterResults,
3084 onpass="Default counter incremented",
3085 onfail="Error incrementing default" +
3086 " counter" )
3087
3088 main.step( "Counters we added have the correct values" )
3089 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3090 utilities.assert_equals( expect=main.TRUE,
3091 actual=incrementCheck,
3092 onpass="Added counters are correct",
3093 onfail="Added counters are incorrect" )
3094
3095 main.step( "Add -8 to then get a default counter on each node" )
3096 pCounters = []
3097 threads = []
3098 addedPValues = []
3099 for i in range( main.numCtrls ):
3100 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3101 name="counterIncrement-" + str( i ),
3102 args=[ pCounterName ],
3103 kwargs={ "delta": -8 } )
3104 pCounterValue += -8
3105 addedPValues.append( pCounterValue )
3106 threads.append( t )
3107 t.start()
3108
3109 for t in threads:
3110 t.join()
3111 pCounters.append( t.result )
3112 # Check that counter incremented numController times
3113 pCounterResults = True
3114 for i in addedPValues:
3115 tmpResult = i in pCounters
3116 pCounterResults = pCounterResults and tmpResult
3117 if not tmpResult:
3118 main.log.error( str( i ) + " is not in partitioned "
3119 "counter incremented results" )
3120 utilities.assert_equals( expect=True,
3121 actual=pCounterResults,
3122 onpass="Default counter incremented",
3123 onfail="Error incrementing default" +
3124 " counter" )
3125
3126 main.step( "Add 5 to then get a default counter on each node" )
3127 pCounters = []
3128 threads = []
3129 addedPValues = []
3130 for i in range( main.numCtrls ):
3131 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3132 name="counterIncrement-" + str( i ),
3133 args=[ pCounterName ],
3134 kwargs={ "delta": 5 } )
3135 pCounterValue += 5
3136 addedPValues.append( pCounterValue )
3137 threads.append( t )
3138 t.start()
3139
3140 for t in threads:
3141 t.join()
3142 pCounters.append( t.result )
3143 # Check that counter incremented numController times
3144 pCounterResults = True
3145 for i in addedPValues:
3146 tmpResult = i in pCounters
3147 pCounterResults = pCounterResults and tmpResult
3148 if not tmpResult:
3149 main.log.error( str( i ) + " is not in partitioned "
3150 "counter incremented results" )
3151 utilities.assert_equals( expect=True,
3152 actual=pCounterResults,
3153 onpass="Default counter incremented",
3154 onfail="Error incrementing default" +
3155 " counter" )
3156
3157 main.step( "Get then add 5 to a default counter on each node" )
3158 pCounters = []
3159 threads = []
3160 addedPValues = []
3161 for i in range( main.numCtrls ):
3162 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3163 name="counterIncrement-" + str( i ),
3164 args=[ pCounterName ],
3165 kwargs={ "delta": 5 } )
3166 addedPValues.append( pCounterValue )
3167 pCounterValue += 5
3168 threads.append( t )
3169 t.start()
3170
3171 for t in threads:
3172 t.join()
3173 pCounters.append( t.result )
3174 # Check that counter incremented numController times
3175 pCounterResults = True
3176 for i in addedPValues:
3177 tmpResult = i in pCounters
3178 pCounterResults = pCounterResults and tmpResult
3179 if not tmpResult:
3180 main.log.error( str( i ) + " is not in partitioned "
3181 "counter incremented results" )
3182 utilities.assert_equals( expect=True,
3183 actual=pCounterResults,
3184 onpass="Default counter incremented",
3185 onfail="Error incrementing default" +
3186 " counter" )
3187
3188 main.step( "Counters we added have the correct values" )
3189 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3190 utilities.assert_equals( expect=main.TRUE,
3191 actual=incrementCheck,
3192 onpass="Added counters are correct",
3193 onfail="Added counters are incorrect" )
3194
3195 # In-Memory counters
3196 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003197 iCounters = []
3198 addedIValues = []
3199 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003200 for i in range( main.numCtrls ):
3201 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003202 name="icounterIncrement-" + str( i ),
3203 args=[ iCounterName ],
3204 kwargs={ "inMemory": True } )
3205 iCounterValue += 1
3206 addedIValues.append( iCounterValue )
3207 threads.append( t )
3208 t.start()
3209
3210 for t in threads:
3211 t.join()
3212 iCounters.append( t.result )
3213 # Check that counter incremented numController times
3214 iCounterResults = True
3215 for i in addedIValues:
3216 tmpResult = i in iCounters
3217 iCounterResults = iCounterResults and tmpResult
3218 if not tmpResult:
3219 main.log.error( str( i ) + " is not in the in-memory "
3220 "counter incremented results" )
3221 utilities.assert_equals( expect=True,
3222 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003223 onpass="In-memory counter incremented",
3224 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003225 " counter" )
3226
Jon Halle1a3b752015-07-22 13:02:46 -07003227 main.step( "Get then Increment a in-memory counter on each node" )
3228 iCounters = []
3229 threads = []
3230 addedIValues = []
3231 for i in range( main.numCtrls ):
3232 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3233 name="counterGetAndAdd-" + str( i ),
3234 args=[ iCounterName ],
3235 kwargs={ "inMemory": True } )
3236 addedIValues.append( iCounterValue )
3237 iCounterValue += 1
3238 threads.append( t )
3239 t.start()
3240
3241 for t in threads:
3242 t.join()
3243 iCounters.append( t.result )
3244 # Check that counter incremented numController times
3245 iCounterResults = True
3246 for i in addedIValues:
3247 tmpResult = i in iCounters
3248 iCounterResults = iCounterResults and tmpResult
3249 if not tmpResult:
3250 main.log.error( str( i ) + " is not in in-memory "
3251 "counter incremented results" )
3252 utilities.assert_equals( expect=True,
3253 actual=iCounterResults,
3254 onpass="In-memory counter incremented",
3255 onfail="Error incrementing in-memory" +
3256 " counter" )
3257
3258 main.step( "Counters we added have the correct values" )
3259 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3260 utilities.assert_equals( expect=main.TRUE,
3261 actual=incrementCheck,
3262 onpass="Added counters are correct",
3263 onfail="Added counters are incorrect" )
3264
3265 main.step( "Add -8 to then get a in-memory counter on each node" )
3266 iCounters = []
3267 threads = []
3268 addedIValues = []
3269 for i in range( main.numCtrls ):
3270 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3271 name="counterIncrement-" + str( i ),
3272 args=[ iCounterName ],
3273 kwargs={ "delta": -8, "inMemory": True } )
3274 iCounterValue += -8
3275 addedIValues.append( iCounterValue )
3276 threads.append( t )
3277 t.start()
3278
3279 for t in threads:
3280 t.join()
3281 iCounters.append( t.result )
3282 # Check that counter incremented numController times
3283 iCounterResults = True
3284 for i in addedIValues:
3285 tmpResult = i in iCounters
3286 iCounterResults = iCounterResults and tmpResult
3287 if not tmpResult:
3288 main.log.error( str( i ) + " is not in in-memory "
3289 "counter incremented results" )
3290 utilities.assert_equals( expect=True,
3291 actual=pCounterResults,
3292 onpass="In-memory counter incremented",
3293 onfail="Error incrementing in-memory" +
3294 " counter" )
3295
3296 main.step( "Add 5 to then get a in-memory counter on each node" )
3297 iCounters = []
3298 threads = []
3299 addedIValues = []
3300 for i in range( main.numCtrls ):
3301 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3302 name="counterIncrement-" + str( i ),
3303 args=[ iCounterName ],
3304 kwargs={ "delta": 5, "inMemory": True } )
3305 iCounterValue += 5
3306 addedIValues.append( iCounterValue )
3307 threads.append( t )
3308 t.start()
3309
3310 for t in threads:
3311 t.join()
3312 iCounters.append( t.result )
3313 # Check that counter incremented numController times
3314 iCounterResults = True
3315 for i in addedIValues:
3316 tmpResult = i in iCounters
3317 iCounterResults = iCounterResults and tmpResult
3318 if not tmpResult:
3319 main.log.error( str( i ) + " is not in in-memory "
3320 "counter incremented results" )
3321 utilities.assert_equals( expect=True,
3322 actual=pCounterResults,
3323 onpass="In-memory counter incremented",
3324 onfail="Error incrementing in-memory" +
3325 " counter" )
3326
3327 main.step( "Get then add 5 to a in-memory counter on each node" )
3328 iCounters = []
3329 threads = []
3330 addedIValues = []
3331 for i in range( main.numCtrls ):
3332 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3333 name="counterIncrement-" + str( i ),
3334 args=[ iCounterName ],
3335 kwargs={ "delta": 5, "inMemory": True } )
3336 addedIValues.append( iCounterValue )
3337 iCounterValue += 5
3338 threads.append( t )
3339 t.start()
3340
3341 for t in threads:
3342 t.join()
3343 iCounters.append( t.result )
3344 # Check that counter incremented numController times
3345 iCounterResults = True
3346 for i in addedIValues:
3347 tmpResult = i in iCounters
3348 iCounterResults = iCounterResults and tmpResult
3349 if not tmpResult:
3350 main.log.error( str( i ) + " is not in in-memory "
3351 "counter incremented results" )
3352 utilities.assert_equals( expect=True,
3353 actual=iCounterResults,
3354 onpass="In-memory counter incremented",
3355 onfail="Error incrementing in-memory" +
3356 " counter" )
3357
3358 main.step( "Counters we added have the correct values" )
3359 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3360 utilities.assert_equals( expect=main.TRUE,
3361 actual=incrementCheck,
3362 onpass="Added counters are correct",
3363 onfail="Added counters are incorrect" )
3364
Jon Hall5cf14d52015-07-16 12:15:19 -07003365 main.step( "Check counters are consistant across nodes" )
3366 onosCounters = []
3367 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003368 for i in range( main.numCtrls ):
3369 t = main.Thread( target=main.CLIs[i].counters,
Jon Hall5cf14d52015-07-16 12:15:19 -07003370 name="counters-" + str( i ) )
3371 threads.append( t )
3372 t.start()
3373 for t in threads:
3374 t.join()
3375 onosCounters.append( t.result )
3376 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3377 if all( tmp ):
3378 main.log.info( "Counters are consistent across all nodes" )
3379 consistentCounterResults = main.TRUE
3380 else:
3381 main.log.error( "Counters are not consistent across all nodes" )
3382 consistentCounterResults = main.FALSE
3383 utilities.assert_equals( expect=main.TRUE,
3384 actual=consistentCounterResults,
3385 onpass="ONOS counters are consistent " +
3386 "across nodes",
3387 onfail="ONOS Counters are inconsistent " +
3388 "across nodes" )
3389
3390 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003391 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3392 incrementCheck = incrementCheck and \
3393 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003394 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003395 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003396 onpass="Added counters are correct",
3397 onfail="Added counters are incorrect" )
Jon Halle1a3b752015-07-22 13:02:46 -07003398
Jon Hall5cf14d52015-07-16 12:15:19 -07003399 # DISTRIBUTED SETS
3400 main.step( "Distributed Set get" )
3401 size = len( onosSet )
3402 getResponses = []
3403 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003404 for i in range( main.numCtrls ):
3405 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003406 name="setTestGet-" + str( i ),
3407 args=[ onosSetName ] )
3408 threads.append( t )
3409 t.start()
3410 for t in threads:
3411 t.join()
3412 getResponses.append( t.result )
3413
3414 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003415 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003416 if isinstance( getResponses[ i ], list):
3417 current = set( getResponses[ i ] )
3418 if len( current ) == len( getResponses[ i ] ):
3419 # no repeats
3420 if onosSet != current:
3421 main.log.error( "ONOS" + str( i + 1 ) +
3422 " has incorrect view" +
3423 " of set " + onosSetName + ":\n" +
3424 str( getResponses[ i ] ) )
3425 main.log.debug( "Expected: " + str( onosSet ) )
3426 main.log.debug( "Actual: " + str( current ) )
3427 getResults = main.FALSE
3428 else:
3429 # error, set is not a set
3430 main.log.error( "ONOS" + str( i + 1 ) +
3431 " has repeat elements in" +
3432 " set " + onosSetName + ":\n" +
3433 str( getResponses[ i ] ) )
3434 getResults = main.FALSE
3435 elif getResponses[ i ] == main.ERROR:
3436 getResults = main.FALSE
3437 utilities.assert_equals( expect=main.TRUE,
3438 actual=getResults,
3439 onpass="Set elements are correct",
3440 onfail="Set elements are incorrect" )
3441
3442 main.step( "Distributed Set size" )
3443 sizeResponses = []
3444 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003445 for i in range( main.numCtrls ):
3446 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003447 name="setTestSize-" + str( i ),
3448 args=[ onosSetName ] )
3449 threads.append( t )
3450 t.start()
3451 for t in threads:
3452 t.join()
3453 sizeResponses.append( t.result )
3454
3455 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003456 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003457 if size != sizeResponses[ i ]:
3458 sizeResults = main.FALSE
3459 main.log.error( "ONOS" + str( i + 1 ) +
3460 " expected a size of " + str( size ) +
3461 " for set " + onosSetName +
3462 " but got " + str( sizeResponses[ i ] ) )
3463 utilities.assert_equals( expect=main.TRUE,
3464 actual=sizeResults,
3465 onpass="Set sizes are correct",
3466 onfail="Set sizes are incorrect" )
3467
3468 main.step( "Distributed Set add()" )
3469 onosSet.add( addValue )
3470 addResponses = []
3471 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003472 for i in range( main.numCtrls ):
3473 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003474 name="setTestAdd-" + str( i ),
3475 args=[ onosSetName, addValue ] )
3476 threads.append( t )
3477 t.start()
3478 for t in threads:
3479 t.join()
3480 addResponses.append( t.result )
3481
3482 # main.TRUE = successfully changed the set
3483 # main.FALSE = action resulted in no change in set
3484 # main.ERROR - Some error in executing the function
3485 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003486 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003487 if addResponses[ i ] == main.TRUE:
3488 # All is well
3489 pass
3490 elif addResponses[ i ] == main.FALSE:
3491 # Already in set, probably fine
3492 pass
3493 elif addResponses[ i ] == main.ERROR:
3494 # Error in execution
3495 addResults = main.FALSE
3496 else:
3497 # unexpected result
3498 addResults = main.FALSE
3499 if addResults != main.TRUE:
3500 main.log.error( "Error executing set add" )
3501
3502 # Check if set is still correct
3503 size = len( onosSet )
3504 getResponses = []
3505 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003506 for i in range( main.numCtrls ):
3507 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003508 name="setTestGet-" + str( i ),
3509 args=[ onosSetName ] )
3510 threads.append( t )
3511 t.start()
3512 for t in threads:
3513 t.join()
3514 getResponses.append( t.result )
3515 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003516 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003517 if isinstance( getResponses[ i ], list):
3518 current = set( getResponses[ i ] )
3519 if len( current ) == len( getResponses[ i ] ):
3520 # no repeats
3521 if onosSet != current:
3522 main.log.error( "ONOS" + str( i + 1 ) +
3523 " has incorrect view" +
3524 " of set " + onosSetName + ":\n" +
3525 str( getResponses[ i ] ) )
3526 main.log.debug( "Expected: " + str( onosSet ) )
3527 main.log.debug( "Actual: " + str( current ) )
3528 getResults = main.FALSE
3529 else:
3530 # error, set is not a set
3531 main.log.error( "ONOS" + str( i + 1 ) +
3532 " has repeat elements in" +
3533 " set " + onosSetName + ":\n" +
3534 str( getResponses[ i ] ) )
3535 getResults = main.FALSE
3536 elif getResponses[ i ] == main.ERROR:
3537 getResults = main.FALSE
3538 sizeResponses = []
3539 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003540 for i in range( main.numCtrls ):
3541 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003542 name="setTestSize-" + str( i ),
3543 args=[ onosSetName ] )
3544 threads.append( t )
3545 t.start()
3546 for t in threads:
3547 t.join()
3548 sizeResponses.append( t.result )
3549 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003550 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003551 if size != sizeResponses[ i ]:
3552 sizeResults = main.FALSE
3553 main.log.error( "ONOS" + str( i + 1 ) +
3554 " expected a size of " + str( size ) +
3555 " for set " + onosSetName +
3556 " but got " + str( sizeResponses[ i ] ) )
3557 addResults = addResults and getResults and sizeResults
3558 utilities.assert_equals( expect=main.TRUE,
3559 actual=addResults,
3560 onpass="Set add correct",
3561 onfail="Set add was incorrect" )
3562
3563 main.step( "Distributed Set addAll()" )
3564 onosSet.update( addAllValue.split() )
3565 addResponses = []
3566 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003567 for i in range( main.numCtrls ):
3568 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003569 name="setTestAddAll-" + str( i ),
3570 args=[ onosSetName, addAllValue ] )
3571 threads.append( t )
3572 t.start()
3573 for t in threads:
3574 t.join()
3575 addResponses.append( t.result )
3576
3577 # main.TRUE = successfully changed the set
3578 # main.FALSE = action resulted in no change in set
3579 # main.ERROR - Some error in executing the function
3580 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003581 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003582 if addResponses[ i ] == main.TRUE:
3583 # All is well
3584 pass
3585 elif addResponses[ i ] == main.FALSE:
3586 # Already in set, probably fine
3587 pass
3588 elif addResponses[ i ] == main.ERROR:
3589 # Error in execution
3590 addAllResults = main.FALSE
3591 else:
3592 # unexpected result
3593 addAllResults = main.FALSE
3594 if addAllResults != main.TRUE:
3595 main.log.error( "Error executing set addAll" )
3596
3597 # Check if set is still correct
3598 size = len( onosSet )
3599 getResponses = []
3600 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003601 for i in range( main.numCtrls ):
3602 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003603 name="setTestGet-" + str( i ),
3604 args=[ onosSetName ] )
3605 threads.append( t )
3606 t.start()
3607 for t in threads:
3608 t.join()
3609 getResponses.append( t.result )
3610 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003611 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003612 if isinstance( getResponses[ i ], list):
3613 current = set( getResponses[ i ] )
3614 if len( current ) == len( getResponses[ i ] ):
3615 # no repeats
3616 if onosSet != current:
3617 main.log.error( "ONOS" + str( i + 1 ) +
3618 " has incorrect view" +
3619 " of set " + onosSetName + ":\n" +
3620 str( getResponses[ i ] ) )
3621 main.log.debug( "Expected: " + str( onosSet ) )
3622 main.log.debug( "Actual: " + str( current ) )
3623 getResults = main.FALSE
3624 else:
3625 # error, set is not a set
3626 main.log.error( "ONOS" + str( i + 1 ) +
3627 " has repeat elements in" +
3628 " set " + onosSetName + ":\n" +
3629 str( getResponses[ i ] ) )
3630 getResults = main.FALSE
3631 elif getResponses[ i ] == main.ERROR:
3632 getResults = main.FALSE
3633 sizeResponses = []
3634 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003635 for i in range( main.numCtrls ):
3636 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003637 name="setTestSize-" + str( i ),
3638 args=[ onosSetName ] )
3639 threads.append( t )
3640 t.start()
3641 for t in threads:
3642 t.join()
3643 sizeResponses.append( t.result )
3644 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003645 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003646 if size != sizeResponses[ i ]:
3647 sizeResults = main.FALSE
3648 main.log.error( "ONOS" + str( i + 1 ) +
3649 " expected a size of " + str( size ) +
3650 " for set " + onosSetName +
3651 " but got " + str( sizeResponses[ i ] ) )
3652 addAllResults = addAllResults and getResults and sizeResults
3653 utilities.assert_equals( expect=main.TRUE,
3654 actual=addAllResults,
3655 onpass="Set addAll correct",
3656 onfail="Set addAll was incorrect" )
3657
3658 main.step( "Distributed Set contains()" )
3659 containsResponses = []
3660 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003661 for i in range( main.numCtrls ):
3662 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003663 name="setContains-" + str( i ),
3664 args=[ onosSetName ],
3665 kwargs={ "values": addValue } )
3666 threads.append( t )
3667 t.start()
3668 for t in threads:
3669 t.join()
3670 # NOTE: This is the tuple
3671 containsResponses.append( t.result )
3672
3673 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003674 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003675 if containsResponses[ i ] == main.ERROR:
3676 containsResults = main.FALSE
3677 else:
3678 containsResults = containsResults and\
3679 containsResponses[ i ][ 1 ]
3680 utilities.assert_equals( expect=main.TRUE,
3681 actual=containsResults,
3682 onpass="Set contains is functional",
3683 onfail="Set contains failed" )
3684
3685 main.step( "Distributed Set containsAll()" )
3686 containsAllResponses = []
3687 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003688 for i in range( main.numCtrls ):
3689 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003690 name="setContainsAll-" + str( i ),
3691 args=[ onosSetName ],
3692 kwargs={ "values": addAllValue } )
3693 threads.append( t )
3694 t.start()
3695 for t in threads:
3696 t.join()
3697 # NOTE: This is the tuple
3698 containsAllResponses.append( t.result )
3699
3700 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003701 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003702 if containsResponses[ i ] == main.ERROR:
3703 containsResults = main.FALSE
3704 else:
3705 containsResults = containsResults and\
3706 containsResponses[ i ][ 1 ]
3707 utilities.assert_equals( expect=main.TRUE,
3708 actual=containsAllResults,
3709 onpass="Set containsAll is functional",
3710 onfail="Set containsAll failed" )
3711
3712 main.step( "Distributed Set remove()" )
3713 onosSet.remove( addValue )
3714 removeResponses = []
3715 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003716 for i in range( main.numCtrls ):
3717 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003718 name="setTestRemove-" + str( i ),
3719 args=[ onosSetName, addValue ] )
3720 threads.append( t )
3721 t.start()
3722 for t in threads:
3723 t.join()
3724 removeResponses.append( t.result )
3725
3726 # main.TRUE = successfully changed the set
3727 # main.FALSE = action resulted in no change in set
3728 # main.ERROR - Some error in executing the function
3729 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003730 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003731 if removeResponses[ i ] == main.TRUE:
3732 # All is well
3733 pass
3734 elif removeResponses[ i ] == main.FALSE:
3735 # not in set, probably fine
3736 pass
3737 elif removeResponses[ i ] == main.ERROR:
3738 # Error in execution
3739 removeResults = main.FALSE
3740 else:
3741 # unexpected result
3742 removeResults = main.FALSE
3743 if removeResults != main.TRUE:
3744 main.log.error( "Error executing set remove" )
3745
3746 # Check if set is still correct
3747 size = len( onosSet )
3748 getResponses = []
3749 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003750 for i in range( main.numCtrls ):
3751 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003752 name="setTestGet-" + str( i ),
3753 args=[ onosSetName ] )
3754 threads.append( t )
3755 t.start()
3756 for t in threads:
3757 t.join()
3758 getResponses.append( t.result )
3759 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003760 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003761 if isinstance( getResponses[ i ], list):
3762 current = set( getResponses[ i ] )
3763 if len( current ) == len( getResponses[ i ] ):
3764 # no repeats
3765 if onosSet != current:
3766 main.log.error( "ONOS" + str( i + 1 ) +
3767 " has incorrect view" +
3768 " of set " + onosSetName + ":\n" +
3769 str( getResponses[ i ] ) )
3770 main.log.debug( "Expected: " + str( onosSet ) )
3771 main.log.debug( "Actual: " + str( current ) )
3772 getResults = main.FALSE
3773 else:
3774 # error, set is not a set
3775 main.log.error( "ONOS" + str( i + 1 ) +
3776 " has repeat elements in" +
3777 " set " + onosSetName + ":\n" +
3778 str( getResponses[ i ] ) )
3779 getResults = main.FALSE
3780 elif getResponses[ i ] == main.ERROR:
3781 getResults = main.FALSE
3782 sizeResponses = []
3783 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003784 for i in range( main.numCtrls ):
3785 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003786 name="setTestSize-" + str( i ),
3787 args=[ onosSetName ] )
3788 threads.append( t )
3789 t.start()
3790 for t in threads:
3791 t.join()
3792 sizeResponses.append( t.result )
3793 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003794 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003795 if size != sizeResponses[ i ]:
3796 sizeResults = main.FALSE
3797 main.log.error( "ONOS" + str( i + 1 ) +
3798 " expected a size of " + str( size ) +
3799 " for set " + onosSetName +
3800 " but got " + str( sizeResponses[ i ] ) )
3801 removeResults = removeResults and getResults and sizeResults
3802 utilities.assert_equals( expect=main.TRUE,
3803 actual=removeResults,
3804 onpass="Set remove correct",
3805 onfail="Set remove was incorrect" )
3806
3807 main.step( "Distributed Set removeAll()" )
3808 onosSet.difference_update( addAllValue.split() )
3809 removeAllResponses = []
3810 threads = []
3811 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003812 for i in range( main.numCtrls ):
3813 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003814 name="setTestRemoveAll-" + str( i ),
3815 args=[ onosSetName, addAllValue ] )
3816 threads.append( t )
3817 t.start()
3818 for t in threads:
3819 t.join()
3820 removeAllResponses.append( t.result )
3821 except Exception, e:
3822 main.log.exception(e)
3823
3824 # main.TRUE = successfully changed the set
3825 # main.FALSE = action resulted in no change in set
3826 # main.ERROR - Some error in executing the function
3827 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003828 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003829 if removeAllResponses[ i ] == main.TRUE:
3830 # All is well
3831 pass
3832 elif removeAllResponses[ i ] == main.FALSE:
3833 # not in set, probably fine
3834 pass
3835 elif removeAllResponses[ i ] == main.ERROR:
3836 # Error in execution
3837 removeAllResults = main.FALSE
3838 else:
3839 # unexpected result
3840 removeAllResults = main.FALSE
3841 if removeAllResults != main.TRUE:
3842 main.log.error( "Error executing set removeAll" )
3843
3844 # Check if set is still correct
3845 size = len( onosSet )
3846 getResponses = []
3847 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003848 for i in range( main.numCtrls ):
3849 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003850 name="setTestGet-" + str( i ),
3851 args=[ onosSetName ] )
3852 threads.append( t )
3853 t.start()
3854 for t in threads:
3855 t.join()
3856 getResponses.append( t.result )
3857 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003858 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003859 if isinstance( getResponses[ i ], list):
3860 current = set( getResponses[ i ] )
3861 if len( current ) == len( getResponses[ i ] ):
3862 # no repeats
3863 if onosSet != current:
3864 main.log.error( "ONOS" + str( i + 1 ) +
3865 " has incorrect view" +
3866 " of set " + onosSetName + ":\n" +
3867 str( getResponses[ i ] ) )
3868 main.log.debug( "Expected: " + str( onosSet ) )
3869 main.log.debug( "Actual: " + str( current ) )
3870 getResults = main.FALSE
3871 else:
3872 # error, set is not a set
3873 main.log.error( "ONOS" + str( i + 1 ) +
3874 " has repeat elements in" +
3875 " set " + onosSetName + ":\n" +
3876 str( getResponses[ i ] ) )
3877 getResults = main.FALSE
3878 elif getResponses[ i ] == main.ERROR:
3879 getResults = main.FALSE
3880 sizeResponses = []
3881 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003882 for i in range( main.numCtrls ):
3883 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003884 name="setTestSize-" + str( i ),
3885 args=[ onosSetName ] )
3886 threads.append( t )
3887 t.start()
3888 for t in threads:
3889 t.join()
3890 sizeResponses.append( t.result )
3891 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003892 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003893 if size != sizeResponses[ i ]:
3894 sizeResults = main.FALSE
3895 main.log.error( "ONOS" + str( i + 1 ) +
3896 " expected a size of " + str( size ) +
3897 " for set " + onosSetName +
3898 " but got " + str( sizeResponses[ i ] ) )
3899 removeAllResults = removeAllResults and getResults and sizeResults
3900 utilities.assert_equals( expect=main.TRUE,
3901 actual=removeAllResults,
3902 onpass="Set removeAll correct",
3903 onfail="Set removeAll was incorrect" )
3904
3905 main.step( "Distributed Set addAll()" )
3906 onosSet.update( addAllValue.split() )
3907 addResponses = []
3908 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003909 for i in range( main.numCtrls ):
3910 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003911 name="setTestAddAll-" + str( i ),
3912 args=[ onosSetName, addAllValue ] )
3913 threads.append( t )
3914 t.start()
3915 for t in threads:
3916 t.join()
3917 addResponses.append( t.result )
3918
3919 # main.TRUE = successfully changed the set
3920 # main.FALSE = action resulted in no change in set
3921 # main.ERROR - Some error in executing the function
3922 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003923 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003924 if addResponses[ i ] == main.TRUE:
3925 # All is well
3926 pass
3927 elif addResponses[ i ] == main.FALSE:
3928 # Already in set, probably fine
3929 pass
3930 elif addResponses[ i ] == main.ERROR:
3931 # Error in execution
3932 addAllResults = main.FALSE
3933 else:
3934 # unexpected result
3935 addAllResults = main.FALSE
3936 if addAllResults != main.TRUE:
3937 main.log.error( "Error executing set addAll" )
3938
3939 # Check if set is still correct
3940 size = len( onosSet )
3941 getResponses = []
3942 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003943 for i in range( main.numCtrls ):
3944 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003945 name="setTestGet-" + str( i ),
3946 args=[ onosSetName ] )
3947 threads.append( t )
3948 t.start()
3949 for t in threads:
3950 t.join()
3951 getResponses.append( t.result )
3952 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003953 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003954 if isinstance( getResponses[ i ], list):
3955 current = set( getResponses[ i ] )
3956 if len( current ) == len( getResponses[ i ] ):
3957 # no repeats
3958 if onosSet != current:
3959 main.log.error( "ONOS" + str( i + 1 ) +
3960 " has incorrect view" +
3961 " of set " + onosSetName + ":\n" +
3962 str( getResponses[ i ] ) )
3963 main.log.debug( "Expected: " + str( onosSet ) )
3964 main.log.debug( "Actual: " + str( current ) )
3965 getResults = main.FALSE
3966 else:
3967 # error, set is not a set
3968 main.log.error( "ONOS" + str( i + 1 ) +
3969 " has repeat elements in" +
3970 " set " + onosSetName + ":\n" +
3971 str( getResponses[ i ] ) )
3972 getResults = main.FALSE
3973 elif getResponses[ i ] == main.ERROR:
3974 getResults = main.FALSE
3975 sizeResponses = []
3976 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003977 for i in range( main.numCtrls ):
3978 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003979 name="setTestSize-" + str( i ),
3980 args=[ onosSetName ] )
3981 threads.append( t )
3982 t.start()
3983 for t in threads:
3984 t.join()
3985 sizeResponses.append( t.result )
3986 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003987 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003988 if size != sizeResponses[ i ]:
3989 sizeResults = main.FALSE
3990 main.log.error( "ONOS" + str( i + 1 ) +
3991 " expected a size of " + str( size ) +
3992 " for set " + onosSetName +
3993 " but got " + str( sizeResponses[ i ] ) )
3994 addAllResults = addAllResults and getResults and sizeResults
3995 utilities.assert_equals( expect=main.TRUE,
3996 actual=addAllResults,
3997 onpass="Set addAll correct",
3998 onfail="Set addAll was incorrect" )
3999
4000 main.step( "Distributed Set clear()" )
4001 onosSet.clear()
4002 clearResponses = []
4003 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004004 for i in range( main.numCtrls ):
4005 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004006 name="setTestClear-" + str( i ),
4007 args=[ onosSetName, " "], # Values doesn't matter
4008 kwargs={ "clear": True } )
4009 threads.append( t )
4010 t.start()
4011 for t in threads:
4012 t.join()
4013 clearResponses.append( t.result )
4014
4015 # main.TRUE = successfully changed the set
4016 # main.FALSE = action resulted in no change in set
4017 # main.ERROR - Some error in executing the function
4018 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004019 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004020 if clearResponses[ i ] == main.TRUE:
4021 # All is well
4022 pass
4023 elif clearResponses[ i ] == main.FALSE:
4024 # Nothing set, probably fine
4025 pass
4026 elif clearResponses[ i ] == main.ERROR:
4027 # Error in execution
4028 clearResults = main.FALSE
4029 else:
4030 # unexpected result
4031 clearResults = main.FALSE
4032 if clearResults != main.TRUE:
4033 main.log.error( "Error executing set clear" )
4034
4035 # Check if set is still correct
4036 size = len( onosSet )
4037 getResponses = []
4038 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004039 for i in range( main.numCtrls ):
4040 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004041 name="setTestGet-" + str( i ),
4042 args=[ onosSetName ] )
4043 threads.append( t )
4044 t.start()
4045 for t in threads:
4046 t.join()
4047 getResponses.append( t.result )
4048 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004049 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004050 if isinstance( getResponses[ i ], list):
4051 current = set( getResponses[ i ] )
4052 if len( current ) == len( getResponses[ i ] ):
4053 # no repeats
4054 if onosSet != current:
4055 main.log.error( "ONOS" + str( i + 1 ) +
4056 " has incorrect view" +
4057 " of set " + onosSetName + ":\n" +
4058 str( getResponses[ i ] ) )
4059 main.log.debug( "Expected: " + str( onosSet ) )
4060 main.log.debug( "Actual: " + str( current ) )
4061 getResults = main.FALSE
4062 else:
4063 # error, set is not a set
4064 main.log.error( "ONOS" + str( i + 1 ) +
4065 " has repeat elements in" +
4066 " set " + onosSetName + ":\n" +
4067 str( getResponses[ i ] ) )
4068 getResults = main.FALSE
4069 elif getResponses[ i ] == main.ERROR:
4070 getResults = main.FALSE
4071 sizeResponses = []
4072 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004073 for i in range( main.numCtrls ):
4074 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004075 name="setTestSize-" + str( i ),
4076 args=[ onosSetName ] )
4077 threads.append( t )
4078 t.start()
4079 for t in threads:
4080 t.join()
4081 sizeResponses.append( t.result )
4082 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004083 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004084 if size != sizeResponses[ i ]:
4085 sizeResults = main.FALSE
4086 main.log.error( "ONOS" + str( i + 1 ) +
4087 " expected a size of " + str( size ) +
4088 " for set " + onosSetName +
4089 " but got " + str( sizeResponses[ i ] ) )
4090 clearResults = clearResults and getResults and sizeResults
4091 utilities.assert_equals( expect=main.TRUE,
4092 actual=clearResults,
4093 onpass="Set clear correct",
4094 onfail="Set clear was incorrect" )
4095
4096 main.step( "Distributed Set addAll()" )
4097 onosSet.update( addAllValue.split() )
4098 addResponses = []
4099 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004100 for i in range( main.numCtrls ):
4101 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004102 name="setTestAddAll-" + str( i ),
4103 args=[ onosSetName, addAllValue ] )
4104 threads.append( t )
4105 t.start()
4106 for t in threads:
4107 t.join()
4108 addResponses.append( t.result )
4109
4110 # main.TRUE = successfully changed the set
4111 # main.FALSE = action resulted in no change in set
4112 # main.ERROR - Some error in executing the function
4113 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004114 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004115 if addResponses[ i ] == main.TRUE:
4116 # All is well
4117 pass
4118 elif addResponses[ i ] == main.FALSE:
4119 # Already in set, probably fine
4120 pass
4121 elif addResponses[ i ] == main.ERROR:
4122 # Error in execution
4123 addAllResults = main.FALSE
4124 else:
4125 # unexpected result
4126 addAllResults = main.FALSE
4127 if addAllResults != main.TRUE:
4128 main.log.error( "Error executing set addAll" )
4129
4130 # Check if set is still correct
4131 size = len( onosSet )
4132 getResponses = []
4133 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004134 for i in range( main.numCtrls ):
4135 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004136 name="setTestGet-" + str( i ),
4137 args=[ onosSetName ] )
4138 threads.append( t )
4139 t.start()
4140 for t in threads:
4141 t.join()
4142 getResponses.append( t.result )
4143 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004144 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004145 if isinstance( getResponses[ i ], list):
4146 current = set( getResponses[ i ] )
4147 if len( current ) == len( getResponses[ i ] ):
4148 # no repeats
4149 if onosSet != current:
4150 main.log.error( "ONOS" + str( i + 1 ) +
4151 " has incorrect view" +
4152 " of set " + onosSetName + ":\n" +
4153 str( getResponses[ i ] ) )
4154 main.log.debug( "Expected: " + str( onosSet ) )
4155 main.log.debug( "Actual: " + str( current ) )
4156 getResults = main.FALSE
4157 else:
4158 # error, set is not a set
4159 main.log.error( "ONOS" + str( i + 1 ) +
4160 " has repeat elements in" +
4161 " set " + onosSetName + ":\n" +
4162 str( getResponses[ i ] ) )
4163 getResults = main.FALSE
4164 elif getResponses[ i ] == main.ERROR:
4165 getResults = main.FALSE
4166 sizeResponses = []
4167 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004168 for i in range( main.numCtrls ):
4169 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004170 name="setTestSize-" + str( i ),
4171 args=[ onosSetName ] )
4172 threads.append( t )
4173 t.start()
4174 for t in threads:
4175 t.join()
4176 sizeResponses.append( t.result )
4177 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004178 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004179 if size != sizeResponses[ i ]:
4180 sizeResults = main.FALSE
4181 main.log.error( "ONOS" + str( i + 1 ) +
4182 " expected a size of " + str( size ) +
4183 " for set " + onosSetName +
4184 " but got " + str( sizeResponses[ i ] ) )
4185 addAllResults = addAllResults and getResults and sizeResults
4186 utilities.assert_equals( expect=main.TRUE,
4187 actual=addAllResults,
4188 onpass="Set addAll correct",
4189 onfail="Set addAll was incorrect" )
4190
4191 main.step( "Distributed Set retain()" )
4192 onosSet.intersection_update( retainValue.split() )
4193 retainResponses = []
4194 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004195 for i in range( main.numCtrls ):
4196 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004197 name="setTestRetain-" + str( i ),
4198 args=[ onosSetName, retainValue ],
4199 kwargs={ "retain": True } )
4200 threads.append( t )
4201 t.start()
4202 for t in threads:
4203 t.join()
4204 retainResponses.append( t.result )
4205
4206 # main.TRUE = successfully changed the set
4207 # main.FALSE = action resulted in no change in set
4208 # main.ERROR - Some error in executing the function
4209 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004210 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004211 if retainResponses[ i ] == main.TRUE:
4212 # All is well
4213 pass
4214 elif retainResponses[ i ] == main.FALSE:
4215 # Already in set, probably fine
4216 pass
4217 elif retainResponses[ i ] == main.ERROR:
4218 # Error in execution
4219 retainResults = main.FALSE
4220 else:
4221 # unexpected result
4222 retainResults = main.FALSE
4223 if retainResults != main.TRUE:
4224 main.log.error( "Error executing set retain" )
4225
4226 # Check if set is still correct
4227 size = len( onosSet )
4228 getResponses = []
4229 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004230 for i in range( main.numCtrls ):
4231 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004232 name="setTestGet-" + str( i ),
4233 args=[ onosSetName ] )
4234 threads.append( t )
4235 t.start()
4236 for t in threads:
4237 t.join()
4238 getResponses.append( t.result )
4239 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004240 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004241 if isinstance( getResponses[ i ], list):
4242 current = set( getResponses[ i ] )
4243 if len( current ) == len( getResponses[ i ] ):
4244 # no repeats
4245 if onosSet != current:
4246 main.log.error( "ONOS" + str( i + 1 ) +
4247 " has incorrect view" +
4248 " of set " + onosSetName + ":\n" +
4249 str( getResponses[ i ] ) )
4250 main.log.debug( "Expected: " + str( onosSet ) )
4251 main.log.debug( "Actual: " + str( current ) )
4252 getResults = main.FALSE
4253 else:
4254 # error, set is not a set
4255 main.log.error( "ONOS" + str( i + 1 ) +
4256 " has repeat elements in" +
4257 " set " + onosSetName + ":\n" +
4258 str( getResponses[ i ] ) )
4259 getResults = main.FALSE
4260 elif getResponses[ i ] == main.ERROR:
4261 getResults = main.FALSE
4262 sizeResponses = []
4263 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004264 for i in range( main.numCtrls ):
4265 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004266 name="setTestSize-" + str( i ),
4267 args=[ onosSetName ] )
4268 threads.append( t )
4269 t.start()
4270 for t in threads:
4271 t.join()
4272 sizeResponses.append( t.result )
4273 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004274 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004275 if size != sizeResponses[ i ]:
4276 sizeResults = main.FALSE
4277 main.log.error( "ONOS" + str( i + 1 ) +
4278 " expected a size of " +
4279 str( size ) + " for set " + onosSetName +
4280 " but got " + str( sizeResponses[ i ] ) )
4281 retainResults = retainResults and getResults and sizeResults
4282 utilities.assert_equals( expect=main.TRUE,
4283 actual=retainResults,
4284 onpass="Set retain correct",
4285 onfail="Set retain was incorrect" )
4286
Jon Hall2a5002c2015-08-21 16:49:11 -07004287 # Transactional maps
4288 main.step( "Partitioned Transactional maps put" )
4289 tMapValue = "Testing"
4290 numKeys = 100
4291 putResult = True
4292 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4293 if len( putResponses ) == 100:
4294 for i in putResponses:
4295 if putResponses[ i ][ 'value' ] != tMapValue:
4296 putResult = False
4297 else:
4298 putResult = False
4299 if not putResult:
4300 main.log.debug( "Put response values: " + str( putResponses ) )
4301 utilities.assert_equals( expect=True,
4302 actual=putResult,
4303 onpass="Partitioned Transactional Map put successful",
4304 onfail="Partitioned Transactional Map put values are incorrect" )
4305
4306 main.step( "Partitioned Transactional maps get" )
4307 getCheck = True
4308 for n in range( 1, numKeys + 1 ):
4309 getResponses = []
4310 threads = []
4311 valueCheck = True
4312 for i in range( main.numCtrls ):
4313 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4314 name="TMap-get-" + str( i ),
4315 args=[ "Key" + str ( n ) ] )
4316 threads.append( t )
4317 t.start()
4318 for t in threads:
4319 t.join()
4320 getResponses.append( t.result )
4321 for node in getResponses:
4322 if node != tMapValue:
4323 valueCheck = False
4324 if not valueCheck:
4325 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4326 main.log.warn( getResponses )
4327 getCheck = getCheck and valueCheck
4328 utilities.assert_equals( expect=True,
4329 actual=getCheck,
4330 onpass="Partitioned Transactional Map get values were correct",
4331 onfail="Partitioned Transactional Map values incorrect" )
4332
4333 main.step( "In-memory Transactional maps put" )
4334 tMapValue = "Testing"
4335 numKeys = 100
4336 putResult = True
4337 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4338 if len( putResponses ) == 100:
4339 for i in putResponses:
4340 if putResponses[ i ][ 'value' ] != tMapValue:
4341 putResult = False
4342 else:
4343 putResult = False
4344 if not putResult:
4345 main.log.debug( "Put response values: " + str( putResponses ) )
4346 utilities.assert_equals( expect=True,
4347 actual=putResult,
4348 onpass="In-Memory Transactional Map put successful",
4349 onfail="In-Memory Transactional Map put values are incorrect" )
4350
4351 main.step( "In-Memory Transactional maps get" )
4352 getCheck = True
4353 for n in range( 1, numKeys + 1 ):
4354 getResponses = []
4355 threads = []
4356 valueCheck = True
4357 for i in range( main.numCtrls ):
4358 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4359 name="TMap-get-" + str( i ),
4360 args=[ "Key" + str ( n ) ],
4361 kwargs={ "inMemory": True } )
4362 threads.append( t )
4363 t.start()
4364 for t in threads:
4365 t.join()
4366 getResponses.append( t.result )
4367 for node in getResponses:
4368 if node != tMapValue:
4369 valueCheck = False
4370 if not valueCheck:
4371 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4372 main.log.warn( getResponses )
4373 getCheck = getCheck and valueCheck
4374 utilities.assert_equals( expect=True,
4375 actual=getCheck,
4376 onpass="In-Memory Transactional Map get values were correct",
4377 onfail="In-Memory Transactional Map values incorrect" )