blob: 4d75d8d6104f84992639e90af30e0e7b27a189eb [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
Jon Hallb3ed8ed2015-10-28 16:43:55 -070012CASE61: The Failure inducing case.
13CASE62: The Failure recovery case.
Jon Hall5cf14d52015-07-16 12:15:19 -070014CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
Jon Hallb3ed8ed2015-10-28 16:43:55 -070028class HAkillNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -070029
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall3b489db2015-10-05 14:38:37 -070052 import pexpect
Jon Hall6e709752016-02-01 13:38:46 -080053 import time
Jon Halla440e872016-03-31 15:15:50 -070054 import json
Jon Hall6e709752016-02-01 13:38:46 -080055 main.log.info( "ONOS HA test: Restart a minority of ONOS nodes - " +
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "initialization" )
57 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070058 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070059 "installing ONOS, starting Mininet and ONOS" +\
60 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070061
62 # load some variables from the params file
63 PULLCODE = False
64 if main.params[ 'Git' ] == 'True':
65 PULLCODE = True
66 gitBranch = main.params[ 'branch' ]
67 cellName = main.params[ 'ENV' ][ 'cellName' ]
68
Jon Halle1a3b752015-07-22 13:02:46 -070069 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070071 if main.ONOSbench.maxNodes < main.numCtrls:
72 main.numCtrls = int( main.ONOSbench.maxNodes )
73 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070074 global ONOS1Port
75 global ONOS2Port
76 global ONOS3Port
77 global ONOS4Port
78 global ONOS5Port
79 global ONOS6Port
80 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070081 # These are for csv plotting in jenkins
82 global labels
83 global data
84 labels = []
85 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070086
87 # FIXME: just get controller port from params?
88 # TODO: do we really need all these?
89 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
90 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
91 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
92 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
93 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
94 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
95 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
96
Jon Halle1a3b752015-07-22 13:02:46 -070097 try:
Jon Hall41d39f12016-04-11 22:54:35 -070098 from tests.HAsanity.dependencies.HA import HA
99 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -0700100 except Exception as e:
101 main.log.exception( e )
102 main.cleanup()
103 main.exit()
104
105 main.CLIs = []
106 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700107 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700108 for i in range( 1, main.numCtrls + 1 ):
109 try:
110 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
111 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
112 ipList.append( main.nodes[ -1 ].ip_address )
113 except AttributeError:
114 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700115
116 main.step( "Create cell file" )
117 cellAppString = main.params[ 'ENV' ][ 'appString' ]
118 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
119 main.Mininet1.ip_address,
120 cellAppString, ipList )
121 main.step( "Applying cell variable to environment" )
122 cellResult = main.ONOSbench.setCell( cellName )
123 verifyResult = main.ONOSbench.verifyCell()
124
125 # FIXME:this is short term fix
126 main.log.info( "Removing raft logs" )
127 main.ONOSbench.onosRemoveRaftLogs()
128
129 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700130 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700131 main.ONOSbench.onosUninstall( node.ip_address )
132
133 # Make sure ONOS is DEAD
134 main.log.info( "Killing any ONOS processes" )
135 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700136 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700137 killed = main.ONOSbench.onosKill( node.ip_address )
138 killResults = killResults and killed
139
140 cleanInstallResult = main.TRUE
141 gitPullResult = main.TRUE
142
143 main.step( "Starting Mininet" )
144 # scp topo file to mininet
145 # TODO: move to params?
146 topoName = "obelisk.py"
147 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700148 main.ONOSbench.scp( main.Mininet1,
149 filePath + topoName,
150 main.Mininet1.home,
151 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700152 mnResult = main.Mininet1.startNet( )
153 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
154 onpass="Mininet Started",
155 onfail="Error starting Mininet" )
156
157 main.step( "Git checkout and pull " + gitBranch )
158 if PULLCODE:
159 main.ONOSbench.gitCheckout( gitBranch )
160 gitPullResult = main.ONOSbench.gitPull()
161 # values of 1 or 3 are good
162 utilities.assert_lesser( expect=0, actual=gitPullResult,
163 onpass="Git pull successful",
164 onfail="Git pull failed" )
165 main.ONOSbench.getVersion( report=True )
166
167 main.step( "Using mvn clean install" )
168 cleanInstallResult = main.TRUE
169 if PULLCODE and gitPullResult == main.TRUE:
170 cleanInstallResult = main.ONOSbench.cleanInstall()
171 else:
172 main.log.warn( "Did not pull new code so skipping mvn " +
173 "clean install" )
174 utilities.assert_equals( expect=main.TRUE,
175 actual=cleanInstallResult,
176 onpass="MCI successful",
177 onfail="MCI failed" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700178
179 main.step( "Make sure ONOS service doesn't automatically respawn" )
180 handle = main.ONOSbench.handle
181 handle.sendline( "sed -i -e 's/^respawn$/#respawn/g' tools/package/init/onos.conf" )
182 handle.expect( "\$" ) # $ from the command
183 handle.expect( "\$" ) # $ from the prompt
184
Jon Hall5cf14d52015-07-16 12:15:19 -0700185 # GRAPHS
186 # NOTE: important params here:
187 # job = name of Jenkins job
188 # Plot Name = Plot-HA, only can be used if multiple plots
189 # index = The number of the graph under plot name
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700190 job = "HAkillNodes"
Jon Hall5cf14d52015-07-16 12:15:19 -0700191 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700192 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700193 graphs = '<ac:structured-macro ac:name="html">\n'
194 graphs += '<ac:plain-text-body><![CDATA[\n'
195 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800196 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700197 '&width=500&height=300"' +\
198 'noborder="0" width="500" height="300" scrolling="yes" ' +\
199 'seamless="seamless"></iframe>\n'
200 graphs += ']]></ac:plain-text-body>\n'
201 graphs += '</ac:structured-macro>\n'
202 main.log.wiki(graphs)
203
204 main.step( "Creating ONOS package" )
Jon Hall3b489db2015-10-05 14:38:37 -0700205 # copy gen-partions file to ONOS
206 # NOTE: this assumes TestON and ONOS are on the same machine
207 srcFile = main.testDir + "/" + main.TEST + "/dependencies/onos-gen-partitions"
208 dstDir = main.ONOSbench.home + "/tools/test/bin/onos-gen-partitions"
209 cpResult = main.ONOSbench.secureCopy( main.ONOSbench.user_name,
210 main.ONOSbench.ip_address,
211 srcFile,
212 dstDir,
213 pwd=main.ONOSbench.pwd,
214 direction="from" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700215 packageResult = main.ONOSbench.onosPackage()
216 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
217 onpass="ONOS package successful",
218 onfail="ONOS package failed" )
219
220 main.step( "Installing ONOS package" )
221 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700222 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700223 tmpResult = main.ONOSbench.onosInstall( options="-f",
224 node=node.ip_address )
225 onosInstallResult = onosInstallResult and tmpResult
226 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
227 onpass="ONOS install successful",
228 onfail="ONOS install failed" )
Jon Hall3b489db2015-10-05 14:38:37 -0700229 # clean up gen-partitions file
230 try:
231 main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
232 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
233 main.ONOSbench.handle.sendline( "git checkout -- tools/test/bin/onos-gen-partitions" )
234 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
235 main.log.info( " Cleaning custom gen partitions file, response was: \n" +
236 str( main.ONOSbench.handle.before ) )
237 except ( pexpect.TIMEOUT, pexpect.EOF ):
238 main.log.exception( "ONOSbench: pexpect exception found:" +
239 main.ONOSbench.handle.before )
240 main.cleanup()
241 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700242
243 main.step( "Checking if ONOS is up yet" )
244 for i in range( 2 ):
245 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700246 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700247 started = main.ONOSbench.isup( node.ip_address )
248 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800249 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700250 onosIsupResult = onosIsupResult and started
251 if onosIsupResult == main.TRUE:
252 break
253 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
254 onpass="ONOS startup successful",
255 onfail="ONOS startup failed" )
256
257 main.log.step( "Starting ONOS CLI sessions" )
258 cliResults = main.TRUE
259 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700260 for i in range( main.numCtrls ):
261 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700262 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700263 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700264 threads.append( t )
265 t.start()
266
267 for t in threads:
268 t.join()
269 cliResults = cliResults and t.result
270 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
271 onpass="ONOS cli startup successful",
272 onfail="ONOS cli startup failed" )
273
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700274 # Create a list of active nodes for use when some nodes are stopped
275 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
276
Jon Hall5cf14d52015-07-16 12:15:19 -0700277 if main.params[ 'tcpdump' ].lower() == "true":
278 main.step( "Start Packet Capture MN" )
279 main.Mininet2.startTcpdump(
280 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
281 + "-MN.pcap",
282 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
283 port=main.params[ 'MNtcpdump' ][ 'port' ] )
284
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700285 main.step( "Clean up ONOS service changes" )
286 handle.sendline( "git checkout -- tools/package/init/onos.conf" )
287 handle.expect( "\$" )
288
Jon Halla440e872016-03-31 15:15:50 -0700289 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700290 nodeResults = utilities.retry( main.HA.nodesCheck,
291 False,
292 args=[main.activeNodes],
293 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700294
Jon Hall41d39f12016-04-11 22:54:35 -0700295 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700296 onpass="Nodes check successful",
297 onfail="Nodes check NOT successful" )
298
299 if not nodeResults:
300 for cli in main.CLIs:
301 main.log.debug( "{} components not ACTIVE: \n{}".format(
302 cli.name,
303 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
304
Jon Hall5cf14d52015-07-16 12:15:19 -0700305 if cliResults == main.FALSE:
306 main.log.error( "Failed to start ONOS, stopping test" )
307 main.cleanup()
308 main.exit()
309
Jon Hall172b7ba2016-04-07 18:12:20 -0700310 main.step( "Activate apps defined in the params file" )
311 # get data from the params
312 apps = main.params.get( 'apps' )
313 if apps:
314 apps = apps.split(',')
315 main.log.warn( apps )
316 activateResult = True
317 for app in apps:
318 main.CLIs[ 0 ].app( app, "Activate" )
319 # TODO: check this worked
320 time.sleep( 10 ) # wait for apps to activate
321 for app in apps:
322 state = main.CLIs[ 0 ].appStatus( app )
323 if state == "ACTIVE":
324 activateResult = activeResult and True
325 else:
326 main.log.error( "{} is in {} state".format( app, state ) )
327 activeResult = False
328 utilities.assert_equals( expect=True,
329 actual=activateResult,
330 onpass="Successfully activated apps",
331 onfail="Failed to activate apps" )
332 else:
333 main.log.warn( "No apps were specified to be loaded after startup" )
334
335 main.step( "Set ONOS configurations" )
336 config = main.params.get( 'ONOS_Configuration' )
337 if config:
338 main.log.debug( config )
339 checkResult = main.TRUE
340 for component in config:
341 for setting in config[component]:
342 value = config[component][setting]
343 check = main.CLIs[ 0 ].setCfg( component, setting, value )
344 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
345 checkResult = check and checkResult
346 utilities.assert_equals( expect=main.TRUE,
347 actual=checkResult,
348 onpass="Successfully set config",
349 onfail="Failed to set config" )
350 else:
351 main.log.warn( "No configurations were specified to be changed after startup" )
352
Jon Hall9d2dcad2016-04-08 10:15:20 -0700353 main.step( "App Ids check" )
354 appCheck = main.TRUE
355 threads = []
356 for i in main.activeNodes:
357 t = main.Thread( target=main.CLIs[i].appToIDCheck,
358 name="appToIDCheck-" + str( i ),
359 args=[] )
360 threads.append( t )
361 t.start()
362
363 for t in threads:
364 t.join()
365 appCheck = appCheck and t.result
366 if appCheck != main.TRUE:
367 node = main.activeNodes[0]
368 main.log.warn( main.CLIs[node].apps() )
369 main.log.warn( main.CLIs[node].appIDs() )
370 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
371 onpass="App Ids seem to be correct",
372 onfail="Something is wrong with app Ids" )
373
Jon Hall5cf14d52015-07-16 12:15:19 -0700374 def CASE2( self, main ):
375 """
376 Assign devices to controllers
377 """
378 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700379 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700380 assert main, "main not defined"
381 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700382 assert main.CLIs, "main.CLIs not defined"
383 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700384 assert ONOS1Port, "ONOS1Port not defined"
385 assert ONOS2Port, "ONOS2Port not defined"
386 assert ONOS3Port, "ONOS3Port not defined"
387 assert ONOS4Port, "ONOS4Port not defined"
388 assert ONOS5Port, "ONOS5Port not defined"
389 assert ONOS6Port, "ONOS6Port not defined"
390 assert ONOS7Port, "ONOS7Port not defined"
391
392 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700393 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700394 "and check that an ONOS node becomes the " +\
395 "master of the device."
396 main.step( "Assign switches to controllers" )
397
398 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700399 for i in range( main.numCtrls ):
400 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700401 swList = []
402 for i in range( 1, 29 ):
403 swList.append( "s" + str( i ) )
404 main.Mininet1.assignSwController( sw=swList, ip=ipList )
405
406 mastershipCheck = main.TRUE
407 for i in range( 1, 29 ):
408 response = main.Mininet1.getSwController( "s" + str( i ) )
409 try:
410 main.log.info( str( response ) )
411 except Exception:
412 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700413 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700414 if re.search( "tcp:" + node.ip_address, response ):
415 mastershipCheck = mastershipCheck and main.TRUE
416 else:
417 main.log.error( "Error, node " + node.ip_address + " is " +
418 "not in the list of controllers s" +
419 str( i ) + " is connecting to." )
420 mastershipCheck = main.FALSE
421 utilities.assert_equals(
422 expect=main.TRUE,
423 actual=mastershipCheck,
424 onpass="Switch mastership assigned correctly",
425 onfail="Switches not assigned correctly to controllers" )
426
427 def CASE21( self, main ):
428 """
429 Assign mastership to controllers
430 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700431 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700432 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700433 assert main, "main not defined"
434 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700435 assert main.CLIs, "main.CLIs not defined"
436 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700437 assert ONOS1Port, "ONOS1Port not defined"
438 assert ONOS2Port, "ONOS2Port not defined"
439 assert ONOS3Port, "ONOS3Port not defined"
440 assert ONOS4Port, "ONOS4Port not defined"
441 assert ONOS5Port, "ONOS5Port not defined"
442 assert ONOS6Port, "ONOS6Port not defined"
443 assert ONOS7Port, "ONOS7Port not defined"
444
445 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700446 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700447 "device. Then manually assign" +\
448 " mastership to specific ONOS nodes using" +\
449 " 'device-role'"
450 main.step( "Assign mastership of switches to specific controllers" )
451 # Manually assign mastership to the controller we want
452 roleCall = main.TRUE
453
454 ipList = [ ]
455 deviceList = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700456 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700457 try:
458 # Assign mastership to specific controllers. This assignment was
459 # determined for a 7 node cluser, but will work with any sized
460 # cluster
461 for i in range( 1, 29 ): # switches 1 through 28
462 # set up correct variables:
463 if i == 1:
464 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700465 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700466 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700467 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700468 c = 1 % main.numCtrls
469 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700470 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700471 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700472 c = 1 % main.numCtrls
473 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700474 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700475 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700476 c = 3 % main.numCtrls
477 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700478 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700479 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700480 c = 2 % main.numCtrls
481 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700482 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700483 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700484 c = 2 % main.numCtrls
485 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700486 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700487 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700488 c = 5 % main.numCtrls
489 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700490 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700491 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700492 c = 4 % main.numCtrls
493 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700494 dpid = '3' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700495 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700496 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700497 c = 6 % main.numCtrls
498 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700499 dpid = '6' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700500 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700501 elif i == 28:
502 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700503 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700504 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700505 else:
506 main.log.error( "You didn't write an else statement for " +
507 "switch s" + str( i ) )
508 roleCall = main.FALSE
509 # Assign switch
510 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
511 # TODO: make this controller dynamic
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700512 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700513 ipList.append( ip )
514 deviceList.append( deviceId )
515 except ( AttributeError, AssertionError ):
516 main.log.exception( "Something is wrong with ONOS device view" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700517 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700518 utilities.assert_equals(
519 expect=main.TRUE,
520 actual=roleCall,
521 onpass="Re-assigned switch mastership to designated controller",
522 onfail="Something wrong with deviceRole calls" )
523
524 main.step( "Check mastership was correctly assigned" )
525 roleCheck = main.TRUE
526 # NOTE: This is due to the fact that device mastership change is not
527 # atomic and is actually a multi step process
528 time.sleep( 5 )
529 for i in range( len( ipList ) ):
530 ip = ipList[i]
531 deviceId = deviceList[i]
532 # Check assignment
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700533 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700534 if ip in master:
535 roleCheck = roleCheck and main.TRUE
536 else:
537 roleCheck = roleCheck and main.FALSE
538 main.log.error( "Error, controller " + ip + " is not" +
539 " master " + "of device " +
540 str( deviceId ) + ". Master is " +
541 repr( master ) + "." )
542 utilities.assert_equals(
543 expect=main.TRUE,
544 actual=roleCheck,
545 onpass="Switches were successfully reassigned to designated " +
546 "controller",
547 onfail="Switches were not successfully reassigned" )
548
549 def CASE3( self, main ):
550 """
551 Assign intents
552 """
553 import time
554 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700555 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700556 assert main, "main not defined"
557 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700558 assert main.CLIs, "main.CLIs not defined"
559 assert main.nodes, "main.nodes not defined"
Jon Halla440e872016-03-31 15:15:50 -0700560 try:
561 labels
562 except NameError:
563 main.log.error( "labels not defined, setting to []" )
564 labels = []
565 try:
566 data
567 except NameError:
568 main.log.error( "data not defined, setting to []" )
569 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700570 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700571 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700572 "assign predetermined host-to-host intents." +\
573 " After installation, check that the intent" +\
574 " is distributed to all nodes and the state" +\
575 " is INSTALLED"
576
577 # install onos-app-fwd
578 main.step( "Install reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700579 onosCli = main.CLIs[ main.activeNodes[0] ]
580 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700581 utilities.assert_equals( expect=main.TRUE, actual=installResults,
582 onpass="Install fwd successful",
583 onfail="Install fwd failed" )
584
585 main.step( "Check app ids" )
586 appCheck = main.TRUE
587 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700588 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700589 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700590 name="appToIDCheck-" + str( i ),
591 args=[] )
592 threads.append( t )
593 t.start()
594
595 for t in threads:
596 t.join()
597 appCheck = appCheck and t.result
598 if appCheck != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700599 main.log.warn( onosCli.apps() )
600 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700601 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
602 onpass="App Ids seem to be correct",
603 onfail="Something is wrong with app Ids" )
604
605 main.step( "Discovering Hosts( Via pingall for now )" )
606 # FIXME: Once we have a host discovery mechanism, use that instead
607 # REACTIVE FWD test
608 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700609 passMsg = "Reactive Pingall test passed"
610 time1 = time.time()
611 pingResult = main.Mininet1.pingall()
612 time2 = time.time()
613 if not pingResult:
614 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700615 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700616 passMsg += " on the second try"
617 utilities.assert_equals(
618 expect=main.TRUE,
619 actual=pingResult,
620 onpass= passMsg,
621 onfail="Reactive Pingall failed, " +
622 "one or more ping pairs failed" )
623 main.log.info( "Time for pingall: %2f seconds" %
624 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700625 # timeout for fwd flows
626 time.sleep( 11 )
627 # uninstall onos-app-fwd
628 main.step( "Uninstall reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700629 node = main.activeNodes[0]
630 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700631 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
632 onpass="Uninstall fwd successful",
633 onfail="Uninstall fwd failed" )
634
635 main.step( "Check app ids" )
636 threads = []
637 appCheck2 = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700638 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700639 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700640 name="appToIDCheck-" + str( i ),
641 args=[] )
642 threads.append( t )
643 t.start()
644
645 for t in threads:
646 t.join()
647 appCheck2 = appCheck2 and t.result
648 if appCheck2 != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700649 node = main.activeNodes[0]
650 main.log.warn( main.CLIs[node].apps() )
651 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700652 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
653 onpass="App Ids seem to be correct",
654 onfail="Something is wrong with app Ids" )
655
656 main.step( "Add host intents via cli" )
657 intentIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700658 # TODO: move the host numbers to params
659 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700660 intentAddResult = True
661 hostResult = main.TRUE
662 for i in range( 8, 18 ):
663 main.log.info( "Adding host intent between h" + str( i ) +
664 " and h" + str( i + 10 ) )
665 host1 = "00:00:00:00:00:" + \
666 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
667 host2 = "00:00:00:00:00:" + \
668 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
669 # NOTE: getHost can return None
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700670 host1Dict = onosCli.getHost( host1 )
671 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700672 host1Id = None
673 host2Id = None
674 if host1Dict and host2Dict:
675 host1Id = host1Dict.get( 'id', None )
676 host2Id = host2Dict.get( 'id', None )
677 if host1Id and host2Id:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700678 nodeNum = ( i % len( main.activeNodes ) )
679 node = main.activeNodes[nodeNum]
680 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700681 if tmpId:
682 main.log.info( "Added intent with id: " + tmpId )
683 intentIds.append( tmpId )
684 else:
685 main.log.error( "addHostIntent returned: " +
686 repr( tmpId ) )
687 else:
688 main.log.error( "Error, getHost() failed for h" + str( i ) +
689 " and/or h" + str( i + 10 ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700690 node = main.activeNodes[0]
691 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700692 main.log.warn( "Hosts output: " )
693 try:
694 main.log.warn( json.dumps( json.loads( hosts ),
695 sort_keys=True,
696 indent=4,
697 separators=( ',', ': ' ) ) )
698 except ( ValueError, TypeError ):
699 main.log.warn( repr( hosts ) )
700 hostResult = main.FALSE
701 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
702 onpass="Found a host id for each host",
703 onfail="Error looking up host ids" )
704
705 intentStart = time.time()
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700706 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700707 main.log.info( "Submitted intents: " + str( intentIds ) )
708 main.log.info( "Intents in ONOS: " + str( onosIds ) )
709 for intent in intentIds:
710 if intent in onosIds:
711 pass # intent submitted is in onos
712 else:
713 intentAddResult = False
714 if intentAddResult:
715 intentStop = time.time()
716 else:
717 intentStop = None
718 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700719 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700720 intentStates = []
721 installedCheck = True
722 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
723 count = 0
724 try:
725 for intent in json.loads( intents ):
726 state = intent.get( 'state', None )
727 if "INSTALLED" not in state:
728 installedCheck = False
729 intentId = intent.get( 'id', None )
730 intentStates.append( ( intentId, state ) )
731 except ( ValueError, TypeError ):
732 main.log.exception( "Error parsing intents" )
733 # add submitted intents not in the store
734 tmplist = [ i for i, s in intentStates ]
735 missingIntents = False
736 for i in intentIds:
737 if i not in tmplist:
738 intentStates.append( ( i, " - " ) )
739 missingIntents = True
740 intentStates.sort()
741 for i, s in intentStates:
742 count += 1
743 main.log.info( "%-6s%-15s%-15s" %
744 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700745 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700746 try:
747 missing = False
748 if leaders:
749 parsedLeaders = json.loads( leaders )
750 main.log.warn( json.dumps( parsedLeaders,
751 sort_keys=True,
752 indent=4,
753 separators=( ',', ': ' ) ) )
754 # check for all intent partitions
755 topics = []
756 for i in range( 14 ):
757 topics.append( "intent-partition-" + str( i ) )
758 main.log.debug( topics )
759 ONOStopics = [ j['topic'] for j in parsedLeaders ]
760 for topic in topics:
761 if topic not in ONOStopics:
762 main.log.error( "Error: " + topic +
763 " not in leaders" )
764 missing = True
765 else:
766 main.log.error( "leaders() returned None" )
767 except ( ValueError, TypeError ):
768 main.log.exception( "Error parsing leaders" )
769 main.log.error( repr( leaders ) )
770 # Check all nodes
771 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700772 for i in main.activeNodes:
773 response = main.CLIs[i].leaders( jsonFormat=False)
774 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700775 str( response ) )
776
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700777 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700778 try:
779 if partitions :
780 parsedPartitions = json.loads( partitions )
781 main.log.warn( json.dumps( parsedPartitions,
782 sort_keys=True,
783 indent=4,
784 separators=( ',', ': ' ) ) )
785 # TODO check for a leader in all paritions
786 # TODO check for consistency among nodes
787 else:
788 main.log.error( "partitions() returned None" )
789 except ( ValueError, TypeError ):
790 main.log.exception( "Error parsing partitions" )
791 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700792 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700793 try:
794 if pendingMap :
795 parsedPending = json.loads( pendingMap )
796 main.log.warn( json.dumps( parsedPending,
797 sort_keys=True,
798 indent=4,
799 separators=( ',', ': ' ) ) )
800 # TODO check something here?
801 else:
802 main.log.error( "pendingMap() returned None" )
803 except ( ValueError, TypeError ):
804 main.log.exception( "Error parsing pending map" )
805 main.log.error( repr( pendingMap ) )
806
807 intentAddResult = bool( intentAddResult and not missingIntents and
808 installedCheck )
809 if not intentAddResult:
810 main.log.error( "Error in pushing host intents to ONOS" )
811
812 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700813 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700814 correct = True
815 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700816 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700817 onosIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700818 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700819 onosIds.append( ids )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700820 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700821 str( sorted( onosIds ) ) )
822 if sorted( ids ) != sorted( intentIds ):
823 main.log.warn( "Set of intent IDs doesn't match" )
824 correct = False
825 break
826 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700827 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700828 for intent in intents:
829 if intent[ 'state' ] != "INSTALLED":
830 main.log.warn( "Intent " + intent[ 'id' ] +
831 " is " + intent[ 'state' ] )
832 correct = False
833 break
834 if correct:
835 break
836 else:
837 time.sleep(1)
838 if not intentStop:
839 intentStop = time.time()
840 global gossipTime
841 gossipTime = intentStop - intentStart
842 main.log.info( "It took about " + str( gossipTime ) +
843 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700844 gossipPeriod = int( main.params['timers']['gossip'] )
845 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700846 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700847 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700848 onpass="ECM anti-entropy for intents worked within " +
849 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700850 onfail="Intent ECM anti-entropy took too long. " +
851 "Expected time:{}, Actual time:{}".format( maxGossipTime,
852 gossipTime ) )
853 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700854 intentAddResult = True
855
856 if not intentAddResult or "key" in pendingMap:
857 import time
858 installedCheck = True
859 main.log.info( "Sleeping 60 seconds to see if intents are found" )
860 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700861 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700862 main.log.info( "Submitted intents: " + str( intentIds ) )
863 main.log.info( "Intents in ONOS: " + str( onosIds ) )
864 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700865 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700866 intentStates = []
867 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
868 count = 0
869 try:
870 for intent in json.loads( intents ):
871 # Iter through intents of a node
872 state = intent.get( 'state', None )
873 if "INSTALLED" not in state:
874 installedCheck = False
875 intentId = intent.get( 'id', None )
876 intentStates.append( ( intentId, state ) )
877 except ( ValueError, TypeError ):
878 main.log.exception( "Error parsing intents" )
879 # add submitted intents not in the store
880 tmplist = [ i for i, s in intentStates ]
881 for i in intentIds:
882 if i not in tmplist:
883 intentStates.append( ( i, " - " ) )
884 intentStates.sort()
885 for i, s in intentStates:
886 count += 1
887 main.log.info( "%-6s%-15s%-15s" %
888 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700889 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700890 try:
891 missing = False
892 if leaders:
893 parsedLeaders = json.loads( leaders )
894 main.log.warn( json.dumps( parsedLeaders,
895 sort_keys=True,
896 indent=4,
897 separators=( ',', ': ' ) ) )
898 # check for all intent partitions
899 # check for election
900 topics = []
901 for i in range( 14 ):
902 topics.append( "intent-partition-" + str( i ) )
903 # FIXME: this should only be after we start the app
904 topics.append( "org.onosproject.election" )
905 main.log.debug( topics )
906 ONOStopics = [ j['topic'] for j in parsedLeaders ]
907 for topic in topics:
908 if topic not in ONOStopics:
909 main.log.error( "Error: " + topic +
910 " not in leaders" )
911 missing = True
912 else:
913 main.log.error( "leaders() returned None" )
914 except ( ValueError, TypeError ):
915 main.log.exception( "Error parsing leaders" )
916 main.log.error( repr( leaders ) )
917 # Check all nodes
918 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700919 for i in main.activeNodes:
920 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700921 response = node.leaders( jsonFormat=False)
922 main.log.warn( str( node.name ) + " leaders output: \n" +
923 str( response ) )
924
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700925 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700926 try:
927 if partitions :
928 parsedPartitions = json.loads( partitions )
929 main.log.warn( json.dumps( parsedPartitions,
930 sort_keys=True,
931 indent=4,
932 separators=( ',', ': ' ) ) )
933 # TODO check for a leader in all paritions
934 # TODO check for consistency among nodes
935 else:
936 main.log.error( "partitions() returned None" )
937 except ( ValueError, TypeError ):
938 main.log.exception( "Error parsing partitions" )
939 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700940 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700941 try:
942 if pendingMap :
943 parsedPending = json.loads( pendingMap )
944 main.log.warn( json.dumps( parsedPending,
945 sort_keys=True,
946 indent=4,
947 separators=( ',', ': ' ) ) )
948 # TODO check something here?
949 else:
950 main.log.error( "pendingMap() returned None" )
951 except ( ValueError, TypeError ):
952 main.log.exception( "Error parsing pending map" )
953 main.log.error( repr( pendingMap ) )
954
955 def CASE4( self, main ):
956 """
957 Ping across added host intents
958 """
959 import json
960 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700961 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700962 assert main, "main not defined"
963 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700964 assert main.CLIs, "main.CLIs not defined"
965 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700966 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700967 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700968 "functionality and check the state of " +\
969 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700970
Jon Hall41d39f12016-04-11 22:54:35 -0700971 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700972 main.step( "Check Intent state" )
973 installedCheck = False
974 loopCount = 0
975 while not installedCheck and loopCount < 40:
976 installedCheck = True
977 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700978 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700979 intentStates = []
980 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
981 count = 0
982 # Iter through intents of a node
983 try:
984 for intent in json.loads( intents ):
985 state = intent.get( 'state', None )
986 if "INSTALLED" not in state:
987 installedCheck = False
988 intentId = intent.get( 'id', None )
989 intentStates.append( ( intentId, state ) )
990 except ( ValueError, TypeError ):
991 main.log.exception( "Error parsing intents." )
992 # Print states
993 intentStates.sort()
994 for i, s in intentStates:
995 count += 1
996 main.log.info( "%-6s%-15s%-15s" %
997 ( str( count ), str( i ), str( s ) ) )
998 if not installedCheck:
999 time.sleep( 1 )
1000 loopCount += 1
1001 utilities.assert_equals( expect=True, actual=installedCheck,
1002 onpass="Intents are all INSTALLED",
1003 onfail="Intents are not all in " +
1004 "INSTALLED state" )
1005
Jon Hall9d2dcad2016-04-08 10:15:20 -07001006 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -07001007 PingResult = main.TRUE
1008 for i in range( 8, 18 ):
1009 ping = main.Mininet1.pingHost( src="h" + str( i ),
1010 target="h" + str( i + 10 ) )
1011 PingResult = PingResult and ping
1012 if ping == main.FALSE:
1013 main.log.warn( "Ping failed between h" + str( i ) +
1014 " and h" + str( i + 10 ) )
1015 elif ping == main.TRUE:
1016 main.log.info( "Ping test passed!" )
1017 # Don't set PingResult or you'd override failures
1018 if PingResult == main.FALSE:
1019 main.log.error(
1020 "Intents have not been installed correctly, pings failed." )
1021 # TODO: pretty print
1022 main.log.warn( "ONOS1 intents: " )
1023 try:
1024 tmpIntents = onosCli.intents()
1025 main.log.warn( json.dumps( json.loads( tmpIntents ),
1026 sort_keys=True,
1027 indent=4,
1028 separators=( ',', ': ' ) ) )
1029 except ( ValueError, TypeError ):
1030 main.log.warn( repr( tmpIntents ) )
1031 utilities.assert_equals(
1032 expect=main.TRUE,
1033 actual=PingResult,
1034 onpass="Intents have been installed correctly and pings work",
1035 onfail="Intents have not been installed correctly, pings failed." )
1036
Jon Hall5cf14d52015-07-16 12:15:19 -07001037 main.step( "Check leadership of topics" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001038 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001039 topicCheck = main.TRUE
1040 try:
1041 if leaders:
1042 parsedLeaders = json.loads( leaders )
1043 main.log.warn( json.dumps( parsedLeaders,
1044 sort_keys=True,
1045 indent=4,
1046 separators=( ',', ': ' ) ) )
1047 # check for all intent partitions
1048 # check for election
1049 # TODO: Look at Devices as topics now that it uses this system
1050 topics = []
1051 for i in range( 14 ):
1052 topics.append( "intent-partition-" + str( i ) )
1053 # FIXME: this should only be after we start the app
1054 # FIXME: topics.append( "org.onosproject.election" )
1055 # Print leaders output
1056 main.log.debug( topics )
1057 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1058 for topic in topics:
1059 if topic not in ONOStopics:
1060 main.log.error( "Error: " + topic +
1061 " not in leaders" )
1062 topicCheck = main.FALSE
1063 else:
1064 main.log.error( "leaders() returned None" )
1065 topicCheck = main.FALSE
1066 except ( ValueError, TypeError ):
1067 topicCheck = main.FALSE
1068 main.log.exception( "Error parsing leaders" )
1069 main.log.error( repr( leaders ) )
1070 # TODO: Check for a leader of these topics
1071 # Check all nodes
1072 if topicCheck:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001073 for i in main.activeNodes:
1074 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001075 response = node.leaders( jsonFormat=False)
1076 main.log.warn( str( node.name ) + " leaders output: \n" +
1077 str( response ) )
1078
1079 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1080 onpass="intent Partitions is in leaders",
1081 onfail="Some topics were lost " )
1082 # Print partitions
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001083 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001084 try:
1085 if partitions :
1086 parsedPartitions = json.loads( partitions )
1087 main.log.warn( json.dumps( parsedPartitions,
1088 sort_keys=True,
1089 indent=4,
1090 separators=( ',', ': ' ) ) )
1091 # TODO check for a leader in all paritions
1092 # TODO check for consistency among nodes
1093 else:
1094 main.log.error( "partitions() returned None" )
1095 except ( ValueError, TypeError ):
1096 main.log.exception( "Error parsing partitions" )
1097 main.log.error( repr( partitions ) )
1098 # Print Pending Map
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001099 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001100 try:
1101 if pendingMap :
1102 parsedPending = json.loads( pendingMap )
1103 main.log.warn( json.dumps( parsedPending,
1104 sort_keys=True,
1105 indent=4,
1106 separators=( ',', ': ' ) ) )
1107 # TODO check something here?
1108 else:
1109 main.log.error( "pendingMap() returned None" )
1110 except ( ValueError, TypeError ):
1111 main.log.exception( "Error parsing pending map" )
1112 main.log.error( repr( pendingMap ) )
1113
1114 if not installedCheck:
1115 main.log.info( "Waiting 60 seconds to see if the state of " +
1116 "intents change" )
1117 time.sleep( 60 )
1118 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001119 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001120 intentStates = []
1121 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1122 count = 0
1123 # Iter through intents of a node
1124 try:
1125 for intent in json.loads( intents ):
1126 state = intent.get( 'state', None )
1127 if "INSTALLED" not in state:
1128 installedCheck = False
1129 intentId = intent.get( 'id', None )
1130 intentStates.append( ( intentId, state ) )
1131 except ( ValueError, TypeError ):
1132 main.log.exception( "Error parsing intents." )
1133 intentStates.sort()
1134 for i, s in intentStates:
1135 count += 1
1136 main.log.info( "%-6s%-15s%-15s" %
1137 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001138 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001139 try:
1140 missing = False
1141 if leaders:
1142 parsedLeaders = json.loads( leaders )
1143 main.log.warn( json.dumps( parsedLeaders,
1144 sort_keys=True,
1145 indent=4,
1146 separators=( ',', ': ' ) ) )
1147 # check for all intent partitions
1148 # check for election
1149 topics = []
1150 for i in range( 14 ):
1151 topics.append( "intent-partition-" + str( i ) )
1152 # FIXME: this should only be after we start the app
1153 topics.append( "org.onosproject.election" )
1154 main.log.debug( topics )
1155 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1156 for topic in topics:
1157 if topic not in ONOStopics:
1158 main.log.error( "Error: " + topic +
1159 " not in leaders" )
1160 missing = True
1161 else:
1162 main.log.error( "leaders() returned None" )
1163 except ( ValueError, TypeError ):
1164 main.log.exception( "Error parsing leaders" )
1165 main.log.error( repr( leaders ) )
1166 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001167 for i in main.activeNodes:
1168 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001169 response = node.leaders( jsonFormat=False)
1170 main.log.warn( str( node.name ) + " leaders output: \n" +
1171 str( response ) )
1172
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001173 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001174 try:
1175 if partitions :
1176 parsedPartitions = json.loads( partitions )
1177 main.log.warn( json.dumps( parsedPartitions,
1178 sort_keys=True,
1179 indent=4,
1180 separators=( ',', ': ' ) ) )
1181 # TODO check for a leader in all paritions
1182 # TODO check for consistency among nodes
1183 else:
1184 main.log.error( "partitions() returned None" )
1185 except ( ValueError, TypeError ):
1186 main.log.exception( "Error parsing partitions" )
1187 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001188 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001189 try:
1190 if pendingMap :
1191 parsedPending = json.loads( pendingMap )
1192 main.log.warn( json.dumps( parsedPending,
1193 sort_keys=True,
1194 indent=4,
1195 separators=( ',', ': ' ) ) )
1196 # TODO check something here?
1197 else:
1198 main.log.error( "pendingMap() returned None" )
1199 except ( ValueError, TypeError ):
1200 main.log.exception( "Error parsing pending map" )
1201 main.log.error( repr( pendingMap ) )
1202 # Print flowrules
Jon Hall41d39f12016-04-11 22:54:35 -07001203 main.log.debug( onosCli.flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001204 main.step( "Wait a minute then ping again" )
1205 # the wait is above
1206 PingResult = main.TRUE
1207 for i in range( 8, 18 ):
1208 ping = main.Mininet1.pingHost( src="h" + str( i ),
1209 target="h" + str( i + 10 ) )
1210 PingResult = PingResult and ping
1211 if ping == main.FALSE:
1212 main.log.warn( "Ping failed between h" + str( i ) +
1213 " and h" + str( i + 10 ) )
1214 elif ping == main.TRUE:
1215 main.log.info( "Ping test passed!" )
1216 # Don't set PingResult or you'd override failures
1217 if PingResult == main.FALSE:
1218 main.log.error(
1219 "Intents have not been installed correctly, pings failed." )
1220 # TODO: pretty print
1221 main.log.warn( "ONOS1 intents: " )
1222 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001223 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001224 main.log.warn( json.dumps( json.loads( tmpIntents ),
1225 sort_keys=True,
1226 indent=4,
1227 separators=( ',', ': ' ) ) )
1228 except ( ValueError, TypeError ):
1229 main.log.warn( repr( tmpIntents ) )
1230 utilities.assert_equals(
1231 expect=main.TRUE,
1232 actual=PingResult,
1233 onpass="Intents have been installed correctly and pings work",
1234 onfail="Intents have not been installed correctly, pings failed." )
1235
1236 def CASE5( self, main ):
1237 """
1238 Reading state of ONOS
1239 """
1240 import json
1241 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001242 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001243 assert main, "main not defined"
1244 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001245 assert main.CLIs, "main.CLIs not defined"
1246 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001247
1248 main.case( "Setting up and gathering data for current state" )
1249 # The general idea for this test case is to pull the state of
1250 # ( intents,flows, topology,... ) from each ONOS node
1251 # We can then compare them with each other and also with past states
1252
1253 main.step( "Check that each switch has a master" )
1254 global mastershipState
1255 mastershipState = '[]'
1256
1257 # Assert that each device has a master
1258 rolesNotNull = main.TRUE
1259 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001260 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001261 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001262 name="rolesNotNull-" + str( i ),
1263 args=[] )
1264 threads.append( t )
1265 t.start()
1266
1267 for t in threads:
1268 t.join()
1269 rolesNotNull = rolesNotNull and t.result
1270 utilities.assert_equals(
1271 expect=main.TRUE,
1272 actual=rolesNotNull,
1273 onpass="Each device has a master",
1274 onfail="Some devices don't have a master assigned" )
1275
1276 main.step( "Get the Mastership of each switch from each controller" )
1277 ONOSMastership = []
1278 mastershipCheck = main.FALSE
1279 consistentMastership = True
1280 rolesResults = True
1281 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001282 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001283 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001284 name="roles-" + str( i ),
1285 args=[] )
1286 threads.append( t )
1287 t.start()
1288
1289 for t in threads:
1290 t.join()
1291 ONOSMastership.append( t.result )
1292
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001293 for i in range( len( ONOSMastership ) ):
1294 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001295 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001296 main.log.error( "Error in getting ONOS" + node + " roles" )
1297 main.log.warn( "ONOS" + node + " mastership response: " +
1298 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001299 rolesResults = False
1300 utilities.assert_equals(
1301 expect=True,
1302 actual=rolesResults,
1303 onpass="No error in reading roles output",
1304 onfail="Error in reading roles from ONOS" )
1305
1306 main.step( "Check for consistency in roles from each controller" )
1307 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1308 main.log.info(
1309 "Switch roles are consistent across all ONOS nodes" )
1310 else:
1311 consistentMastership = False
1312 utilities.assert_equals(
1313 expect=True,
1314 actual=consistentMastership,
1315 onpass="Switch roles are consistent across all ONOS nodes",
1316 onfail="ONOS nodes have different views of switch roles" )
1317
1318 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001319 for i in range( len( main.activeNodes ) ):
1320 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001321 try:
1322 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001323 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001324 json.dumps(
1325 json.loads( ONOSMastership[ i ] ),
1326 sort_keys=True,
1327 indent=4,
1328 separators=( ',', ': ' ) ) )
1329 except ( ValueError, TypeError ):
1330 main.log.warn( repr( ONOSMastership[ i ] ) )
1331 elif rolesResults and consistentMastership:
1332 mastershipCheck = main.TRUE
1333 mastershipState = ONOSMastership[ 0 ]
1334
1335 main.step( "Get the intents from each controller" )
1336 global intentState
1337 intentState = []
1338 ONOSIntents = []
1339 intentCheck = main.FALSE
1340 consistentIntents = True
1341 intentsResults = True
1342 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001343 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001344 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001345 name="intents-" + str( i ),
1346 args=[],
1347 kwargs={ 'jsonFormat': True } )
1348 threads.append( t )
1349 t.start()
1350
1351 for t in threads:
1352 t.join()
1353 ONOSIntents.append( t.result )
1354
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001355 for i in range( len( ONOSIntents ) ):
1356 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001357 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001358 main.log.error( "Error in getting ONOS" + node + " intents" )
1359 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001360 repr( ONOSIntents[ i ] ) )
1361 intentsResults = False
1362 utilities.assert_equals(
1363 expect=True,
1364 actual=intentsResults,
1365 onpass="No error in reading intents output",
1366 onfail="Error in reading intents from ONOS" )
1367
1368 main.step( "Check for consistency in Intents from each controller" )
1369 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1370 main.log.info( "Intents are consistent across all ONOS " +
1371 "nodes" )
1372 else:
1373 consistentIntents = False
1374 main.log.error( "Intents not consistent" )
1375 utilities.assert_equals(
1376 expect=True,
1377 actual=consistentIntents,
1378 onpass="Intents are consistent across all ONOS nodes",
1379 onfail="ONOS nodes have different views of intents" )
1380
1381 if intentsResults:
1382 # Try to make it easy to figure out what is happening
1383 #
1384 # Intent ONOS1 ONOS2 ...
1385 # 0x01 INSTALLED INSTALLING
1386 # ... ... ...
1387 # ... ... ...
1388 title = " Id"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001389 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001390 title += " " * 10 + "ONOS" + str( n + 1 )
1391 main.log.warn( title )
1392 # get all intent keys in the cluster
1393 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001394 try:
1395 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001396 for nodeStr in ONOSIntents:
1397 node = json.loads( nodeStr )
1398 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001399 keys.append( intent.get( 'id' ) )
1400 keys = set( keys )
1401 # For each intent key, print the state on each node
1402 for key in keys:
1403 row = "%-13s" % key
1404 for nodeStr in ONOSIntents:
1405 node = json.loads( nodeStr )
1406 for intent in node:
1407 if intent.get( 'id', "Error" ) == key:
1408 row += "%-15s" % intent.get( 'state' )
1409 main.log.warn( row )
1410 # End of intent state table
1411 except ValueError as e:
1412 main.log.exception( e )
1413 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001414
1415 if intentsResults and not consistentIntents:
1416 # print the json objects
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001417 n = str( main.activeNodes[-1] + 1 )
1418 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001419 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1420 sort_keys=True,
1421 indent=4,
1422 separators=( ',', ': ' ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001423 for i in range( len( ONOSIntents ) ):
1424 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001425 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001426 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001427 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1428 sort_keys=True,
1429 indent=4,
1430 separators=( ',', ': ' ) ) )
1431 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001432 main.log.debug( "ONOS" + node + " intents match ONOS" +
1433 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001434 elif intentsResults and consistentIntents:
1435 intentCheck = main.TRUE
1436 intentState = ONOSIntents[ 0 ]
1437
1438 main.step( "Get the flows from each controller" )
1439 global flowState
1440 flowState = []
1441 ONOSFlows = []
1442 ONOSFlowsJson = []
1443 flowCheck = main.FALSE
1444 consistentFlows = True
1445 flowsResults = True
1446 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001447 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001448 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001449 name="flows-" + str( i ),
1450 args=[],
1451 kwargs={ 'jsonFormat': True } )
1452 threads.append( t )
1453 t.start()
1454
1455 # NOTE: Flows command can take some time to run
1456 time.sleep(30)
1457 for t in threads:
1458 t.join()
1459 result = t.result
1460 ONOSFlows.append( result )
1461
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001462 for i in range( len( ONOSFlows ) ):
1463 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001464 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1465 main.log.error( "Error in getting ONOS" + num + " flows" )
1466 main.log.warn( "ONOS" + num + " flows response: " +
1467 repr( ONOSFlows[ i ] ) )
1468 flowsResults = False
1469 ONOSFlowsJson.append( None )
1470 else:
1471 try:
1472 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1473 except ( ValueError, TypeError ):
1474 # FIXME: change this to log.error?
1475 main.log.exception( "Error in parsing ONOS" + num +
1476 " response as json." )
1477 main.log.error( repr( ONOSFlows[ i ] ) )
1478 ONOSFlowsJson.append( None )
1479 flowsResults = False
1480 utilities.assert_equals(
1481 expect=True,
1482 actual=flowsResults,
1483 onpass="No error in reading flows output",
1484 onfail="Error in reading flows from ONOS" )
1485
1486 main.step( "Check for consistency in Flows from each controller" )
1487 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1488 if all( tmp ):
1489 main.log.info( "Flow count is consistent across all ONOS nodes" )
1490 else:
1491 consistentFlows = False
1492 utilities.assert_equals(
1493 expect=True,
1494 actual=consistentFlows,
1495 onpass="The flow count is consistent across all ONOS nodes",
1496 onfail="ONOS nodes have different flow counts" )
1497
1498 if flowsResults and not consistentFlows:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001499 for i in range( len( ONOSFlows ) ):
1500 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001501 try:
1502 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001503 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001504 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1505 indent=4, separators=( ',', ': ' ) ) )
1506 except ( ValueError, TypeError ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001507 main.log.warn( "ONOS" + node + " flows: " +
1508 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001509 elif flowsResults and consistentFlows:
1510 flowCheck = main.TRUE
1511 flowState = ONOSFlows[ 0 ]
1512
1513 main.step( "Get the OF Table entries" )
1514 global flows
1515 flows = []
1516 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001517 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001518 if flowCheck == main.FALSE:
1519 for table in flows:
1520 main.log.warn( table )
1521 # TODO: Compare switch flow tables with ONOS flow tables
1522
1523 main.step( "Start continuous pings" )
1524 main.Mininet2.pingLong(
1525 src=main.params[ 'PING' ][ 'source1' ],
1526 target=main.params[ 'PING' ][ 'target1' ],
1527 pingTime=500 )
1528 main.Mininet2.pingLong(
1529 src=main.params[ 'PING' ][ 'source2' ],
1530 target=main.params[ 'PING' ][ 'target2' ],
1531 pingTime=500 )
1532 main.Mininet2.pingLong(
1533 src=main.params[ 'PING' ][ 'source3' ],
1534 target=main.params[ 'PING' ][ 'target3' ],
1535 pingTime=500 )
1536 main.Mininet2.pingLong(
1537 src=main.params[ 'PING' ][ 'source4' ],
1538 target=main.params[ 'PING' ][ 'target4' ],
1539 pingTime=500 )
1540 main.Mininet2.pingLong(
1541 src=main.params[ 'PING' ][ 'source5' ],
1542 target=main.params[ 'PING' ][ 'target5' ],
1543 pingTime=500 )
1544 main.Mininet2.pingLong(
1545 src=main.params[ 'PING' ][ 'source6' ],
1546 target=main.params[ 'PING' ][ 'target6' ],
1547 pingTime=500 )
1548 main.Mininet2.pingLong(
1549 src=main.params[ 'PING' ][ 'source7' ],
1550 target=main.params[ 'PING' ][ 'target7' ],
1551 pingTime=500 )
1552 main.Mininet2.pingLong(
1553 src=main.params[ 'PING' ][ 'source8' ],
1554 target=main.params[ 'PING' ][ 'target8' ],
1555 pingTime=500 )
1556 main.Mininet2.pingLong(
1557 src=main.params[ 'PING' ][ 'source9' ],
1558 target=main.params[ 'PING' ][ 'target9' ],
1559 pingTime=500 )
1560 main.Mininet2.pingLong(
1561 src=main.params[ 'PING' ][ 'source10' ],
1562 target=main.params[ 'PING' ][ 'target10' ],
1563 pingTime=500 )
1564
1565 main.step( "Collecting topology information from ONOS" )
1566 devices = []
1567 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001568 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001569 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001570 name="devices-" + str( i ),
1571 args=[ ] )
1572 threads.append( t )
1573 t.start()
1574
1575 for t in threads:
1576 t.join()
1577 devices.append( t.result )
1578 hosts = []
1579 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001580 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001581 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001582 name="hosts-" + str( i ),
1583 args=[ ] )
1584 threads.append( t )
1585 t.start()
1586
1587 for t in threads:
1588 t.join()
1589 try:
1590 hosts.append( json.loads( t.result ) )
1591 except ( ValueError, TypeError ):
1592 # FIXME: better handling of this, print which node
1593 # Maybe use thread name?
1594 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001595 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001596 hosts.append( None )
1597
1598 ports = []
1599 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001600 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001601 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001602 name="ports-" + str( i ),
1603 args=[ ] )
1604 threads.append( t )
1605 t.start()
1606
1607 for t in threads:
1608 t.join()
1609 ports.append( t.result )
1610 links = []
1611 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001612 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001613 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001614 name="links-" + str( i ),
1615 args=[ ] )
1616 threads.append( t )
1617 t.start()
1618
1619 for t in threads:
1620 t.join()
1621 links.append( t.result )
1622 clusters = []
1623 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001624 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001625 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001626 name="clusters-" + str( i ),
1627 args=[ ] )
1628 threads.append( t )
1629 t.start()
1630
1631 for t in threads:
1632 t.join()
1633 clusters.append( t.result )
1634 # Compare json objects for hosts and dataplane clusters
1635
1636 # hosts
1637 main.step( "Host view is consistent across ONOS nodes" )
1638 consistentHostsResult = main.TRUE
1639 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001640 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001641 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001642 if hosts[ controller ] == hosts[ 0 ]:
1643 continue
1644 else: # hosts not consistent
1645 main.log.error( "hosts from ONOS" +
1646 controllerStr +
1647 " is inconsistent with ONOS1" )
1648 main.log.warn( repr( hosts[ controller ] ) )
1649 consistentHostsResult = main.FALSE
1650
1651 else:
1652 main.log.error( "Error in getting ONOS hosts from ONOS" +
1653 controllerStr )
1654 consistentHostsResult = main.FALSE
1655 main.log.warn( "ONOS" + controllerStr +
1656 " hosts response: " +
1657 repr( hosts[ controller ] ) )
1658 utilities.assert_equals(
1659 expect=main.TRUE,
1660 actual=consistentHostsResult,
1661 onpass="Hosts view is consistent across all ONOS nodes",
1662 onfail="ONOS nodes have different views of hosts" )
1663
1664 main.step( "Each host has an IP address" )
1665 ipResult = main.TRUE
1666 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001667 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001668 if hosts[ controller ]:
1669 for host in hosts[ controller ]:
1670 if not host.get( 'ipAddresses', [ ] ):
1671 main.log.error( "Error with host ips on controller" +
1672 controllerStr + ": " + str( host ) )
1673 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001674 utilities.assert_equals(
1675 expect=main.TRUE,
1676 actual=ipResult,
1677 onpass="The ips of the hosts aren't empty",
1678 onfail="The ip of at least one host is missing" )
1679
1680 # Strongly connected clusters of devices
1681 main.step( "Cluster view is consistent across ONOS nodes" )
1682 consistentClustersResult = main.TRUE
1683 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001684 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001685 if "Error" not in clusters[ controller ]:
1686 if clusters[ controller ] == clusters[ 0 ]:
1687 continue
1688 else: # clusters not consistent
1689 main.log.error( "clusters from ONOS" + controllerStr +
1690 " is inconsistent with ONOS1" )
1691 consistentClustersResult = main.FALSE
1692
1693 else:
1694 main.log.error( "Error in getting dataplane clusters " +
1695 "from ONOS" + controllerStr )
1696 consistentClustersResult = main.FALSE
1697 main.log.warn( "ONOS" + controllerStr +
1698 " clusters response: " +
1699 repr( clusters[ controller ] ) )
1700 utilities.assert_equals(
1701 expect=main.TRUE,
1702 actual=consistentClustersResult,
1703 onpass="Clusters view is consistent across all ONOS nodes",
1704 onfail="ONOS nodes have different views of clusters" )
Jon Hall172b7ba2016-04-07 18:12:20 -07001705 if consistentClustersResult != main.TRUE:
1706 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07001707 # there should always only be one cluster
1708 main.step( "Cluster view correct across ONOS nodes" )
1709 try:
1710 numClusters = len( json.loads( clusters[ 0 ] ) )
1711 except ( ValueError, TypeError ):
1712 main.log.exception( "Error parsing clusters[0]: " +
1713 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001714 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001715 clusterResults = main.FALSE
1716 if numClusters == 1:
1717 clusterResults = main.TRUE
1718 utilities.assert_equals(
1719 expect=1,
1720 actual=numClusters,
1721 onpass="ONOS shows 1 SCC",
1722 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1723
1724 main.step( "Comparing ONOS topology to MN" )
1725 devicesResults = main.TRUE
1726 linksResults = main.TRUE
1727 hostsResults = main.TRUE
1728 mnSwitches = main.Mininet1.getSwitches()
1729 mnLinks = main.Mininet1.getLinks()
1730 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001731 for controller in main.activeNodes:
1732 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001733 if devices[ controller ] and ports[ controller ] and\
1734 "Error" not in devices[ controller ] and\
1735 "Error" not in ports[ controller ]:
Jon Hall6e709752016-02-01 13:38:46 -08001736 currentDevicesResult = main.Mininet1.compareSwitches(
1737 mnSwitches,
1738 json.loads( devices[ controller ] ),
1739 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001740 else:
1741 currentDevicesResult = main.FALSE
1742 utilities.assert_equals( expect=main.TRUE,
1743 actual=currentDevicesResult,
1744 onpass="ONOS" + controllerStr +
1745 " Switches view is correct",
1746 onfail="ONOS" + controllerStr +
1747 " Switches view is incorrect" )
1748 if links[ controller ] and "Error" not in links[ controller ]:
1749 currentLinksResult = main.Mininet1.compareLinks(
1750 mnSwitches, mnLinks,
1751 json.loads( links[ controller ] ) )
1752 else:
1753 currentLinksResult = main.FALSE
1754 utilities.assert_equals( expect=main.TRUE,
1755 actual=currentLinksResult,
1756 onpass="ONOS" + controllerStr +
1757 " links view is correct",
1758 onfail="ONOS" + controllerStr +
1759 " links view is incorrect" )
1760
Jon Hall657cdf62015-12-17 14:40:51 -08001761 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001762 currentHostsResult = main.Mininet1.compareHosts(
1763 mnHosts,
1764 hosts[ controller ] )
1765 else:
1766 currentHostsResult = main.FALSE
1767 utilities.assert_equals( expect=main.TRUE,
1768 actual=currentHostsResult,
1769 onpass="ONOS" + controllerStr +
1770 " hosts exist in Mininet",
1771 onfail="ONOS" + controllerStr +
1772 " hosts don't match Mininet" )
1773
1774 devicesResults = devicesResults and currentDevicesResult
1775 linksResults = linksResults and currentLinksResult
1776 hostsResults = hostsResults and currentHostsResult
1777
1778 main.step( "Device information is correct" )
1779 utilities.assert_equals(
1780 expect=main.TRUE,
1781 actual=devicesResults,
1782 onpass="Device information is correct",
1783 onfail="Device information is incorrect" )
1784
1785 main.step( "Links are correct" )
1786 utilities.assert_equals(
1787 expect=main.TRUE,
1788 actual=linksResults,
1789 onpass="Link are correct",
1790 onfail="Links are incorrect" )
1791
1792 main.step( "Hosts are correct" )
1793 utilities.assert_equals(
1794 expect=main.TRUE,
1795 actual=hostsResults,
1796 onpass="Hosts are correct",
1797 onfail="Hosts are incorrect" )
1798
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001799 def CASE61( self, main ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001800 """
1801 The Failure case.
1802 """
Jon Halle1a3b752015-07-22 13:02:46 -07001803 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001804 assert main, "main not defined"
1805 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001806 assert main.CLIs, "main.CLIs not defined"
1807 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001808 main.case( "Kill minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -07001809
1810 main.step( "Checking ONOS Logs for errors" )
1811 for node in main.nodes:
1812 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1813 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1814
Jon Hall3b489db2015-10-05 14:38:37 -07001815 n = len( main.nodes ) # Number of nodes
1816 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
1817 main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
1818 if n > 3:
1819 main.kill.append( p - 1 )
1820 # NOTE: This only works for cluster sizes of 3,5, or 7.
1821
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001822 main.step( "Kill " + str( len( main.kill ) ) + " ONOS nodes" )
Jon Hall3b489db2015-10-05 14:38:37 -07001823 killResults = main.TRUE
1824 for i in main.kill:
1825 killResults = killResults and\
1826 main.ONOSbench.onosKill( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001827 main.activeNodes.remove( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001828 utilities.assert_equals( expect=main.TRUE, actual=killResults,
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001829 onpass="ONOS nodes killed successfully",
1830 onfail="ONOS nodes NOT successfully killed" )
1831
1832 def CASE62( self, main ):
1833 """
1834 The bring up stopped nodes
1835 """
1836 import time
1837 assert main.numCtrls, "main.numCtrls not defined"
1838 assert main, "main not defined"
1839 assert utilities.assert_equals, "utilities.assert_equals not defined"
1840 assert main.CLIs, "main.CLIs not defined"
1841 assert main.nodes, "main.nodes not defined"
1842 assert main.kill, "main.kill not defined"
1843 main.case( "Restart minority of ONOS nodes" )
1844
1845 main.step( "Restarting " + str( len( main.kill ) ) + " ONOS nodes" )
1846 startResults = main.TRUE
1847 restartTime = time.time()
1848 for i in main.kill:
1849 startResults = startResults and\
1850 main.ONOSbench.onosStart( main.nodes[i].ip_address )
1851 utilities.assert_equals( expect=main.TRUE, actual=startResults,
1852 onpass="ONOS nodes started successfully",
1853 onfail="ONOS nodes NOT successfully started" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001854
1855 main.step( "Checking if ONOS is up yet" )
1856 count = 0
1857 onosIsupResult = main.FALSE
1858 while onosIsupResult == main.FALSE and count < 10:
Jon Hall3b489db2015-10-05 14:38:37 -07001859 onosIsupResult = main.TRUE
1860 for i in main.kill:
1861 onosIsupResult = onosIsupResult and\
1862 main.ONOSbench.isup( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001863 count = count + 1
Jon Hall5cf14d52015-07-16 12:15:19 -07001864 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1865 onpass="ONOS restarted successfully",
1866 onfail="ONOS restart NOT successful" )
1867
Jon Halle1a3b752015-07-22 13:02:46 -07001868 main.step( "Restarting ONOS main.CLIs" )
Jon Hall3b489db2015-10-05 14:38:37 -07001869 cliResults = main.TRUE
1870 for i in main.kill:
1871 cliResults = cliResults and\
1872 main.CLIs[i].startOnosCli( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001873 main.activeNodes.append( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001874 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1875 onpass="ONOS cli restarted",
1876 onfail="ONOS cli did not restart" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001877 main.activeNodes.sort()
1878 try:
1879 assert list( set( main.activeNodes ) ) == main.activeNodes,\
1880 "List of active nodes has duplicates, this likely indicates something was run out of order"
1881 except AssertionError:
1882 main.log.exception( "" )
1883 main.cleanup()
1884 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -07001885
1886 # Grab the time of restart so we chan check how long the gossip
1887 # protocol has had time to work
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001888 main.restartTime = time.time() - restartTime
Jon Hall5cf14d52015-07-16 12:15:19 -07001889 main.log.debug( "Restart time: " + str( main.restartTime ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001890 # TODO: MAke this configurable. Also, we are breaking the above timer
1891 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001892 node = main.activeNodes[0]
1893 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1894 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1895 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001896
Jon Halla440e872016-03-31 15:15:50 -07001897 main.step( "Rerun for election on the node(s) that were killed" )
1898 runResults = main.TRUE
1899 for i in main.kill:
1900 runResults = runResults and\
1901 main.CLIs[i].electionTestRun()
1902 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1903 onpass="ONOS nodes reran for election topic",
1904 onfail="Errror rerunning for election" )
1905
Jon Hall5cf14d52015-07-16 12:15:19 -07001906 def CASE7( self, main ):
1907 """
1908 Check state after ONOS failure
1909 """
1910 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001911 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001912 assert main, "main not defined"
1913 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001914 assert main.CLIs, "main.CLIs not defined"
1915 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001916 try:
1917 main.kill
1918 except AttributeError:
1919 main.kill = []
1920
Jon Hall5cf14d52015-07-16 12:15:19 -07001921 main.case( "Running ONOS Constant State Tests" )
1922
1923 main.step( "Check that each switch has a master" )
1924 # Assert that each device has a master
1925 rolesNotNull = main.TRUE
1926 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001927 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001928 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001929 name="rolesNotNull-" + str( i ),
1930 args=[ ] )
1931 threads.append( t )
1932 t.start()
1933
1934 for t in threads:
1935 t.join()
1936 rolesNotNull = rolesNotNull and t.result
1937 utilities.assert_equals(
1938 expect=main.TRUE,
1939 actual=rolesNotNull,
1940 onpass="Each device has a master",
1941 onfail="Some devices don't have a master assigned" )
1942
1943 main.step( "Read device roles from ONOS" )
1944 ONOSMastership = []
Jon Halla440e872016-03-31 15:15:50 -07001945 mastershipCheck = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001946 consistentMastership = True
1947 rolesResults = True
1948 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001949 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001950 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001951 name="roles-" + str( i ),
1952 args=[] )
1953 threads.append( t )
1954 t.start()
1955
1956 for t in threads:
1957 t.join()
1958 ONOSMastership.append( t.result )
1959
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001960 for i in range( len( ONOSMastership ) ):
1961 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001962 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001963 main.log.error( "Error in getting ONOS" + node + " roles" )
1964 main.log.warn( "ONOS" + node + " mastership response: " +
1965 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001966 rolesResults = False
1967 utilities.assert_equals(
1968 expect=True,
1969 actual=rolesResults,
1970 onpass="No error in reading roles output",
1971 onfail="Error in reading roles from ONOS" )
1972
1973 main.step( "Check for consistency in roles from each controller" )
1974 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1975 main.log.info(
1976 "Switch roles are consistent across all ONOS nodes" )
1977 else:
1978 consistentMastership = False
1979 utilities.assert_equals(
1980 expect=True,
1981 actual=consistentMastership,
1982 onpass="Switch roles are consistent across all ONOS nodes",
1983 onfail="ONOS nodes have different views of switch roles" )
1984
1985 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001986 for i in range( len( ONOSMastership ) ):
1987 node = str( main.activeNodes[i] + 1 )
1988 main.log.warn( "ONOS" + node + " roles: ",
1989 json.dumps( json.loads( ONOSMastership[ i ] ),
1990 sort_keys=True,
1991 indent=4,
1992 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001993 elif rolesResults and consistentMastership:
1994 mastershipCheck = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07001995
1996 # NOTE: we expect mastership to change on controller failure
Jon Hall5cf14d52015-07-16 12:15:19 -07001997
1998 main.step( "Get the intents and compare across all nodes" )
1999 ONOSIntents = []
2000 intentCheck = main.FALSE
2001 consistentIntents = True
2002 intentsResults = True
2003 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002004 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002005 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07002006 name="intents-" + str( i ),
2007 args=[],
2008 kwargs={ 'jsonFormat': True } )
2009 threads.append( t )
2010 t.start()
2011
2012 for t in threads:
2013 t.join()
2014 ONOSIntents.append( t.result )
2015
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002016 for i in range( len( ONOSIntents) ):
2017 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002018 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002019 main.log.error( "Error in getting ONOS" + node + " intents" )
2020 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002021 repr( ONOSIntents[ i ] ) )
2022 intentsResults = False
2023 utilities.assert_equals(
2024 expect=True,
2025 actual=intentsResults,
2026 onpass="No error in reading intents output",
2027 onfail="Error in reading intents from ONOS" )
2028
2029 main.step( "Check for consistency in Intents from each controller" )
2030 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2031 main.log.info( "Intents are consistent across all ONOS " +
2032 "nodes" )
2033 else:
2034 consistentIntents = False
2035
2036 # Try to make it easy to figure out what is happening
2037 #
2038 # Intent ONOS1 ONOS2 ...
2039 # 0x01 INSTALLED INSTALLING
2040 # ... ... ...
2041 # ... ... ...
2042 title = " ID"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002043 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002044 title += " " * 10 + "ONOS" + str( n + 1 )
2045 main.log.warn( title )
2046 # get all intent keys in the cluster
2047 keys = []
2048 for nodeStr in ONOSIntents:
2049 node = json.loads( nodeStr )
2050 for intent in node:
2051 keys.append( intent.get( 'id' ) )
2052 keys = set( keys )
2053 for key in keys:
2054 row = "%-13s" % key
2055 for nodeStr in ONOSIntents:
2056 node = json.loads( nodeStr )
2057 for intent in node:
2058 if intent.get( 'id' ) == key:
2059 row += "%-15s" % intent.get( 'state' )
2060 main.log.warn( row )
2061 # End table view
2062
2063 utilities.assert_equals(
2064 expect=True,
2065 actual=consistentIntents,
2066 onpass="Intents are consistent across all ONOS nodes",
2067 onfail="ONOS nodes have different views of intents" )
2068 intentStates = []
2069 for node in ONOSIntents: # Iter through ONOS nodes
2070 nodeStates = []
2071 # Iter through intents of a node
2072 try:
2073 for intent in json.loads( node ):
2074 nodeStates.append( intent[ 'state' ] )
2075 except ( ValueError, TypeError ):
2076 main.log.exception( "Error in parsing intents" )
2077 main.log.error( repr( node ) )
2078 intentStates.append( nodeStates )
2079 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2080 main.log.info( dict( out ) )
2081
2082 if intentsResults and not consistentIntents:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002083 for i in range( len( main.activeNodes ) ):
2084 node = str( main.activeNodes[i] + 1 )
2085 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002086 main.log.warn( json.dumps(
2087 json.loads( ONOSIntents[ i ] ),
2088 sort_keys=True,
2089 indent=4,
2090 separators=( ',', ': ' ) ) )
2091 elif intentsResults and consistentIntents:
2092 intentCheck = main.TRUE
2093
2094 # NOTE: Store has no durability, so intents are lost across system
2095 # restarts
2096 main.step( "Compare current intents with intents before the failure" )
2097 # NOTE: this requires case 5 to pass for intentState to be set.
2098 # maybe we should stop the test if that fails?
2099 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002100 try:
2101 intentState
2102 except NameError:
2103 main.log.warn( "No previous intent state was saved" )
2104 else:
2105 if intentState and intentState == ONOSIntents[ 0 ]:
2106 sameIntents = main.TRUE
2107 main.log.info( "Intents are consistent with before failure" )
2108 # TODO: possibly the states have changed? we may need to figure out
2109 # what the acceptable states are
2110 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2111 sameIntents = main.TRUE
2112 try:
2113 before = json.loads( intentState )
2114 after = json.loads( ONOSIntents[ 0 ] )
2115 for intent in before:
2116 if intent not in after:
2117 sameIntents = main.FALSE
2118 main.log.debug( "Intent is not currently in ONOS " +
2119 "(at least in the same form):" )
2120 main.log.debug( json.dumps( intent ) )
2121 except ( ValueError, TypeError ):
2122 main.log.exception( "Exception printing intents" )
2123 main.log.debug( repr( ONOSIntents[0] ) )
2124 main.log.debug( repr( intentState ) )
2125 if sameIntents == main.FALSE:
2126 try:
2127 main.log.debug( "ONOS intents before: " )
2128 main.log.debug( json.dumps( json.loads( intentState ),
2129 sort_keys=True, indent=4,
2130 separators=( ',', ': ' ) ) )
2131 main.log.debug( "Current ONOS intents: " )
2132 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2133 sort_keys=True, indent=4,
2134 separators=( ',', ': ' ) ) )
2135 except ( ValueError, TypeError ):
2136 main.log.exception( "Exception printing intents" )
2137 main.log.debug( repr( ONOSIntents[0] ) )
2138 main.log.debug( repr( intentState ) )
2139 utilities.assert_equals(
2140 expect=main.TRUE,
2141 actual=sameIntents,
2142 onpass="Intents are consistent with before failure",
2143 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002144 intentCheck = intentCheck and sameIntents
2145
2146 main.step( "Get the OF Table entries and compare to before " +
2147 "component failure" )
2148 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002149 for i in range( 28 ):
2150 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002151 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002152 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2153 FlowTables = FlowTables and curSwitch
2154 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002155 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002156 utilities.assert_equals(
2157 expect=main.TRUE,
2158 actual=FlowTables,
2159 onpass="No changes were found in the flow tables",
2160 onfail="Changes were found in the flow tables" )
2161
2162 main.Mininet2.pingLongKill()
2163 '''
2164 main.step( "Check the continuous pings to ensure that no packets " +
2165 "were dropped during component failure" )
2166 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2167 main.params[ 'TESTONIP' ] )
2168 LossInPings = main.FALSE
2169 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2170 for i in range( 8, 18 ):
2171 main.log.info(
2172 "Checking for a loss in pings along flow from s" +
2173 str( i ) )
2174 LossInPings = main.Mininet2.checkForLoss(
2175 "/tmp/ping.h" +
2176 str( i ) ) or LossInPings
2177 if LossInPings == main.TRUE:
2178 main.log.info( "Loss in ping detected" )
2179 elif LossInPings == main.ERROR:
2180 main.log.info( "There are multiple mininet process running" )
2181 elif LossInPings == main.FALSE:
2182 main.log.info( "No Loss in the pings" )
2183 main.log.info( "No loss of dataplane connectivity" )
2184 utilities.assert_equals(
2185 expect=main.FALSE,
2186 actual=LossInPings,
2187 onpass="No Loss of connectivity",
2188 onfail="Loss of dataplane connectivity detected" )
2189 '''
2190
2191 main.step( "Leadership Election is still functional" )
2192 # Test of LeadershipElection
2193 leaderList = []
Jon Hall5cf14d52015-07-16 12:15:19 -07002194
Jon Hall3b489db2015-10-05 14:38:37 -07002195 restarted = []
2196 for i in main.kill:
2197 restarted.append( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002198 leaderResult = main.TRUE
Jon Hall3b489db2015-10-05 14:38:37 -07002199
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002200 for i in main.activeNodes:
2201 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002202 leaderN = cli.electionTestLeader()
2203 leaderList.append( leaderN )
2204 if leaderN == main.FALSE:
2205 # error in response
2206 main.log.error( "Something is wrong with " +
2207 "electionTestLeader function, check the" +
2208 " error logs" )
2209 leaderResult = main.FALSE
2210 elif leaderN is None:
2211 main.log.error( cli.name +
2212 " shows no leader for the election-app was" +
2213 " elected after the old one died" )
2214 leaderResult = main.FALSE
2215 elif leaderN in restarted:
2216 main.log.error( cli.name + " shows " + str( leaderN ) +
2217 " as leader for the election-app, but it " +
2218 "was restarted" )
2219 leaderResult = main.FALSE
2220 if len( set( leaderList ) ) != 1:
2221 leaderResult = main.FALSE
2222 main.log.error(
2223 "Inconsistent view of leader for the election test app" )
2224 # TODO: print the list
2225 utilities.assert_equals(
2226 expect=main.TRUE,
2227 actual=leaderResult,
2228 onpass="Leadership election passed",
2229 onfail="Something went wrong with Leadership election" )
2230
2231 def CASE8( self, main ):
2232 """
2233 Compare topo
2234 """
2235 import json
2236 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002237 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002238 assert main, "main not defined"
2239 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002240 assert main.CLIs, "main.CLIs not defined"
2241 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002242
2243 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002244 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002245 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002246 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002247 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002248 elapsed = 0
2249 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002250 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002251 startTime = time.time()
2252 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002253 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002254 devicesResults = main.TRUE
2255 linksResults = main.TRUE
2256 hostsResults = main.TRUE
2257 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002258 count += 1
2259 cliStart = time.time()
2260 devices = []
2261 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002262 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002263 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002264 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002265 args=[ main.CLIs[i].devices, [ None ] ],
2266 kwargs= { 'sleep': 5, 'attempts': 5,
2267 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002268 threads.append( t )
2269 t.start()
2270
2271 for t in threads:
2272 t.join()
2273 devices.append( t.result )
2274 hosts = []
2275 ipResult = main.TRUE
2276 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002277 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002278 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002279 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002280 args=[ main.CLIs[i].hosts, [ None ] ],
2281 kwargs= { 'sleep': 5, 'attempts': 5,
2282 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002283 threads.append( t )
2284 t.start()
2285
2286 for t in threads:
2287 t.join()
2288 try:
2289 hosts.append( json.loads( t.result ) )
2290 except ( ValueError, TypeError ):
2291 main.log.exception( "Error parsing hosts results" )
2292 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002293 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002294 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002295 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002296 if hosts[ controller ]:
2297 for host in hosts[ controller ]:
2298 if host is None or host.get( 'ipAddresses', [] ) == []:
2299 main.log.error(
2300 "Error with host ipAddresses on controller" +
2301 controllerStr + ": " + str( host ) )
2302 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002303 ports = []
2304 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002305 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002306 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002307 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002308 args=[ main.CLIs[i].ports, [ None ] ],
2309 kwargs= { 'sleep': 5, 'attempts': 5,
2310 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002311 threads.append( t )
2312 t.start()
2313
2314 for t in threads:
2315 t.join()
2316 ports.append( t.result )
2317 links = []
2318 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002319 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002320 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002321 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002322 args=[ main.CLIs[i].links, [ None ] ],
2323 kwargs= { 'sleep': 5, 'attempts': 5,
2324 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002325 threads.append( t )
2326 t.start()
2327
2328 for t in threads:
2329 t.join()
2330 links.append( t.result )
2331 clusters = []
2332 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002333 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002334 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002335 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002336 args=[ main.CLIs[i].clusters, [ None ] ],
2337 kwargs= { 'sleep': 5, 'attempts': 5,
2338 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002339 threads.append( t )
2340 t.start()
2341
2342 for t in threads:
2343 t.join()
2344 clusters.append( t.result )
2345
2346 elapsed = time.time() - startTime
2347 cliTime = time.time() - cliStart
2348 print "Elapsed time: " + str( elapsed )
2349 print "CLI time: " + str( cliTime )
2350
Jon Hall6e709752016-02-01 13:38:46 -08002351 if all( e is None for e in devices ) and\
2352 all( e is None for e in hosts ) and\
2353 all( e is None for e in ports ) and\
2354 all( e is None for e in links ) and\
2355 all( e is None for e in clusters ):
2356 topoFailMsg = "Could not get topology from ONOS"
2357 main.log.error( topoFailMsg )
2358 continue # Try again, No use trying to compare
2359
Jon Hall5cf14d52015-07-16 12:15:19 -07002360 mnSwitches = main.Mininet1.getSwitches()
2361 mnLinks = main.Mininet1.getLinks()
2362 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002363 for controller in range( len( main.activeNodes ) ):
2364 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002365 if devices[ controller ] and ports[ controller ] and\
2366 "Error" not in devices[ controller ] and\
2367 "Error" not in ports[ controller ]:
2368
Jon Hallc6793552016-01-19 14:18:37 -08002369 try:
2370 currentDevicesResult = main.Mininet1.compareSwitches(
2371 mnSwitches,
2372 json.loads( devices[ controller ] ),
2373 json.loads( ports[ controller ] ) )
2374 except ( TypeError, ValueError ) as e:
2375 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2376 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002377 else:
2378 currentDevicesResult = main.FALSE
2379 utilities.assert_equals( expect=main.TRUE,
2380 actual=currentDevicesResult,
2381 onpass="ONOS" + controllerStr +
2382 " Switches view is correct",
2383 onfail="ONOS" + controllerStr +
2384 " Switches view is incorrect" )
2385
2386 if links[ controller ] and "Error" not in links[ controller ]:
2387 currentLinksResult = main.Mininet1.compareLinks(
2388 mnSwitches, mnLinks,
2389 json.loads( links[ controller ] ) )
2390 else:
2391 currentLinksResult = main.FALSE
2392 utilities.assert_equals( expect=main.TRUE,
2393 actual=currentLinksResult,
2394 onpass="ONOS" + controllerStr +
2395 " links view is correct",
2396 onfail="ONOS" + controllerStr +
2397 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002398 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002399 currentHostsResult = main.Mininet1.compareHosts(
2400 mnHosts,
2401 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002402 elif hosts[ controller ] == []:
2403 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002404 else:
2405 currentHostsResult = main.FALSE
2406 utilities.assert_equals( expect=main.TRUE,
2407 actual=currentHostsResult,
2408 onpass="ONOS" + controllerStr +
2409 " hosts exist in Mininet",
2410 onfail="ONOS" + controllerStr +
2411 " hosts don't match Mininet" )
2412 # CHECKING HOST ATTACHMENT POINTS
2413 hostAttachment = True
2414 zeroHosts = False
2415 # FIXME: topo-HA/obelisk specific mappings:
2416 # key is mac and value is dpid
2417 mappings = {}
2418 for i in range( 1, 29 ): # hosts 1 through 28
2419 # set up correct variables:
2420 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2421 if i == 1:
2422 deviceId = "1000".zfill(16)
2423 elif i == 2:
2424 deviceId = "2000".zfill(16)
2425 elif i == 3:
2426 deviceId = "3000".zfill(16)
2427 elif i == 4:
2428 deviceId = "3004".zfill(16)
2429 elif i == 5:
2430 deviceId = "5000".zfill(16)
2431 elif i == 6:
2432 deviceId = "6000".zfill(16)
2433 elif i == 7:
2434 deviceId = "6007".zfill(16)
2435 elif i >= 8 and i <= 17:
2436 dpid = '3' + str( i ).zfill( 3 )
2437 deviceId = dpid.zfill(16)
2438 elif i >= 18 and i <= 27:
2439 dpid = '6' + str( i ).zfill( 3 )
2440 deviceId = dpid.zfill(16)
2441 elif i == 28:
2442 deviceId = "2800".zfill(16)
2443 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002444 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002445 if hosts[ controller ] == []:
2446 main.log.warn( "There are no hosts discovered" )
2447 zeroHosts = True
2448 else:
2449 for host in hosts[ controller ]:
2450 mac = None
2451 location = None
2452 device = None
2453 port = None
2454 try:
2455 mac = host.get( 'mac' )
2456 assert mac, "mac field could not be found for this host object"
2457
2458 location = host.get( 'location' )
2459 assert location, "location field could not be found for this host object"
2460
2461 # Trim the protocol identifier off deviceId
2462 device = str( location.get( 'elementId' ) ).split(':')[1]
2463 assert device, "elementId field could not be found for this host location object"
2464
2465 port = location.get( 'port' )
2466 assert port, "port field could not be found for this host location object"
2467
2468 # Now check if this matches where they should be
2469 if mac and device and port:
2470 if str( port ) != "1":
2471 main.log.error( "The attachment port is incorrect for " +
2472 "host " + str( mac ) +
2473 ". Expected: 1 Actual: " + str( port) )
2474 hostAttachment = False
2475 if device != mappings[ str( mac ) ]:
2476 main.log.error( "The attachment device is incorrect for " +
2477 "host " + str( mac ) +
2478 ". Expected: " + mappings[ str( mac ) ] +
2479 " Actual: " + device )
2480 hostAttachment = False
2481 else:
2482 hostAttachment = False
2483 except AssertionError:
2484 main.log.exception( "Json object not as expected" )
2485 main.log.error( repr( host ) )
2486 hostAttachment = False
2487 else:
2488 main.log.error( "No hosts json output or \"Error\"" +
2489 " in output. hosts = " +
2490 repr( hosts[ controller ] ) )
2491 if zeroHosts is False:
2492 hostAttachment = True
2493
2494 # END CHECKING HOST ATTACHMENT POINTS
2495 devicesResults = devicesResults and currentDevicesResult
2496 linksResults = linksResults and currentLinksResult
2497 hostsResults = hostsResults and currentHostsResult
2498 hostAttachmentResults = hostAttachmentResults and\
2499 hostAttachment
Jon Halla440e872016-03-31 15:15:50 -07002500 topoResult = ( devicesResults and linksResults
2501 and hostsResults and ipResult and
2502 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002503 utilities.assert_equals( expect=True,
2504 actual=topoResult,
2505 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002506 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002507 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002508
2509 # Compare json objects for hosts and dataplane clusters
2510
2511 # hosts
2512 main.step( "Hosts view is consistent across all ONOS nodes" )
2513 consistentHostsResult = main.TRUE
2514 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002515 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002516 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002517 if hosts[ controller ] == hosts[ 0 ]:
2518 continue
2519 else: # hosts not consistent
2520 main.log.error( "hosts from ONOS" + controllerStr +
2521 " is inconsistent with ONOS1" )
2522 main.log.warn( repr( hosts[ controller ] ) )
2523 consistentHostsResult = main.FALSE
2524
2525 else:
2526 main.log.error( "Error in getting ONOS hosts from ONOS" +
2527 controllerStr )
2528 consistentHostsResult = main.FALSE
2529 main.log.warn( "ONOS" + controllerStr +
2530 " hosts response: " +
2531 repr( hosts[ controller ] ) )
2532 utilities.assert_equals(
2533 expect=main.TRUE,
2534 actual=consistentHostsResult,
2535 onpass="Hosts view is consistent across all ONOS nodes",
2536 onfail="ONOS nodes have different views of hosts" )
2537
2538 main.step( "Hosts information is correct" )
2539 hostsResults = hostsResults and ipResult
2540 utilities.assert_equals(
2541 expect=main.TRUE,
2542 actual=hostsResults,
2543 onpass="Host information is correct",
2544 onfail="Host information is incorrect" )
2545
2546 main.step( "Host attachment points to the network" )
2547 utilities.assert_equals(
2548 expect=True,
2549 actual=hostAttachmentResults,
2550 onpass="Hosts are correctly attached to the network",
2551 onfail="ONOS did not correctly attach hosts to the network" )
2552
2553 # Strongly connected clusters of devices
2554 main.step( "Clusters view is consistent across all ONOS nodes" )
2555 consistentClustersResult = main.TRUE
2556 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002557 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002558 if "Error" not in clusters[ controller ]:
2559 if clusters[ controller ] == clusters[ 0 ]:
2560 continue
2561 else: # clusters not consistent
2562 main.log.error( "clusters from ONOS" +
2563 controllerStr +
2564 " is inconsistent with ONOS1" )
2565 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002566 else:
2567 main.log.error( "Error in getting dataplane clusters " +
2568 "from ONOS" + controllerStr )
2569 consistentClustersResult = main.FALSE
2570 main.log.warn( "ONOS" + controllerStr +
2571 " clusters response: " +
2572 repr( clusters[ controller ] ) )
2573 utilities.assert_equals(
2574 expect=main.TRUE,
2575 actual=consistentClustersResult,
2576 onpass="Clusters view is consistent across all ONOS nodes",
2577 onfail="ONOS nodes have different views of clusters" )
2578
2579 main.step( "There is only one SCC" )
2580 # there should always only be one cluster
2581 try:
2582 numClusters = len( json.loads( clusters[ 0 ] ) )
2583 except ( ValueError, TypeError ):
2584 main.log.exception( "Error parsing clusters[0]: " +
2585 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002586 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002587 clusterResults = main.FALSE
2588 if numClusters == 1:
2589 clusterResults = main.TRUE
2590 utilities.assert_equals(
2591 expect=1,
2592 actual=numClusters,
2593 onpass="ONOS shows 1 SCC",
2594 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2595
2596 topoResult = ( devicesResults and linksResults
2597 and hostsResults and consistentHostsResult
2598 and consistentClustersResult and clusterResults
2599 and ipResult and hostAttachmentResults )
2600
2601 topoResult = topoResult and int( count <= 2 )
2602 note = "note it takes about " + str( int( cliTime ) ) + \
2603 " seconds for the test to make all the cli calls to fetch " +\
2604 "the topology from each ONOS instance"
2605 main.log.info(
2606 "Very crass estimate for topology discovery/convergence( " +
2607 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2608 str( count ) + " tries" )
2609
2610 main.step( "Device information is correct" )
2611 utilities.assert_equals(
2612 expect=main.TRUE,
2613 actual=devicesResults,
2614 onpass="Device information is correct",
2615 onfail="Device information is incorrect" )
2616
2617 main.step( "Links are correct" )
2618 utilities.assert_equals(
2619 expect=main.TRUE,
2620 actual=linksResults,
2621 onpass="Link are correct",
2622 onfail="Links are incorrect" )
2623
Jon Halla440e872016-03-31 15:15:50 -07002624 main.step( "Hosts are correct" )
2625 utilities.assert_equals(
2626 expect=main.TRUE,
2627 actual=hostsResults,
2628 onpass="Hosts are correct",
2629 onfail="Hosts are incorrect" )
2630
Jon Hall5cf14d52015-07-16 12:15:19 -07002631 # FIXME: move this to an ONOS state case
2632 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002633 nodeResults = utilities.retry( main.HA.nodesCheck,
2634 False,
2635 args=[main.activeNodes],
2636 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002637
Jon Hall41d39f12016-04-11 22:54:35 -07002638 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002639 onpass="Nodes check successful",
2640 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002641 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002642 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002643 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002644 main.CLIs[i].name,
2645 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002646
2647 def CASE9( self, main ):
2648 """
2649 Link s3-s28 down
2650 """
2651 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002652 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002653 assert main, "main not defined"
2654 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002655 assert main.CLIs, "main.CLIs not defined"
2656 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002657 # NOTE: You should probably run a topology check after this
2658
2659 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2660
2661 description = "Turn off a link to ensure that Link Discovery " +\
2662 "is working properly"
2663 main.case( description )
2664
2665 main.step( "Kill Link between s3 and s28" )
2666 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2667 main.log.info( "Waiting " + str( linkSleep ) +
2668 " seconds for link down to be discovered" )
2669 time.sleep( linkSleep )
2670 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2671 onpass="Link down successful",
2672 onfail="Failed to bring link down" )
2673 # TODO do some sort of check here
2674
2675 def CASE10( self, main ):
2676 """
2677 Link s3-s28 up
2678 """
2679 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002680 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002681 assert main, "main not defined"
2682 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002683 assert main.CLIs, "main.CLIs not defined"
2684 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002685 # NOTE: You should probably run a topology check after this
2686
2687 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2688
2689 description = "Restore a link to ensure that Link Discovery is " + \
2690 "working properly"
2691 main.case( description )
2692
2693 main.step( "Bring link between s3 and s28 back up" )
2694 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2695 main.log.info( "Waiting " + str( linkSleep ) +
2696 " seconds for link up to be discovered" )
2697 time.sleep( linkSleep )
2698 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2699 onpass="Link up successful",
2700 onfail="Failed to bring link up" )
2701 # TODO do some sort of check here
2702
2703 def CASE11( self, main ):
2704 """
2705 Switch Down
2706 """
2707 # NOTE: You should probably run a topology check after this
2708 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002709 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002710 assert main, "main not defined"
2711 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002712 assert main.CLIs, "main.CLIs not defined"
2713 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002714
2715 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2716
2717 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002718 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002719 main.case( description )
2720 switch = main.params[ 'kill' ][ 'switch' ]
2721 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2722
2723 # TODO: Make this switch parameterizable
2724 main.step( "Kill " + switch )
2725 main.log.info( "Deleting " + switch )
2726 main.Mininet1.delSwitch( switch )
2727 main.log.info( "Waiting " + str( switchSleep ) +
2728 " seconds for switch down to be discovered" )
2729 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002730 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002731 # Peek at the deleted switch
2732 main.log.warn( str( device ) )
2733 result = main.FALSE
2734 if device and device[ 'available' ] is False:
2735 result = main.TRUE
2736 utilities.assert_equals( expect=main.TRUE, actual=result,
2737 onpass="Kill switch successful",
2738 onfail="Failed to kill switch?" )
2739
2740 def CASE12( self, main ):
2741 """
2742 Switch Up
2743 """
2744 # NOTE: You should probably run a topology check after this
2745 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002746 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002747 assert main, "main not defined"
2748 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002749 assert main.CLIs, "main.CLIs not defined"
2750 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002751 assert ONOS1Port, "ONOS1Port not defined"
2752 assert ONOS2Port, "ONOS2Port not defined"
2753 assert ONOS3Port, "ONOS3Port not defined"
2754 assert ONOS4Port, "ONOS4Port not defined"
2755 assert ONOS5Port, "ONOS5Port not defined"
2756 assert ONOS6Port, "ONOS6Port not defined"
2757 assert ONOS7Port, "ONOS7Port not defined"
2758
2759 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2760 switch = main.params[ 'kill' ][ 'switch' ]
2761 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2762 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002763 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002764 description = "Adding a switch to ensure it is discovered correctly"
2765 main.case( description )
2766
2767 main.step( "Add back " + switch )
2768 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2769 for peer in links:
2770 main.Mininet1.addLink( switch, peer )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002771 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002772 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2773 main.log.info( "Waiting " + str( switchSleep ) +
2774 " seconds for switch up to be discovered" )
2775 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002776 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002777 # Peek at the deleted switch
2778 main.log.warn( str( device ) )
2779 result = main.FALSE
2780 if device and device[ 'available' ]:
2781 result = main.TRUE
2782 utilities.assert_equals( expect=main.TRUE, actual=result,
2783 onpass="add switch successful",
2784 onfail="Failed to add switch?" )
2785
2786 def CASE13( self, main ):
2787 """
2788 Clean up
2789 """
2790 import os
2791 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002792 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002793 assert main, "main not defined"
2794 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002795 assert main.CLIs, "main.CLIs not defined"
2796 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002797
2798 # printing colors to terminal
2799 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2800 'blue': '\033[94m', 'green': '\033[92m',
2801 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2802 main.case( "Test Cleanup" )
2803 main.step( "Killing tcpdumps" )
2804 main.Mininet2.stopTcpdump()
2805
2806 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002807 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002808 main.step( "Copying MN pcap and ONOS log files to test station" )
2809 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2810 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002811 # NOTE: MN Pcap file is being saved to logdir.
2812 # We scp this file as MN and TestON aren't necessarily the same vm
2813
2814 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002815 # TODO: Load these from params
2816 # NOTE: must end in /
2817 logFolder = "/opt/onos/log/"
2818 logFiles = [ "karaf.log", "karaf.log.1" ]
2819 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002820 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002821 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002822 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002823 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2824 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002825 # std*.log's
2826 # NOTE: must end in /
2827 logFolder = "/opt/onos/var/"
2828 logFiles = [ "stderr.log", "stdout.log" ]
2829 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002830 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002831 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002832 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002833 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2834 logFolder + f, dstName )
2835 else:
2836 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002837
2838 main.step( "Stopping Mininet" )
2839 mnResult = main.Mininet1.stopNet()
2840 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2841 onpass="Mininet stopped",
2842 onfail="MN cleanup NOT successful" )
2843
2844 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002845 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002846 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2847 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002848
2849 try:
2850 timerLog = open( main.logdir + "/Timers.csv", 'w')
2851 # Overwrite with empty line and close
2852 labels = "Gossip Intents, Restart"
2853 data = str( gossipTime ) + ", " + str( main.restartTime )
2854 timerLog.write( labels + "\n" + data )
2855 timerLog.close()
2856 except NameError, e:
2857 main.log.exception(e)
2858
2859 def CASE14( self, main ):
2860 """
2861 start election app on all onos nodes
2862 """
Jon Halle1a3b752015-07-22 13:02:46 -07002863 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002864 assert main, "main not defined"
2865 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002866 assert main.CLIs, "main.CLIs not defined"
2867 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002868
2869 main.case("Start Leadership Election app")
2870 main.step( "Install leadership election app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002871 onosCli = main.CLIs[ main.activeNodes[0] ]
2872 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002873 utilities.assert_equals(
2874 expect=main.TRUE,
2875 actual=appResult,
2876 onpass="Election app installed",
2877 onfail="Something went wrong with installing Leadership election" )
2878
2879 main.step( "Run for election on each node" )
2880 leaderResult = main.TRUE
2881 leaders = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002882 for i in main.activeNodes:
2883 main.CLIs[i].electionTestRun()
2884 for i in main.activeNodes:
2885 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002886 leader = cli.electionTestLeader()
2887 if leader is None or leader == main.FALSE:
2888 main.log.error( cli.name + ": Leader for the election app " +
2889 "should be an ONOS node, instead got '" +
2890 str( leader ) + "'" )
2891 leaderResult = main.FALSE
2892 leaders.append( leader )
2893 utilities.assert_equals(
2894 expect=main.TRUE,
2895 actual=leaderResult,
2896 onpass="Successfully ran for leadership",
2897 onfail="Failed to run for leadership" )
2898
2899 main.step( "Check that each node shows the same leader" )
2900 sameLeader = main.TRUE
2901 if len( set( leaders ) ) != 1:
2902 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002903 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002904 str( leaders ) )
2905 utilities.assert_equals(
2906 expect=main.TRUE,
2907 actual=sameLeader,
2908 onpass="Leadership is consistent for the election topic",
2909 onfail="Nodes have different leaders" )
2910
2911 def CASE15( self, main ):
2912 """
2913 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002914 15.1 Run election on each node
2915 15.2 Check that each node has the same leaders and candidates
2916 15.3 Find current leader and withdraw
2917 15.4 Check that a new node was elected leader
2918 15.5 Check that that new leader was the candidate of old leader
2919 15.6 Run for election on old leader
2920 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2921 15.8 Make sure that the old leader was added to the candidate list
2922
2923 old and new variable prefixes refer to data from before vs after
2924 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002925 """
2926 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002927 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002928 assert main, "main not defined"
2929 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002930 assert main.CLIs, "main.CLIs not defined"
2931 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002932
Jon Hall5cf14d52015-07-16 12:15:19 -07002933 description = "Check that Leadership Election is still functional"
2934 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002935 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002936
Jon Halla440e872016-03-31 15:15:50 -07002937 oldLeaders = [] # list of lists of each nodes' candidates before
2938 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002939 oldLeader = '' # the old leader from oldLeaders, None if not same
2940 newLeader = '' # the new leaders fron newLoeaders, None if not same
2941 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2942 expectNoLeader = False # True when there is only one leader
2943 if main.numCtrls == 1:
2944 expectNoLeader = True
2945
2946 main.step( "Run for election on each node" )
2947 electionResult = main.TRUE
2948
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002949 for i in main.activeNodes: # run test election on each node
2950 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002951 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002952 utilities.assert_equals(
2953 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002954 actual=electionResult,
2955 onpass="All nodes successfully ran for leadership",
2956 onfail="At least one node failed to run for leadership" )
2957
acsmars3a72bde2015-09-02 14:16:22 -07002958 if electionResult == main.FALSE:
2959 main.log.error(
2960 "Skipping Test Case because Election Test App isn't loaded" )
2961 main.skipCase()
2962
acsmars71adceb2015-08-31 15:09:26 -07002963 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002964 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002965 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002966 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002967 if sameResult:
2968 oldLeader = oldLeaders[ 0 ][ 0 ]
2969 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07002970 else:
Jon Halla440e872016-03-31 15:15:50 -07002971 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002972 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002973 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002974 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002975 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002976 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002977
2978 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002979 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002980 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002981 if oldLeader is None:
2982 main.log.error( "Leadership isn't consistent." )
2983 withdrawResult = main.FALSE
2984 # Get the CLI of the oldLeader
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002985 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002986 if oldLeader == main.nodes[ i ].ip_address:
2987 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002988 break
2989 else: # FOR/ELSE statement
2990 main.log.error( "Leader election, could not find current leader" )
2991 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002992 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002993 utilities.assert_equals(
2994 expect=main.TRUE,
2995 actual=withdrawResult,
2996 onpass="Node was withdrawn from election",
2997 onfail="Node was not withdrawn from election" )
2998
acsmars71adceb2015-08-31 15:09:26 -07002999 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07003000 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07003001 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07003002 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07003003 if newLeaders[ 0 ][ 0 ] == 'none':
3004 main.log.error( "No leader was elected on at least 1 node" )
3005 if not expectNoLeader:
3006 newLeaderResult = False
3007 if newLeaderResult:
3008 newLeader = newLeaders[ 0 ][ 0 ]
Jon Hall5cf14d52015-07-16 12:15:19 -07003009 else:
Jon Halla440e872016-03-31 15:15:50 -07003010 newLeader = None
acsmars71adceb2015-08-31 15:09:26 -07003011
3012 # Check that the new leader is not the older leader, which was withdrawn
3013 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07003014 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08003015 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07003016 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003017 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003018 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003019 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003020 onpass="Leadership election passed",
3021 onfail="Something went wrong with Leadership election" )
3022
Jon Halla440e872016-03-31 15:15:50 -07003023 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08003024 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07003025 correctCandidateResult = main.TRUE
3026 if expectNoLeader:
3027 if newLeader == 'none':
3028 main.log.info( "No leader expected. None found. Pass" )
3029 correctCandidateResult = main.TRUE
3030 else:
3031 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3032 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003033 elif len( oldLeaders[0] ) >= 3:
3034 if newLeader == oldLeaders[ 0 ][ 2 ]:
3035 # correct leader was elected
3036 correctCandidateResult = main.TRUE
3037 else:
3038 correctCandidateResult = main.FALSE
3039 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3040 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003041 else:
3042 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003043 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003044 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07003045 utilities.assert_equals(
3046 expect=main.TRUE,
3047 actual=correctCandidateResult,
3048 onpass="Correct Candidate Elected",
3049 onfail="Incorrect Candidate Elected" )
3050
Jon Hall5cf14d52015-07-16 12:15:19 -07003051 main.step( "Run for election on old leader( just so everyone " +
3052 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07003053 if oldLeaderCLI is not None:
3054 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003055 else:
acsmars71adceb2015-08-31 15:09:26 -07003056 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003057 runResult = main.FALSE
3058 utilities.assert_equals(
3059 expect=main.TRUE,
3060 actual=runResult,
3061 onpass="App re-ran for election",
3062 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003063
acsmars71adceb2015-08-31 15:09:26 -07003064 main.step(
3065 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003066 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003067 # Get new leaders and candidates
3068 reRunLeaders = []
3069 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003070 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003071
3072 # Check that the re-elected node is last on the candidate List
Jon Halla440e872016-03-31 15:15:50 -07003073 if oldLeader != reRunLeaders[ 0 ][ -1 ]:
3074 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3075 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07003076 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003077
3078 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003079 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003080 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003081 onpass="Old leader successfully re-ran for election",
3082 onfail="Something went wrong with Leadership election after " +
3083 "the old leader re-ran for election" )
3084
3085 def CASE16( self, main ):
3086 """
3087 Install Distributed Primitives app
3088 """
3089 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003090 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003091 assert main, "main not defined"
3092 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003093 assert main.CLIs, "main.CLIs not defined"
3094 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003095
3096 # Variables for the distributed primitives tests
3097 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003098 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003099 global onosSet
3100 global onosSetName
3101 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003102 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003103 onosSet = set([])
3104 onosSetName = "TestON-set"
3105
3106 description = "Install Primitives app"
3107 main.case( description )
3108 main.step( "Install Primitives app" )
3109 appName = "org.onosproject.distributedprimitives"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003110 node = main.activeNodes[0]
3111 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003112 utilities.assert_equals( expect=main.TRUE,
3113 actual=appResults,
3114 onpass="Primitives app activated",
3115 onfail="Primitives app not activated" )
3116 time.sleep( 5 ) # To allow all nodes to activate
3117
3118 def CASE17( self, main ):
3119 """
3120 Check for basic functionality with distributed primitives
3121 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003122 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003123 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003124 assert main, "main not defined"
3125 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003126 assert main.CLIs, "main.CLIs not defined"
3127 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003128 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003129 assert onosSetName, "onosSetName not defined"
3130 # NOTE: assert fails if value is 0/None/Empty/False
3131 try:
3132 pCounterValue
3133 except NameError:
3134 main.log.error( "pCounterValue not defined, setting to 0" )
3135 pCounterValue = 0
3136 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003137 onosSet
3138 except NameError:
3139 main.log.error( "onosSet not defined, setting to empty Set" )
3140 onosSet = set([])
3141 # Variables for the distributed primitives tests. These are local only
3142 addValue = "a"
3143 addAllValue = "a b c d e f"
3144 retainValue = "c d e f"
3145
3146 description = "Check for basic functionality with distributed " +\
3147 "primitives"
3148 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003149 main.caseExplanation = "Test the methods of the distributed " +\
3150 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003151 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003152 # Partitioned counters
3153 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003154 pCounters = []
3155 threads = []
3156 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003157 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003158 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3159 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003160 args=[ pCounterName ] )
3161 pCounterValue += 1
3162 addedPValues.append( pCounterValue )
3163 threads.append( t )
3164 t.start()
3165
3166 for t in threads:
3167 t.join()
3168 pCounters.append( t.result )
3169 # Check that counter incremented numController times
3170 pCounterResults = True
3171 for i in addedPValues:
3172 tmpResult = i in pCounters
3173 pCounterResults = pCounterResults and tmpResult
3174 if not tmpResult:
3175 main.log.error( str( i ) + " is not in partitioned "
3176 "counter incremented results" )
3177 utilities.assert_equals( expect=True,
3178 actual=pCounterResults,
3179 onpass="Default counter incremented",
3180 onfail="Error incrementing default" +
3181 " counter" )
3182
Jon Halle1a3b752015-07-22 13:02:46 -07003183 main.step( "Get then Increment a default counter on each node" )
3184 pCounters = []
3185 threads = []
3186 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003187 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003188 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3189 name="counterGetAndAdd-" + str( i ),
3190 args=[ pCounterName ] )
3191 addedPValues.append( pCounterValue )
3192 pCounterValue += 1
3193 threads.append( t )
3194 t.start()
3195
3196 for t in threads:
3197 t.join()
3198 pCounters.append( t.result )
3199 # Check that counter incremented numController times
3200 pCounterResults = True
3201 for i in addedPValues:
3202 tmpResult = i in pCounters
3203 pCounterResults = pCounterResults and tmpResult
3204 if not tmpResult:
3205 main.log.error( str( i ) + " is not in partitioned "
3206 "counter incremented results" )
3207 utilities.assert_equals( expect=True,
3208 actual=pCounterResults,
3209 onpass="Default counter incremented",
3210 onfail="Error incrementing default" +
3211 " counter" )
3212
3213 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003214 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003215 utilities.assert_equals( expect=main.TRUE,
3216 actual=incrementCheck,
3217 onpass="Added counters are correct",
3218 onfail="Added counters are incorrect" )
3219
3220 main.step( "Add -8 to then get a default counter on each node" )
3221 pCounters = []
3222 threads = []
3223 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003224 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003225 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3226 name="counterIncrement-" + str( i ),
3227 args=[ pCounterName ],
3228 kwargs={ "delta": -8 } )
3229 pCounterValue += -8
3230 addedPValues.append( pCounterValue )
3231 threads.append( t )
3232 t.start()
3233
3234 for t in threads:
3235 t.join()
3236 pCounters.append( t.result )
3237 # Check that counter incremented numController times
3238 pCounterResults = True
3239 for i in addedPValues:
3240 tmpResult = i in pCounters
3241 pCounterResults = pCounterResults and tmpResult
3242 if not tmpResult:
3243 main.log.error( str( i ) + " is not in partitioned "
3244 "counter incremented results" )
3245 utilities.assert_equals( expect=True,
3246 actual=pCounterResults,
3247 onpass="Default counter incremented",
3248 onfail="Error incrementing default" +
3249 " counter" )
3250
3251 main.step( "Add 5 to then get a default counter on each node" )
3252 pCounters = []
3253 threads = []
3254 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003255 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003256 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3257 name="counterIncrement-" + str( i ),
3258 args=[ pCounterName ],
3259 kwargs={ "delta": 5 } )
3260 pCounterValue += 5
3261 addedPValues.append( pCounterValue )
3262 threads.append( t )
3263 t.start()
3264
3265 for t in threads:
3266 t.join()
3267 pCounters.append( t.result )
3268 # Check that counter incremented numController times
3269 pCounterResults = True
3270 for i in addedPValues:
3271 tmpResult = i in pCounters
3272 pCounterResults = pCounterResults and tmpResult
3273 if not tmpResult:
3274 main.log.error( str( i ) + " is not in partitioned "
3275 "counter incremented results" )
3276 utilities.assert_equals( expect=True,
3277 actual=pCounterResults,
3278 onpass="Default counter incremented",
3279 onfail="Error incrementing default" +
3280 " counter" )
3281
3282 main.step( "Get then add 5 to a default counter on each node" )
3283 pCounters = []
3284 threads = []
3285 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003286 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003287 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3288 name="counterIncrement-" + str( i ),
3289 args=[ pCounterName ],
3290 kwargs={ "delta": 5 } )
3291 addedPValues.append( pCounterValue )
3292 pCounterValue += 5
3293 threads.append( t )
3294 t.start()
3295
3296 for t in threads:
3297 t.join()
3298 pCounters.append( t.result )
3299 # Check that counter incremented numController times
3300 pCounterResults = True
3301 for i in addedPValues:
3302 tmpResult = i in pCounters
3303 pCounterResults = pCounterResults and tmpResult
3304 if not tmpResult:
3305 main.log.error( str( i ) + " is not in partitioned "
3306 "counter incremented results" )
3307 utilities.assert_equals( expect=True,
3308 actual=pCounterResults,
3309 onpass="Default counter incremented",
3310 onfail="Error incrementing default" +
3311 " counter" )
3312
3313 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003314 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003315 utilities.assert_equals( expect=main.TRUE,
3316 actual=incrementCheck,
3317 onpass="Added counters are correct",
3318 onfail="Added counters are incorrect" )
3319
Jon Hall5cf14d52015-07-16 12:15:19 -07003320 # DISTRIBUTED SETS
3321 main.step( "Distributed Set get" )
3322 size = len( onosSet )
3323 getResponses = []
3324 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003325 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003326 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003327 name="setTestGet-" + str( i ),
3328 args=[ onosSetName ] )
3329 threads.append( t )
3330 t.start()
3331 for t in threads:
3332 t.join()
3333 getResponses.append( t.result )
3334
3335 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003336 for i in range( len( main.activeNodes ) ):
3337 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003338 if isinstance( getResponses[ i ], list):
3339 current = set( getResponses[ i ] )
3340 if len( current ) == len( getResponses[ i ] ):
3341 # no repeats
3342 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003343 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003344 " has incorrect view" +
3345 " of set " + onosSetName + ":\n" +
3346 str( getResponses[ i ] ) )
3347 main.log.debug( "Expected: " + str( onosSet ) )
3348 main.log.debug( "Actual: " + str( current ) )
3349 getResults = main.FALSE
3350 else:
3351 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003352 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003353 " has repeat elements in" +
3354 " set " + onosSetName + ":\n" +
3355 str( getResponses[ i ] ) )
3356 getResults = main.FALSE
3357 elif getResponses[ i ] == main.ERROR:
3358 getResults = main.FALSE
3359 utilities.assert_equals( expect=main.TRUE,
3360 actual=getResults,
3361 onpass="Set elements are correct",
3362 onfail="Set elements are incorrect" )
3363
3364 main.step( "Distributed Set size" )
3365 sizeResponses = []
3366 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003367 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003368 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003369 name="setTestSize-" + str( i ),
3370 args=[ onosSetName ] )
3371 threads.append( t )
3372 t.start()
3373 for t in threads:
3374 t.join()
3375 sizeResponses.append( t.result )
3376
3377 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003378 for i in range( len( main.activeNodes ) ):
3379 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003380 if size != sizeResponses[ i ]:
3381 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003382 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003383 " expected a size of " + str( size ) +
3384 " for set " + onosSetName +
3385 " but got " + str( sizeResponses[ i ] ) )
3386 utilities.assert_equals( expect=main.TRUE,
3387 actual=sizeResults,
3388 onpass="Set sizes are correct",
3389 onfail="Set sizes are incorrect" )
3390
3391 main.step( "Distributed Set add()" )
3392 onosSet.add( addValue )
3393 addResponses = []
3394 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003395 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003396 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003397 name="setTestAdd-" + str( i ),
3398 args=[ onosSetName, addValue ] )
3399 threads.append( t )
3400 t.start()
3401 for t in threads:
3402 t.join()
3403 addResponses.append( t.result )
3404
3405 # main.TRUE = successfully changed the set
3406 # main.FALSE = action resulted in no change in set
3407 # main.ERROR - Some error in executing the function
3408 addResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003409 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003410 if addResponses[ i ] == main.TRUE:
3411 # All is well
3412 pass
3413 elif addResponses[ i ] == main.FALSE:
3414 # Already in set, probably fine
3415 pass
3416 elif addResponses[ i ] == main.ERROR:
3417 # Error in execution
3418 addResults = main.FALSE
3419 else:
3420 # unexpected result
3421 addResults = main.FALSE
3422 if addResults != main.TRUE:
3423 main.log.error( "Error executing set add" )
3424
3425 # Check if set is still correct
3426 size = len( onosSet )
3427 getResponses = []
3428 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003429 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003430 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003431 name="setTestGet-" + str( i ),
3432 args=[ onosSetName ] )
3433 threads.append( t )
3434 t.start()
3435 for t in threads:
3436 t.join()
3437 getResponses.append( t.result )
3438 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003439 for i in range( len( main.activeNodes ) ):
3440 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003441 if isinstance( getResponses[ i ], list):
3442 current = set( getResponses[ i ] )
3443 if len( current ) == len( getResponses[ i ] ):
3444 # no repeats
3445 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003446 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003447 " of set " + onosSetName + ":\n" +
3448 str( getResponses[ i ] ) )
3449 main.log.debug( "Expected: " + str( onosSet ) )
3450 main.log.debug( "Actual: " + str( current ) )
3451 getResults = main.FALSE
3452 else:
3453 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003454 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003455 " set " + onosSetName + ":\n" +
3456 str( getResponses[ i ] ) )
3457 getResults = main.FALSE
3458 elif getResponses[ i ] == main.ERROR:
3459 getResults = main.FALSE
3460 sizeResponses = []
3461 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003462 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003463 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003464 name="setTestSize-" + str( i ),
3465 args=[ onosSetName ] )
3466 threads.append( t )
3467 t.start()
3468 for t in threads:
3469 t.join()
3470 sizeResponses.append( t.result )
3471 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003472 for i in range( len( main.activeNodes ) ):
3473 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003474 if size != sizeResponses[ i ]:
3475 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003476 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003477 " expected a size of " + str( size ) +
3478 " for set " + onosSetName +
3479 " but got " + str( sizeResponses[ i ] ) )
3480 addResults = addResults and getResults and sizeResults
3481 utilities.assert_equals( expect=main.TRUE,
3482 actual=addResults,
3483 onpass="Set add correct",
3484 onfail="Set add was incorrect" )
3485
3486 main.step( "Distributed Set addAll()" )
3487 onosSet.update( addAllValue.split() )
3488 addResponses = []
3489 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003490 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003491 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003492 name="setTestAddAll-" + str( i ),
3493 args=[ onosSetName, addAllValue ] )
3494 threads.append( t )
3495 t.start()
3496 for t in threads:
3497 t.join()
3498 addResponses.append( t.result )
3499
3500 # main.TRUE = successfully changed the set
3501 # main.FALSE = action resulted in no change in set
3502 # main.ERROR - Some error in executing the function
3503 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003504 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003505 if addResponses[ i ] == main.TRUE:
3506 # All is well
3507 pass
3508 elif addResponses[ i ] == main.FALSE:
3509 # Already in set, probably fine
3510 pass
3511 elif addResponses[ i ] == main.ERROR:
3512 # Error in execution
3513 addAllResults = main.FALSE
3514 else:
3515 # unexpected result
3516 addAllResults = main.FALSE
3517 if addAllResults != main.TRUE:
3518 main.log.error( "Error executing set addAll" )
3519
3520 # Check if set is still correct
3521 size = len( onosSet )
3522 getResponses = []
3523 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003524 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003525 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003526 name="setTestGet-" + str( i ),
3527 args=[ onosSetName ] )
3528 threads.append( t )
3529 t.start()
3530 for t in threads:
3531 t.join()
3532 getResponses.append( t.result )
3533 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003534 for i in range( len( main.activeNodes ) ):
3535 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003536 if isinstance( getResponses[ i ], list):
3537 current = set( getResponses[ i ] )
3538 if len( current ) == len( getResponses[ i ] ):
3539 # no repeats
3540 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003541 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003542 " has incorrect view" +
3543 " of set " + onosSetName + ":\n" +
3544 str( getResponses[ i ] ) )
3545 main.log.debug( "Expected: " + str( onosSet ) )
3546 main.log.debug( "Actual: " + str( current ) )
3547 getResults = main.FALSE
3548 else:
3549 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003550 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003551 " has repeat elements in" +
3552 " set " + onosSetName + ":\n" +
3553 str( getResponses[ i ] ) )
3554 getResults = main.FALSE
3555 elif getResponses[ i ] == main.ERROR:
3556 getResults = main.FALSE
3557 sizeResponses = []
3558 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003559 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003560 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003561 name="setTestSize-" + str( i ),
3562 args=[ onosSetName ] )
3563 threads.append( t )
3564 t.start()
3565 for t in threads:
3566 t.join()
3567 sizeResponses.append( t.result )
3568 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003569 for i in range( len( main.activeNodes ) ):
3570 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003571 if size != sizeResponses[ i ]:
3572 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003573 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003574 " expected a size of " + str( size ) +
3575 " for set " + onosSetName +
3576 " but got " + str( sizeResponses[ i ] ) )
3577 addAllResults = addAllResults and getResults and sizeResults
3578 utilities.assert_equals( expect=main.TRUE,
3579 actual=addAllResults,
3580 onpass="Set addAll correct",
3581 onfail="Set addAll was incorrect" )
3582
3583 main.step( "Distributed Set contains()" )
3584 containsResponses = []
3585 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003586 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003587 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003588 name="setContains-" + str( i ),
3589 args=[ onosSetName ],
3590 kwargs={ "values": addValue } )
3591 threads.append( t )
3592 t.start()
3593 for t in threads:
3594 t.join()
3595 # NOTE: This is the tuple
3596 containsResponses.append( t.result )
3597
3598 containsResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003599 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003600 if containsResponses[ i ] == main.ERROR:
3601 containsResults = main.FALSE
3602 else:
3603 containsResults = containsResults and\
3604 containsResponses[ i ][ 1 ]
3605 utilities.assert_equals( expect=main.TRUE,
3606 actual=containsResults,
3607 onpass="Set contains is functional",
3608 onfail="Set contains failed" )
3609
3610 main.step( "Distributed Set containsAll()" )
3611 containsAllResponses = []
3612 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003613 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003614 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003615 name="setContainsAll-" + str( i ),
3616 args=[ onosSetName ],
3617 kwargs={ "values": addAllValue } )
3618 threads.append( t )
3619 t.start()
3620 for t in threads:
3621 t.join()
3622 # NOTE: This is the tuple
3623 containsAllResponses.append( t.result )
3624
3625 containsAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003626 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003627 if containsResponses[ i ] == main.ERROR:
3628 containsResults = main.FALSE
3629 else:
3630 containsResults = containsResults and\
3631 containsResponses[ i ][ 1 ]
3632 utilities.assert_equals( expect=main.TRUE,
3633 actual=containsAllResults,
3634 onpass="Set containsAll is functional",
3635 onfail="Set containsAll failed" )
3636
3637 main.step( "Distributed Set remove()" )
3638 onosSet.remove( addValue )
3639 removeResponses = []
3640 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003641 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003642 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003643 name="setTestRemove-" + str( i ),
3644 args=[ onosSetName, addValue ] )
3645 threads.append( t )
3646 t.start()
3647 for t in threads:
3648 t.join()
3649 removeResponses.append( t.result )
3650
3651 # main.TRUE = successfully changed the set
3652 # main.FALSE = action resulted in no change in set
3653 # main.ERROR - Some error in executing the function
3654 removeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003655 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003656 if removeResponses[ i ] == main.TRUE:
3657 # All is well
3658 pass
3659 elif removeResponses[ i ] == main.FALSE:
3660 # not in set, probably fine
3661 pass
3662 elif removeResponses[ i ] == main.ERROR:
3663 # Error in execution
3664 removeResults = main.FALSE
3665 else:
3666 # unexpected result
3667 removeResults = main.FALSE
3668 if removeResults != main.TRUE:
3669 main.log.error( "Error executing set remove" )
3670
3671 # Check if set is still correct
3672 size = len( onosSet )
3673 getResponses = []
3674 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003675 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003676 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003677 name="setTestGet-" + str( i ),
3678 args=[ onosSetName ] )
3679 threads.append( t )
3680 t.start()
3681 for t in threads:
3682 t.join()
3683 getResponses.append( t.result )
3684 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003685 for i in range( len( main.activeNodes ) ):
3686 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003687 if isinstance( getResponses[ i ], list):
3688 current = set( getResponses[ i ] )
3689 if len( current ) == len( getResponses[ i ] ):
3690 # no repeats
3691 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003692 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003693 " has incorrect view" +
3694 " of set " + onosSetName + ":\n" +
3695 str( getResponses[ i ] ) )
3696 main.log.debug( "Expected: " + str( onosSet ) )
3697 main.log.debug( "Actual: " + str( current ) )
3698 getResults = main.FALSE
3699 else:
3700 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003701 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003702 " has repeat elements in" +
3703 " set " + onosSetName + ":\n" +
3704 str( getResponses[ i ] ) )
3705 getResults = main.FALSE
3706 elif getResponses[ i ] == main.ERROR:
3707 getResults = main.FALSE
3708 sizeResponses = []
3709 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003710 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003711 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003712 name="setTestSize-" + str( i ),
3713 args=[ onosSetName ] )
3714 threads.append( t )
3715 t.start()
3716 for t in threads:
3717 t.join()
3718 sizeResponses.append( t.result )
3719 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003720 for i in range( len( main.activeNodes ) ):
3721 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003722 if size != sizeResponses[ i ]:
3723 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003724 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003725 " expected a size of " + str( size ) +
3726 " for set " + onosSetName +
3727 " but got " + str( sizeResponses[ i ] ) )
3728 removeResults = removeResults and getResults and sizeResults
3729 utilities.assert_equals( expect=main.TRUE,
3730 actual=removeResults,
3731 onpass="Set remove correct",
3732 onfail="Set remove was incorrect" )
3733
3734 main.step( "Distributed Set removeAll()" )
3735 onosSet.difference_update( addAllValue.split() )
3736 removeAllResponses = []
3737 threads = []
3738 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003739 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003740 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003741 name="setTestRemoveAll-" + str( i ),
3742 args=[ onosSetName, addAllValue ] )
3743 threads.append( t )
3744 t.start()
3745 for t in threads:
3746 t.join()
3747 removeAllResponses.append( t.result )
3748 except Exception, e:
3749 main.log.exception(e)
3750
3751 # main.TRUE = successfully changed the set
3752 # main.FALSE = action resulted in no change in set
3753 # main.ERROR - Some error in executing the function
3754 removeAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003755 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003756 if removeAllResponses[ i ] == main.TRUE:
3757 # All is well
3758 pass
3759 elif removeAllResponses[ i ] == main.FALSE:
3760 # not in set, probably fine
3761 pass
3762 elif removeAllResponses[ i ] == main.ERROR:
3763 # Error in execution
3764 removeAllResults = main.FALSE
3765 else:
3766 # unexpected result
3767 removeAllResults = main.FALSE
3768 if removeAllResults != main.TRUE:
3769 main.log.error( "Error executing set removeAll" )
3770
3771 # Check if set is still correct
3772 size = len( onosSet )
3773 getResponses = []
3774 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003775 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003776 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003777 name="setTestGet-" + str( i ),
3778 args=[ onosSetName ] )
3779 threads.append( t )
3780 t.start()
3781 for t in threads:
3782 t.join()
3783 getResponses.append( t.result )
3784 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003785 for i in range( len( main.activeNodes ) ):
3786 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003787 if isinstance( getResponses[ i ], list):
3788 current = set( getResponses[ i ] )
3789 if len( current ) == len( getResponses[ i ] ):
3790 # no repeats
3791 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003792 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003793 " has incorrect view" +
3794 " of set " + onosSetName + ":\n" +
3795 str( getResponses[ i ] ) )
3796 main.log.debug( "Expected: " + str( onosSet ) )
3797 main.log.debug( "Actual: " + str( current ) )
3798 getResults = main.FALSE
3799 else:
3800 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003801 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003802 " has repeat elements in" +
3803 " set " + onosSetName + ":\n" +
3804 str( getResponses[ i ] ) )
3805 getResults = main.FALSE
3806 elif getResponses[ i ] == main.ERROR:
3807 getResults = main.FALSE
3808 sizeResponses = []
3809 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003810 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003811 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003812 name="setTestSize-" + str( i ),
3813 args=[ onosSetName ] )
3814 threads.append( t )
3815 t.start()
3816 for t in threads:
3817 t.join()
3818 sizeResponses.append( t.result )
3819 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003820 for i in range( len( main.activeNodes ) ):
3821 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003822 if size != sizeResponses[ i ]:
3823 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003824 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003825 " expected a size of " + str( size ) +
3826 " for set " + onosSetName +
3827 " but got " + str( sizeResponses[ i ] ) )
3828 removeAllResults = removeAllResults and getResults and sizeResults
3829 utilities.assert_equals( expect=main.TRUE,
3830 actual=removeAllResults,
3831 onpass="Set removeAll correct",
3832 onfail="Set removeAll was incorrect" )
3833
3834 main.step( "Distributed Set addAll()" )
3835 onosSet.update( addAllValue.split() )
3836 addResponses = []
3837 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003838 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003839 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003840 name="setTestAddAll-" + str( i ),
3841 args=[ onosSetName, addAllValue ] )
3842 threads.append( t )
3843 t.start()
3844 for t in threads:
3845 t.join()
3846 addResponses.append( t.result )
3847
3848 # main.TRUE = successfully changed the set
3849 # main.FALSE = action resulted in no change in set
3850 # main.ERROR - Some error in executing the function
3851 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003852 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003853 if addResponses[ i ] == main.TRUE:
3854 # All is well
3855 pass
3856 elif addResponses[ i ] == main.FALSE:
3857 # Already in set, probably fine
3858 pass
3859 elif addResponses[ i ] == main.ERROR:
3860 # Error in execution
3861 addAllResults = main.FALSE
3862 else:
3863 # unexpected result
3864 addAllResults = main.FALSE
3865 if addAllResults != main.TRUE:
3866 main.log.error( "Error executing set addAll" )
3867
3868 # Check if set is still correct
3869 size = len( onosSet )
3870 getResponses = []
3871 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003872 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003873 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003874 name="setTestGet-" + str( i ),
3875 args=[ onosSetName ] )
3876 threads.append( t )
3877 t.start()
3878 for t in threads:
3879 t.join()
3880 getResponses.append( t.result )
3881 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003882 for i in range( len( main.activeNodes ) ):
3883 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003884 if isinstance( getResponses[ i ], list):
3885 current = set( getResponses[ i ] )
3886 if len( current ) == len( getResponses[ i ] ):
3887 # no repeats
3888 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003889 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003890 " has incorrect view" +
3891 " of set " + onosSetName + ":\n" +
3892 str( getResponses[ i ] ) )
3893 main.log.debug( "Expected: " + str( onosSet ) )
3894 main.log.debug( "Actual: " + str( current ) )
3895 getResults = main.FALSE
3896 else:
3897 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003898 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003899 " has repeat elements in" +
3900 " set " + onosSetName + ":\n" +
3901 str( getResponses[ i ] ) )
3902 getResults = main.FALSE
3903 elif getResponses[ i ] == main.ERROR:
3904 getResults = main.FALSE
3905 sizeResponses = []
3906 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003907 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003908 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003909 name="setTestSize-" + str( i ),
3910 args=[ onosSetName ] )
3911 threads.append( t )
3912 t.start()
3913 for t in threads:
3914 t.join()
3915 sizeResponses.append( t.result )
3916 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003917 for i in range( len( main.activeNodes ) ):
3918 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003919 if size != sizeResponses[ i ]:
3920 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003921 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003922 " expected a size of " + str( size ) +
3923 " for set " + onosSetName +
3924 " but got " + str( sizeResponses[ i ] ) )
3925 addAllResults = addAllResults and getResults and sizeResults
3926 utilities.assert_equals( expect=main.TRUE,
3927 actual=addAllResults,
3928 onpass="Set addAll correct",
3929 onfail="Set addAll was incorrect" )
3930
3931 main.step( "Distributed Set clear()" )
3932 onosSet.clear()
3933 clearResponses = []
3934 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003935 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003936 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003937 name="setTestClear-" + str( i ),
3938 args=[ onosSetName, " "], # Values doesn't matter
3939 kwargs={ "clear": True } )
3940 threads.append( t )
3941 t.start()
3942 for t in threads:
3943 t.join()
3944 clearResponses.append( t.result )
3945
3946 # main.TRUE = successfully changed the set
3947 # main.FALSE = action resulted in no change in set
3948 # main.ERROR - Some error in executing the function
3949 clearResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003950 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003951 if clearResponses[ i ] == main.TRUE:
3952 # All is well
3953 pass
3954 elif clearResponses[ i ] == main.FALSE:
3955 # Nothing set, probably fine
3956 pass
3957 elif clearResponses[ i ] == main.ERROR:
3958 # Error in execution
3959 clearResults = main.FALSE
3960 else:
3961 # unexpected result
3962 clearResults = main.FALSE
3963 if clearResults != main.TRUE:
3964 main.log.error( "Error executing set clear" )
3965
3966 # Check if set is still correct
3967 size = len( onosSet )
3968 getResponses = []
3969 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003970 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003971 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003972 name="setTestGet-" + str( i ),
3973 args=[ onosSetName ] )
3974 threads.append( t )
3975 t.start()
3976 for t in threads:
3977 t.join()
3978 getResponses.append( t.result )
3979 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003980 for i in range( len( main.activeNodes ) ):
3981 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003982 if isinstance( getResponses[ i ], list):
3983 current = set( getResponses[ i ] )
3984 if len( current ) == len( getResponses[ i ] ):
3985 # no repeats
3986 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003987 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003988 " has incorrect view" +
3989 " of set " + onosSetName + ":\n" +
3990 str( getResponses[ i ] ) )
3991 main.log.debug( "Expected: " + str( onosSet ) )
3992 main.log.debug( "Actual: " + str( current ) )
3993 getResults = main.FALSE
3994 else:
3995 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003996 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003997 " has repeat elements in" +
3998 " set " + onosSetName + ":\n" +
3999 str( getResponses[ i ] ) )
4000 getResults = main.FALSE
4001 elif getResponses[ i ] == main.ERROR:
4002 getResults = main.FALSE
4003 sizeResponses = []
4004 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004005 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004006 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004007 name="setTestSize-" + str( i ),
4008 args=[ onosSetName ] )
4009 threads.append( t )
4010 t.start()
4011 for t in threads:
4012 t.join()
4013 sizeResponses.append( t.result )
4014 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004015 for i in range( len( main.activeNodes ) ):
4016 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004017 if size != sizeResponses[ i ]:
4018 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004019 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004020 " expected a size of " + str( size ) +
4021 " for set " + onosSetName +
4022 " but got " + str( sizeResponses[ i ] ) )
4023 clearResults = clearResults and getResults and sizeResults
4024 utilities.assert_equals( expect=main.TRUE,
4025 actual=clearResults,
4026 onpass="Set clear correct",
4027 onfail="Set clear was incorrect" )
4028
4029 main.step( "Distributed Set addAll()" )
4030 onosSet.update( addAllValue.split() )
4031 addResponses = []
4032 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004033 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004034 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004035 name="setTestAddAll-" + str( i ),
4036 args=[ onosSetName, addAllValue ] )
4037 threads.append( t )
4038 t.start()
4039 for t in threads:
4040 t.join()
4041 addResponses.append( t.result )
4042
4043 # main.TRUE = successfully changed the set
4044 # main.FALSE = action resulted in no change in set
4045 # main.ERROR - Some error in executing the function
4046 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004047 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004048 if addResponses[ i ] == main.TRUE:
4049 # All is well
4050 pass
4051 elif addResponses[ i ] == main.FALSE:
4052 # Already in set, probably fine
4053 pass
4054 elif addResponses[ i ] == main.ERROR:
4055 # Error in execution
4056 addAllResults = main.FALSE
4057 else:
4058 # unexpected result
4059 addAllResults = main.FALSE
4060 if addAllResults != main.TRUE:
4061 main.log.error( "Error executing set addAll" )
4062
4063 # Check if set is still correct
4064 size = len( onosSet )
4065 getResponses = []
4066 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004067 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004068 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004069 name="setTestGet-" + str( i ),
4070 args=[ onosSetName ] )
4071 threads.append( t )
4072 t.start()
4073 for t in threads:
4074 t.join()
4075 getResponses.append( t.result )
4076 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004077 for i in range( len( main.activeNodes ) ):
4078 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004079 if isinstance( getResponses[ i ], list):
4080 current = set( getResponses[ i ] )
4081 if len( current ) == len( getResponses[ i ] ):
4082 # no repeats
4083 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004084 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004085 " has incorrect view" +
4086 " of set " + onosSetName + ":\n" +
4087 str( getResponses[ i ] ) )
4088 main.log.debug( "Expected: " + str( onosSet ) )
4089 main.log.debug( "Actual: " + str( current ) )
4090 getResults = main.FALSE
4091 else:
4092 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004093 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004094 " has repeat elements in" +
4095 " set " + onosSetName + ":\n" +
4096 str( getResponses[ i ] ) )
4097 getResults = main.FALSE
4098 elif getResponses[ i ] == main.ERROR:
4099 getResults = main.FALSE
4100 sizeResponses = []
4101 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004102 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004103 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004104 name="setTestSize-" + str( i ),
4105 args=[ onosSetName ] )
4106 threads.append( t )
4107 t.start()
4108 for t in threads:
4109 t.join()
4110 sizeResponses.append( t.result )
4111 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004112 for i in range( len( main.activeNodes ) ):
4113 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004114 if size != sizeResponses[ i ]:
4115 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004116 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004117 " expected a size of " + str( size ) +
4118 " for set " + onosSetName +
4119 " but got " + str( sizeResponses[ i ] ) )
4120 addAllResults = addAllResults and getResults and sizeResults
4121 utilities.assert_equals( expect=main.TRUE,
4122 actual=addAllResults,
4123 onpass="Set addAll correct",
4124 onfail="Set addAll was incorrect" )
4125
4126 main.step( "Distributed Set retain()" )
4127 onosSet.intersection_update( retainValue.split() )
4128 retainResponses = []
4129 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004130 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004131 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004132 name="setTestRetain-" + str( i ),
4133 args=[ onosSetName, retainValue ],
4134 kwargs={ "retain": True } )
4135 threads.append( t )
4136 t.start()
4137 for t in threads:
4138 t.join()
4139 retainResponses.append( t.result )
4140
4141 # main.TRUE = successfully changed the set
4142 # main.FALSE = action resulted in no change in set
4143 # main.ERROR - Some error in executing the function
4144 retainResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004145 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004146 if retainResponses[ i ] == main.TRUE:
4147 # All is well
4148 pass
4149 elif retainResponses[ i ] == main.FALSE:
4150 # Already in set, probably fine
4151 pass
4152 elif retainResponses[ i ] == main.ERROR:
4153 # Error in execution
4154 retainResults = main.FALSE
4155 else:
4156 # unexpected result
4157 retainResults = main.FALSE
4158 if retainResults != main.TRUE:
4159 main.log.error( "Error executing set retain" )
4160
4161 # Check if set is still correct
4162 size = len( onosSet )
4163 getResponses = []
4164 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004165 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004166 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004167 name="setTestGet-" + str( i ),
4168 args=[ onosSetName ] )
4169 threads.append( t )
4170 t.start()
4171 for t in threads:
4172 t.join()
4173 getResponses.append( t.result )
4174 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004175 for i in range( len( main.activeNodes ) ):
4176 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004177 if isinstance( getResponses[ i ], list):
4178 current = set( getResponses[ i ] )
4179 if len( current ) == len( getResponses[ i ] ):
4180 # no repeats
4181 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004182 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004183 " has incorrect view" +
4184 " of set " + onosSetName + ":\n" +
4185 str( getResponses[ i ] ) )
4186 main.log.debug( "Expected: " + str( onosSet ) )
4187 main.log.debug( "Actual: " + str( current ) )
4188 getResults = main.FALSE
4189 else:
4190 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004191 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004192 " has repeat elements in" +
4193 " set " + onosSetName + ":\n" +
4194 str( getResponses[ i ] ) )
4195 getResults = main.FALSE
4196 elif getResponses[ i ] == main.ERROR:
4197 getResults = main.FALSE
4198 sizeResponses = []
4199 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004200 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004201 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004202 name="setTestSize-" + str( i ),
4203 args=[ onosSetName ] )
4204 threads.append( t )
4205 t.start()
4206 for t in threads:
4207 t.join()
4208 sizeResponses.append( t.result )
4209 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004210 for i in range( len( main.activeNodes ) ):
4211 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004212 if size != sizeResponses[ i ]:
4213 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004214 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004215 str( size ) + " for set " + onosSetName +
4216 " but got " + str( sizeResponses[ i ] ) )
4217 retainResults = retainResults and getResults and sizeResults
4218 utilities.assert_equals( expect=main.TRUE,
4219 actual=retainResults,
4220 onpass="Set retain correct",
4221 onfail="Set retain was incorrect" )
4222
Jon Hall2a5002c2015-08-21 16:49:11 -07004223 # Transactional maps
4224 main.step( "Partitioned Transactional maps put" )
4225 tMapValue = "Testing"
4226 numKeys = 100
4227 putResult = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004228 node = main.activeNodes[0]
4229 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004230 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004231 for i in putResponses:
4232 if putResponses[ i ][ 'value' ] != tMapValue:
4233 putResult = False
4234 else:
4235 putResult = False
4236 if not putResult:
4237 main.log.debug( "Put response values: " + str( putResponses ) )
4238 utilities.assert_equals( expect=True,
4239 actual=putResult,
4240 onpass="Partitioned Transactional Map put successful",
4241 onfail="Partitioned Transactional Map put values are incorrect" )
4242
4243 main.step( "Partitioned Transactional maps get" )
4244 getCheck = True
4245 for n in range( 1, numKeys + 1 ):
4246 getResponses = []
4247 threads = []
4248 valueCheck = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004249 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004250 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4251 name="TMap-get-" + str( i ),
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004252 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004253 threads.append( t )
4254 t.start()
4255 for t in threads:
4256 t.join()
4257 getResponses.append( t.result )
4258 for node in getResponses:
4259 if node != tMapValue:
4260 valueCheck = False
4261 if not valueCheck:
4262 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4263 main.log.warn( getResponses )
4264 getCheck = getCheck and valueCheck
4265 utilities.assert_equals( expect=True,
4266 actual=getCheck,
4267 onpass="Partitioned Transactional Map get values were correct",
4268 onfail="Partitioned Transactional Map values incorrect" )