blob: bda572fc86e7d7f433d9bc3c2efd0e11d1ebc2a2 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
Jon Hallb3ed8ed2015-10-28 16:43:55 -070012CASE61: The Failure inducing case.
13CASE62: The Failure recovery case.
Jon Hall5cf14d52015-07-16 12:15:19 -070014CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
Jon Hallb3ed8ed2015-10-28 16:43:55 -070028class HAkillNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -070029
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall3b489db2015-10-05 14:38:37 -070052 import pexpect
Jon Hall6e709752016-02-01 13:38:46 -080053 import time
Jon Halla440e872016-03-31 15:15:50 -070054 import json
Jon Hall6e709752016-02-01 13:38:46 -080055 main.log.info( "ONOS HA test: Restart a minority of ONOS nodes - " +
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "initialization" )
57 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070058 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070059 "installing ONOS, starting Mininet and ONOS" +\
60 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070061
62 # load some variables from the params file
63 PULLCODE = False
64 if main.params[ 'Git' ] == 'True':
65 PULLCODE = True
66 gitBranch = main.params[ 'branch' ]
67 cellName = main.params[ 'ENV' ][ 'cellName' ]
68
Jon Halle1a3b752015-07-22 13:02:46 -070069 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070071 if main.ONOSbench.maxNodes < main.numCtrls:
72 main.numCtrls = int( main.ONOSbench.maxNodes )
73 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070074 global ONOS1Port
75 global ONOS2Port
76 global ONOS3Port
77 global ONOS4Port
78 global ONOS5Port
79 global ONOS6Port
80 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070081 # These are for csv plotting in jenkins
82 global labels
83 global data
84 labels = []
85 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070086
87 # FIXME: just get controller port from params?
88 # TODO: do we really need all these?
89 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
90 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
91 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
92 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
93 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
94 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
95 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
96
Jon Halle1a3b752015-07-22 13:02:46 -070097 try:
Jon Hall53c5e662016-04-13 16:06:56 -070098 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070099 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -0700100 except Exception as e:
101 main.log.exception( e )
102 main.cleanup()
103 main.exit()
104
105 main.CLIs = []
106 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700107 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700108 for i in range( 1, main.numCtrls + 1 ):
109 try:
110 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
111 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
112 ipList.append( main.nodes[ -1 ].ip_address )
113 except AttributeError:
114 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700115
116 main.step( "Create cell file" )
117 cellAppString = main.params[ 'ENV' ][ 'appString' ]
118 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
119 main.Mininet1.ip_address,
120 cellAppString, ipList )
121 main.step( "Applying cell variable to environment" )
122 cellResult = main.ONOSbench.setCell( cellName )
123 verifyResult = main.ONOSbench.verifyCell()
124
125 # FIXME:this is short term fix
126 main.log.info( "Removing raft logs" )
127 main.ONOSbench.onosRemoveRaftLogs()
128
129 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700130 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700131 main.ONOSbench.onosUninstall( node.ip_address )
132
133 # Make sure ONOS is DEAD
134 main.log.info( "Killing any ONOS processes" )
135 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700136 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700137 killed = main.ONOSbench.onosKill( node.ip_address )
138 killResults = killResults and killed
139
140 cleanInstallResult = main.TRUE
141 gitPullResult = main.TRUE
142
143 main.step( "Starting Mininet" )
144 # scp topo file to mininet
145 # TODO: move to params?
146 topoName = "obelisk.py"
147 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700148 main.ONOSbench.scp( main.Mininet1,
149 filePath + topoName,
150 main.Mininet1.home,
151 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700152 mnResult = main.Mininet1.startNet( )
153 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
154 onpass="Mininet Started",
155 onfail="Error starting Mininet" )
156
157 main.step( "Git checkout and pull " + gitBranch )
158 if PULLCODE:
159 main.ONOSbench.gitCheckout( gitBranch )
160 gitPullResult = main.ONOSbench.gitPull()
161 # values of 1 or 3 are good
162 utilities.assert_lesser( expect=0, actual=gitPullResult,
163 onpass="Git pull successful",
164 onfail="Git pull failed" )
165 main.ONOSbench.getVersion( report=True )
166
167 main.step( "Using mvn clean install" )
168 cleanInstallResult = main.TRUE
169 if PULLCODE and gitPullResult == main.TRUE:
170 cleanInstallResult = main.ONOSbench.cleanInstall()
171 else:
172 main.log.warn( "Did not pull new code so skipping mvn " +
173 "clean install" )
174 utilities.assert_equals( expect=main.TRUE,
175 actual=cleanInstallResult,
176 onpass="MCI successful",
177 onfail="MCI failed" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700178
179 main.step( "Make sure ONOS service doesn't automatically respawn" )
180 handle = main.ONOSbench.handle
181 handle.sendline( "sed -i -e 's/^respawn$/#respawn/g' tools/package/init/onos.conf" )
182 handle.expect( "\$" ) # $ from the command
183 handle.expect( "\$" ) # $ from the prompt
184
Jon Hall5cf14d52015-07-16 12:15:19 -0700185 # GRAPHS
186 # NOTE: important params here:
187 # job = name of Jenkins job
188 # Plot Name = Plot-HA, only can be used if multiple plots
189 # index = The number of the graph under plot name
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700190 job = "HAkillNodes"
Jon Hall5cf14d52015-07-16 12:15:19 -0700191 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700192 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700193 graphs = '<ac:structured-macro ac:name="html">\n'
194 graphs += '<ac:plain-text-body><![CDATA[\n'
195 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800196 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700197 '&width=500&height=300"' +\
198 'noborder="0" width="500" height="300" scrolling="yes" ' +\
199 'seamless="seamless"></iframe>\n'
200 graphs += ']]></ac:plain-text-body>\n'
201 graphs += '</ac:structured-macro>\n'
202 main.log.wiki(graphs)
203
204 main.step( "Creating ONOS package" )
Jon Hall3b489db2015-10-05 14:38:37 -0700205 # copy gen-partions file to ONOS
206 # NOTE: this assumes TestON and ONOS are on the same machine
Jon Hall53c5e662016-04-13 16:06:56 -0700207 srcFile = main.testDir + "/HA/dependencies/onos-gen-partitions"
Jon Hall3b489db2015-10-05 14:38:37 -0700208 dstDir = main.ONOSbench.home + "/tools/test/bin/onos-gen-partitions"
209 cpResult = main.ONOSbench.secureCopy( main.ONOSbench.user_name,
210 main.ONOSbench.ip_address,
211 srcFile,
212 dstDir,
213 pwd=main.ONOSbench.pwd,
214 direction="from" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700215 packageResult = main.ONOSbench.onosPackage()
216 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
217 onpass="ONOS package successful",
218 onfail="ONOS package failed" )
219
220 main.step( "Installing ONOS package" )
221 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700222 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700223 tmpResult = main.ONOSbench.onosInstall( options="-f",
224 node=node.ip_address )
225 onosInstallResult = onosInstallResult and tmpResult
226 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
227 onpass="ONOS install successful",
228 onfail="ONOS install failed" )
Jon Hall3b489db2015-10-05 14:38:37 -0700229 # clean up gen-partitions file
230 try:
231 main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
232 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
233 main.ONOSbench.handle.sendline( "git checkout -- tools/test/bin/onos-gen-partitions" )
234 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
235 main.log.info( " Cleaning custom gen partitions file, response was: \n" +
236 str( main.ONOSbench.handle.before ) )
237 except ( pexpect.TIMEOUT, pexpect.EOF ):
238 main.log.exception( "ONOSbench: pexpect exception found:" +
239 main.ONOSbench.handle.before )
240 main.cleanup()
241 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700242
243 main.step( "Checking if ONOS is up yet" )
244 for i in range( 2 ):
245 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700246 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700247 started = main.ONOSbench.isup( node.ip_address )
248 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800249 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700250 onosIsupResult = onosIsupResult and started
251 if onosIsupResult == main.TRUE:
252 break
253 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
254 onpass="ONOS startup successful",
255 onfail="ONOS startup failed" )
256
257 main.log.step( "Starting ONOS CLI sessions" )
258 cliResults = main.TRUE
259 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700260 for i in range( main.numCtrls ):
261 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700262 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700263 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700264 threads.append( t )
265 t.start()
266
267 for t in threads:
268 t.join()
269 cliResults = cliResults and t.result
270 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
271 onpass="ONOS cli startup successful",
272 onfail="ONOS cli startup failed" )
273
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700274 # Create a list of active nodes for use when some nodes are stopped
275 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
276
Jon Hall5cf14d52015-07-16 12:15:19 -0700277 if main.params[ 'tcpdump' ].lower() == "true":
278 main.step( "Start Packet Capture MN" )
279 main.Mininet2.startTcpdump(
280 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
281 + "-MN.pcap",
282 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
283 port=main.params[ 'MNtcpdump' ][ 'port' ] )
284
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700285 main.step( "Clean up ONOS service changes" )
286 handle.sendline( "git checkout -- tools/package/init/onos.conf" )
287 handle.expect( "\$" )
288
Jon Halla440e872016-03-31 15:15:50 -0700289 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700290 nodeResults = utilities.retry( main.HA.nodesCheck,
291 False,
292 args=[main.activeNodes],
293 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700294
Jon Hall41d39f12016-04-11 22:54:35 -0700295 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700296 onpass="Nodes check successful",
297 onfail="Nodes check NOT successful" )
298
299 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700300 for i in main.activeNodes:
301 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700302 main.log.debug( "{} components not ACTIVE: \n{}".format(
303 cli.name,
304 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700305 main.log.error( "Failed to start ONOS, stopping test" )
306 main.cleanup()
307 main.exit()
308
Jon Hall172b7ba2016-04-07 18:12:20 -0700309 main.step( "Activate apps defined in the params file" )
310 # get data from the params
311 apps = main.params.get( 'apps' )
312 if apps:
313 apps = apps.split(',')
314 main.log.warn( apps )
315 activateResult = True
316 for app in apps:
317 main.CLIs[ 0 ].app( app, "Activate" )
318 # TODO: check this worked
319 time.sleep( 10 ) # wait for apps to activate
320 for app in apps:
321 state = main.CLIs[ 0 ].appStatus( app )
322 if state == "ACTIVE":
323 activateResult = activeResult and True
324 else:
325 main.log.error( "{} is in {} state".format( app, state ) )
326 activeResult = False
327 utilities.assert_equals( expect=True,
328 actual=activateResult,
329 onpass="Successfully activated apps",
330 onfail="Failed to activate apps" )
331 else:
332 main.log.warn( "No apps were specified to be loaded after startup" )
333
334 main.step( "Set ONOS configurations" )
335 config = main.params.get( 'ONOS_Configuration' )
336 if config:
337 main.log.debug( config )
338 checkResult = main.TRUE
339 for component in config:
340 for setting in config[component]:
341 value = config[component][setting]
342 check = main.CLIs[ 0 ].setCfg( component, setting, value )
343 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
344 checkResult = check and checkResult
345 utilities.assert_equals( expect=main.TRUE,
346 actual=checkResult,
347 onpass="Successfully set config",
348 onfail="Failed to set config" )
349 else:
350 main.log.warn( "No configurations were specified to be changed after startup" )
351
Jon Hall9d2dcad2016-04-08 10:15:20 -0700352 main.step( "App Ids check" )
353 appCheck = main.TRUE
354 threads = []
355 for i in main.activeNodes:
356 t = main.Thread( target=main.CLIs[i].appToIDCheck,
357 name="appToIDCheck-" + str( i ),
358 args=[] )
359 threads.append( t )
360 t.start()
361
362 for t in threads:
363 t.join()
364 appCheck = appCheck and t.result
365 if appCheck != main.TRUE:
366 node = main.activeNodes[0]
367 main.log.warn( main.CLIs[node].apps() )
368 main.log.warn( main.CLIs[node].appIDs() )
369 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
370 onpass="App Ids seem to be correct",
371 onfail="Something is wrong with app Ids" )
372
Jon Hall5cf14d52015-07-16 12:15:19 -0700373 def CASE2( self, main ):
374 """
375 Assign devices to controllers
376 """
377 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700378 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700379 assert main, "main not defined"
380 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700381 assert main.CLIs, "main.CLIs not defined"
382 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700383 assert ONOS1Port, "ONOS1Port not defined"
384 assert ONOS2Port, "ONOS2Port not defined"
385 assert ONOS3Port, "ONOS3Port not defined"
386 assert ONOS4Port, "ONOS4Port not defined"
387 assert ONOS5Port, "ONOS5Port not defined"
388 assert ONOS6Port, "ONOS6Port not defined"
389 assert ONOS7Port, "ONOS7Port not defined"
390
391 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700392 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700393 "and check that an ONOS node becomes the " +\
394 "master of the device."
395 main.step( "Assign switches to controllers" )
396
397 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700398 for i in range( main.numCtrls ):
399 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700400 swList = []
401 for i in range( 1, 29 ):
402 swList.append( "s" + str( i ) )
403 main.Mininet1.assignSwController( sw=swList, ip=ipList )
404
405 mastershipCheck = main.TRUE
406 for i in range( 1, 29 ):
407 response = main.Mininet1.getSwController( "s" + str( i ) )
408 try:
409 main.log.info( str( response ) )
410 except Exception:
411 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700412 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700413 if re.search( "tcp:" + node.ip_address, response ):
414 mastershipCheck = mastershipCheck and main.TRUE
415 else:
416 main.log.error( "Error, node " + node.ip_address + " is " +
417 "not in the list of controllers s" +
418 str( i ) + " is connecting to." )
419 mastershipCheck = main.FALSE
420 utilities.assert_equals(
421 expect=main.TRUE,
422 actual=mastershipCheck,
423 onpass="Switch mastership assigned correctly",
424 onfail="Switches not assigned correctly to controllers" )
425
426 def CASE21( self, main ):
427 """
428 Assign mastership to controllers
429 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700430 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700431 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700432 assert main, "main not defined"
433 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700434 assert main.CLIs, "main.CLIs not defined"
435 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700436 assert ONOS1Port, "ONOS1Port not defined"
437 assert ONOS2Port, "ONOS2Port not defined"
438 assert ONOS3Port, "ONOS3Port not defined"
439 assert ONOS4Port, "ONOS4Port not defined"
440 assert ONOS5Port, "ONOS5Port not defined"
441 assert ONOS6Port, "ONOS6Port not defined"
442 assert ONOS7Port, "ONOS7Port not defined"
443
444 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700445 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700446 "device. Then manually assign" +\
447 " mastership to specific ONOS nodes using" +\
448 " 'device-role'"
449 main.step( "Assign mastership of switches to specific controllers" )
450 # Manually assign mastership to the controller we want
451 roleCall = main.TRUE
452
453 ipList = [ ]
454 deviceList = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700455 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700456 try:
457 # Assign mastership to specific controllers. This assignment was
458 # determined for a 7 node cluser, but will work with any sized
459 # cluster
460 for i in range( 1, 29 ): # switches 1 through 28
461 # set up correct variables:
462 if i == 1:
463 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700464 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700465 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700466 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700467 c = 1 % main.numCtrls
468 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700469 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700470 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700471 c = 1 % main.numCtrls
472 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700473 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700474 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700475 c = 3 % main.numCtrls
476 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700477 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700478 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700479 c = 2 % main.numCtrls
480 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700481 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700482 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700483 c = 2 % main.numCtrls
484 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700485 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700486 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700487 c = 5 % main.numCtrls
488 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700489 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700490 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700491 c = 4 % main.numCtrls
492 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700493 dpid = '3' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700494 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700495 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700496 c = 6 % main.numCtrls
497 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700498 dpid = '6' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700499 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700500 elif i == 28:
501 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700502 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700503 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700504 else:
505 main.log.error( "You didn't write an else statement for " +
506 "switch s" + str( i ) )
507 roleCall = main.FALSE
508 # Assign switch
509 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
510 # TODO: make this controller dynamic
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700511 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700512 ipList.append( ip )
513 deviceList.append( deviceId )
514 except ( AttributeError, AssertionError ):
515 main.log.exception( "Something is wrong with ONOS device view" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700516 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700517 utilities.assert_equals(
518 expect=main.TRUE,
519 actual=roleCall,
520 onpass="Re-assigned switch mastership to designated controller",
521 onfail="Something wrong with deviceRole calls" )
522
523 main.step( "Check mastership was correctly assigned" )
524 roleCheck = main.TRUE
525 # NOTE: This is due to the fact that device mastership change is not
526 # atomic and is actually a multi step process
527 time.sleep( 5 )
528 for i in range( len( ipList ) ):
529 ip = ipList[i]
530 deviceId = deviceList[i]
531 # Check assignment
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700532 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700533 if ip in master:
534 roleCheck = roleCheck and main.TRUE
535 else:
536 roleCheck = roleCheck and main.FALSE
537 main.log.error( "Error, controller " + ip + " is not" +
538 " master " + "of device " +
539 str( deviceId ) + ". Master is " +
540 repr( master ) + "." )
541 utilities.assert_equals(
542 expect=main.TRUE,
543 actual=roleCheck,
544 onpass="Switches were successfully reassigned to designated " +
545 "controller",
546 onfail="Switches were not successfully reassigned" )
547
548 def CASE3( self, main ):
549 """
550 Assign intents
551 """
552 import time
553 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700554 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700555 assert main, "main not defined"
556 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700557 assert main.CLIs, "main.CLIs not defined"
558 assert main.nodes, "main.nodes not defined"
Jon Halla440e872016-03-31 15:15:50 -0700559 try:
560 labels
561 except NameError:
562 main.log.error( "labels not defined, setting to []" )
563 labels = []
564 try:
565 data
566 except NameError:
567 main.log.error( "data not defined, setting to []" )
568 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700569 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700570 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700571 "assign predetermined host-to-host intents." +\
572 " After installation, check that the intent" +\
573 " is distributed to all nodes and the state" +\
574 " is INSTALLED"
575
576 # install onos-app-fwd
577 main.step( "Install reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700578 onosCli = main.CLIs[ main.activeNodes[0] ]
579 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700580 utilities.assert_equals( expect=main.TRUE, actual=installResults,
581 onpass="Install fwd successful",
582 onfail="Install fwd failed" )
583
584 main.step( "Check app ids" )
585 appCheck = main.TRUE
586 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700587 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700588 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700589 name="appToIDCheck-" + str( i ),
590 args=[] )
591 threads.append( t )
592 t.start()
593
594 for t in threads:
595 t.join()
596 appCheck = appCheck and t.result
597 if appCheck != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700598 main.log.warn( onosCli.apps() )
599 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700600 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
601 onpass="App Ids seem to be correct",
602 onfail="Something is wrong with app Ids" )
603
604 main.step( "Discovering Hosts( Via pingall for now )" )
605 # FIXME: Once we have a host discovery mechanism, use that instead
606 # REACTIVE FWD test
607 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700608 passMsg = "Reactive Pingall test passed"
609 time1 = time.time()
610 pingResult = main.Mininet1.pingall()
611 time2 = time.time()
612 if not pingResult:
613 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700614 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700615 passMsg += " on the second try"
616 utilities.assert_equals(
617 expect=main.TRUE,
618 actual=pingResult,
619 onpass= passMsg,
620 onfail="Reactive Pingall failed, " +
621 "one or more ping pairs failed" )
622 main.log.info( "Time for pingall: %2f seconds" %
623 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700624 # timeout for fwd flows
625 time.sleep( 11 )
626 # uninstall onos-app-fwd
627 main.step( "Uninstall reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700628 node = main.activeNodes[0]
629 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700630 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
631 onpass="Uninstall fwd successful",
632 onfail="Uninstall fwd failed" )
633
634 main.step( "Check app ids" )
635 threads = []
636 appCheck2 = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700637 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700638 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700639 name="appToIDCheck-" + str( i ),
640 args=[] )
641 threads.append( t )
642 t.start()
643
644 for t in threads:
645 t.join()
646 appCheck2 = appCheck2 and t.result
647 if appCheck2 != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700648 node = main.activeNodes[0]
649 main.log.warn( main.CLIs[node].apps() )
650 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700651 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
652 onpass="App Ids seem to be correct",
653 onfail="Something is wrong with app Ids" )
654
655 main.step( "Add host intents via cli" )
656 intentIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700657 # TODO: move the host numbers to params
658 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700659 intentAddResult = True
660 hostResult = main.TRUE
661 for i in range( 8, 18 ):
662 main.log.info( "Adding host intent between h" + str( i ) +
663 " and h" + str( i + 10 ) )
664 host1 = "00:00:00:00:00:" + \
665 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
666 host2 = "00:00:00:00:00:" + \
667 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
668 # NOTE: getHost can return None
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700669 host1Dict = onosCli.getHost( host1 )
670 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700671 host1Id = None
672 host2Id = None
673 if host1Dict and host2Dict:
674 host1Id = host1Dict.get( 'id', None )
675 host2Id = host2Dict.get( 'id', None )
676 if host1Id and host2Id:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700677 nodeNum = ( i % len( main.activeNodes ) )
678 node = main.activeNodes[nodeNum]
679 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700680 if tmpId:
681 main.log.info( "Added intent with id: " + tmpId )
682 intentIds.append( tmpId )
683 else:
684 main.log.error( "addHostIntent returned: " +
685 repr( tmpId ) )
686 else:
687 main.log.error( "Error, getHost() failed for h" + str( i ) +
688 " and/or h" + str( i + 10 ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700689 node = main.activeNodes[0]
690 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700691 main.log.warn( "Hosts output: " )
692 try:
693 main.log.warn( json.dumps( json.loads( hosts ),
694 sort_keys=True,
695 indent=4,
696 separators=( ',', ': ' ) ) )
697 except ( ValueError, TypeError ):
698 main.log.warn( repr( hosts ) )
699 hostResult = main.FALSE
700 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
701 onpass="Found a host id for each host",
702 onfail="Error looking up host ids" )
703
704 intentStart = time.time()
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700705 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700706 main.log.info( "Submitted intents: " + str( intentIds ) )
707 main.log.info( "Intents in ONOS: " + str( onosIds ) )
708 for intent in intentIds:
709 if intent in onosIds:
710 pass # intent submitted is in onos
711 else:
712 intentAddResult = False
713 if intentAddResult:
714 intentStop = time.time()
715 else:
716 intentStop = None
717 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700718 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700719 intentStates = []
720 installedCheck = True
721 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
722 count = 0
723 try:
724 for intent in json.loads( intents ):
725 state = intent.get( 'state', None )
726 if "INSTALLED" not in state:
727 installedCheck = False
728 intentId = intent.get( 'id', None )
729 intentStates.append( ( intentId, state ) )
730 except ( ValueError, TypeError ):
731 main.log.exception( "Error parsing intents" )
732 # add submitted intents not in the store
733 tmplist = [ i for i, s in intentStates ]
734 missingIntents = False
735 for i in intentIds:
736 if i not in tmplist:
737 intentStates.append( ( i, " - " ) )
738 missingIntents = True
739 intentStates.sort()
740 for i, s in intentStates:
741 count += 1
742 main.log.info( "%-6s%-15s%-15s" %
743 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700744 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700745 try:
746 missing = False
747 if leaders:
748 parsedLeaders = json.loads( leaders )
749 main.log.warn( json.dumps( parsedLeaders,
750 sort_keys=True,
751 indent=4,
752 separators=( ',', ': ' ) ) )
753 # check for all intent partitions
754 topics = []
755 for i in range( 14 ):
756 topics.append( "intent-partition-" + str( i ) )
757 main.log.debug( topics )
758 ONOStopics = [ j['topic'] for j in parsedLeaders ]
759 for topic in topics:
760 if topic not in ONOStopics:
761 main.log.error( "Error: " + topic +
762 " not in leaders" )
763 missing = True
764 else:
765 main.log.error( "leaders() returned None" )
766 except ( ValueError, TypeError ):
767 main.log.exception( "Error parsing leaders" )
768 main.log.error( repr( leaders ) )
769 # Check all nodes
770 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700771 for i in main.activeNodes:
772 response = main.CLIs[i].leaders( jsonFormat=False)
773 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700774 str( response ) )
775
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700776 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700777 try:
778 if partitions :
779 parsedPartitions = json.loads( partitions )
780 main.log.warn( json.dumps( parsedPartitions,
781 sort_keys=True,
782 indent=4,
783 separators=( ',', ': ' ) ) )
784 # TODO check for a leader in all paritions
785 # TODO check for consistency among nodes
786 else:
787 main.log.error( "partitions() returned None" )
788 except ( ValueError, TypeError ):
789 main.log.exception( "Error parsing partitions" )
790 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700791 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700792 try:
793 if pendingMap :
794 parsedPending = json.loads( pendingMap )
795 main.log.warn( json.dumps( parsedPending,
796 sort_keys=True,
797 indent=4,
798 separators=( ',', ': ' ) ) )
799 # TODO check something here?
800 else:
801 main.log.error( "pendingMap() returned None" )
802 except ( ValueError, TypeError ):
803 main.log.exception( "Error parsing pending map" )
804 main.log.error( repr( pendingMap ) )
805
806 intentAddResult = bool( intentAddResult and not missingIntents and
807 installedCheck )
808 if not intentAddResult:
809 main.log.error( "Error in pushing host intents to ONOS" )
810
811 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700812 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700813 correct = True
814 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700815 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700816 onosIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700817 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700818 onosIds.append( ids )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700819 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700820 str( sorted( onosIds ) ) )
821 if sorted( ids ) != sorted( intentIds ):
822 main.log.warn( "Set of intent IDs doesn't match" )
823 correct = False
824 break
825 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700826 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700827 for intent in intents:
828 if intent[ 'state' ] != "INSTALLED":
829 main.log.warn( "Intent " + intent[ 'id' ] +
830 " is " + intent[ 'state' ] )
831 correct = False
832 break
833 if correct:
834 break
835 else:
836 time.sleep(1)
837 if not intentStop:
838 intentStop = time.time()
839 global gossipTime
840 gossipTime = intentStop - intentStart
841 main.log.info( "It took about " + str( gossipTime ) +
842 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700843 gossipPeriod = int( main.params['timers']['gossip'] )
844 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700845 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700846 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700847 onpass="ECM anti-entropy for intents worked within " +
848 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700849 onfail="Intent ECM anti-entropy took too long. " +
850 "Expected time:{}, Actual time:{}".format( maxGossipTime,
851 gossipTime ) )
852 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700853 intentAddResult = True
854
855 if not intentAddResult or "key" in pendingMap:
856 import time
857 installedCheck = True
858 main.log.info( "Sleeping 60 seconds to see if intents are found" )
859 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700860 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700861 main.log.info( "Submitted intents: " + str( intentIds ) )
862 main.log.info( "Intents in ONOS: " + str( onosIds ) )
863 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700864 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700865 intentStates = []
866 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
867 count = 0
868 try:
869 for intent in json.loads( intents ):
870 # Iter through intents of a node
871 state = intent.get( 'state', None )
872 if "INSTALLED" not in state:
873 installedCheck = False
874 intentId = intent.get( 'id', None )
875 intentStates.append( ( intentId, state ) )
876 except ( ValueError, TypeError ):
877 main.log.exception( "Error parsing intents" )
878 # add submitted intents not in the store
879 tmplist = [ i for i, s in intentStates ]
880 for i in intentIds:
881 if i not in tmplist:
882 intentStates.append( ( i, " - " ) )
883 intentStates.sort()
884 for i, s in intentStates:
885 count += 1
886 main.log.info( "%-6s%-15s%-15s" %
887 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700888 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700889 try:
890 missing = False
891 if leaders:
892 parsedLeaders = json.loads( leaders )
893 main.log.warn( json.dumps( parsedLeaders,
894 sort_keys=True,
895 indent=4,
896 separators=( ',', ': ' ) ) )
897 # check for all intent partitions
898 # check for election
899 topics = []
900 for i in range( 14 ):
901 topics.append( "intent-partition-" + str( i ) )
902 # FIXME: this should only be after we start the app
903 topics.append( "org.onosproject.election" )
904 main.log.debug( topics )
905 ONOStopics = [ j['topic'] for j in parsedLeaders ]
906 for topic in topics:
907 if topic not in ONOStopics:
908 main.log.error( "Error: " + topic +
909 " not in leaders" )
910 missing = True
911 else:
912 main.log.error( "leaders() returned None" )
913 except ( ValueError, TypeError ):
914 main.log.exception( "Error parsing leaders" )
915 main.log.error( repr( leaders ) )
916 # Check all nodes
917 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700918 for i in main.activeNodes:
919 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700920 response = node.leaders( jsonFormat=False)
921 main.log.warn( str( node.name ) + " leaders output: \n" +
922 str( response ) )
923
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700924 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700925 try:
926 if partitions :
927 parsedPartitions = json.loads( partitions )
928 main.log.warn( json.dumps( parsedPartitions,
929 sort_keys=True,
930 indent=4,
931 separators=( ',', ': ' ) ) )
932 # TODO check for a leader in all paritions
933 # TODO check for consistency among nodes
934 else:
935 main.log.error( "partitions() returned None" )
936 except ( ValueError, TypeError ):
937 main.log.exception( "Error parsing partitions" )
938 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700939 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700940 try:
941 if pendingMap :
942 parsedPending = json.loads( pendingMap )
943 main.log.warn( json.dumps( parsedPending,
944 sort_keys=True,
945 indent=4,
946 separators=( ',', ': ' ) ) )
947 # TODO check something here?
948 else:
949 main.log.error( "pendingMap() returned None" )
950 except ( ValueError, TypeError ):
951 main.log.exception( "Error parsing pending map" )
952 main.log.error( repr( pendingMap ) )
953
954 def CASE4( self, main ):
955 """
956 Ping across added host intents
957 """
958 import json
959 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700960 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700961 assert main, "main not defined"
962 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700963 assert main.CLIs, "main.CLIs not defined"
964 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700965 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700966 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700967 "functionality and check the state of " +\
968 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700969
Jon Hall41d39f12016-04-11 22:54:35 -0700970 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700971 main.step( "Check Intent state" )
972 installedCheck = False
973 loopCount = 0
974 while not installedCheck and loopCount < 40:
975 installedCheck = True
976 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700977 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700978 intentStates = []
979 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
980 count = 0
981 # Iter through intents of a node
982 try:
983 for intent in json.loads( intents ):
984 state = intent.get( 'state', None )
985 if "INSTALLED" not in state:
986 installedCheck = False
987 intentId = intent.get( 'id', None )
988 intentStates.append( ( intentId, state ) )
989 except ( ValueError, TypeError ):
990 main.log.exception( "Error parsing intents." )
991 # Print states
992 intentStates.sort()
993 for i, s in intentStates:
994 count += 1
995 main.log.info( "%-6s%-15s%-15s" %
996 ( str( count ), str( i ), str( s ) ) )
997 if not installedCheck:
998 time.sleep( 1 )
999 loopCount += 1
1000 utilities.assert_equals( expect=True, actual=installedCheck,
1001 onpass="Intents are all INSTALLED",
1002 onfail="Intents are not all in " +
1003 "INSTALLED state" )
1004
Jon Hall9d2dcad2016-04-08 10:15:20 -07001005 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -07001006 PingResult = main.TRUE
1007 for i in range( 8, 18 ):
1008 ping = main.Mininet1.pingHost( src="h" + str( i ),
1009 target="h" + str( i + 10 ) )
1010 PingResult = PingResult and ping
1011 if ping == main.FALSE:
1012 main.log.warn( "Ping failed between h" + str( i ) +
1013 " and h" + str( i + 10 ) )
1014 elif ping == main.TRUE:
1015 main.log.info( "Ping test passed!" )
1016 # Don't set PingResult or you'd override failures
1017 if PingResult == main.FALSE:
1018 main.log.error(
1019 "Intents have not been installed correctly, pings failed." )
1020 # TODO: pretty print
1021 main.log.warn( "ONOS1 intents: " )
1022 try:
1023 tmpIntents = onosCli.intents()
1024 main.log.warn( json.dumps( json.loads( tmpIntents ),
1025 sort_keys=True,
1026 indent=4,
1027 separators=( ',', ': ' ) ) )
1028 except ( ValueError, TypeError ):
1029 main.log.warn( repr( tmpIntents ) )
1030 utilities.assert_equals(
1031 expect=main.TRUE,
1032 actual=PingResult,
1033 onpass="Intents have been installed correctly and pings work",
1034 onfail="Intents have not been installed correctly, pings failed." )
1035
Jon Hall5cf14d52015-07-16 12:15:19 -07001036 main.step( "Check leadership of topics" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001037 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001038 topicCheck = main.TRUE
1039 try:
1040 if leaders:
1041 parsedLeaders = json.loads( leaders )
1042 main.log.warn( json.dumps( parsedLeaders,
1043 sort_keys=True,
1044 indent=4,
1045 separators=( ',', ': ' ) ) )
1046 # check for all intent partitions
1047 # check for election
1048 # TODO: Look at Devices as topics now that it uses this system
1049 topics = []
1050 for i in range( 14 ):
1051 topics.append( "intent-partition-" + str( i ) )
1052 # FIXME: this should only be after we start the app
1053 # FIXME: topics.append( "org.onosproject.election" )
1054 # Print leaders output
1055 main.log.debug( topics )
1056 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1057 for topic in topics:
1058 if topic not in ONOStopics:
1059 main.log.error( "Error: " + topic +
1060 " not in leaders" )
1061 topicCheck = main.FALSE
1062 else:
1063 main.log.error( "leaders() returned None" )
1064 topicCheck = main.FALSE
1065 except ( ValueError, TypeError ):
1066 topicCheck = main.FALSE
1067 main.log.exception( "Error parsing leaders" )
1068 main.log.error( repr( leaders ) )
1069 # TODO: Check for a leader of these topics
1070 # Check all nodes
1071 if topicCheck:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001072 for i in main.activeNodes:
1073 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001074 response = node.leaders( jsonFormat=False)
1075 main.log.warn( str( node.name ) + " leaders output: \n" +
1076 str( response ) )
1077
1078 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1079 onpass="intent Partitions is in leaders",
1080 onfail="Some topics were lost " )
1081 # Print partitions
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001082 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001083 try:
1084 if partitions :
1085 parsedPartitions = json.loads( partitions )
1086 main.log.warn( json.dumps( parsedPartitions,
1087 sort_keys=True,
1088 indent=4,
1089 separators=( ',', ': ' ) ) )
1090 # TODO check for a leader in all paritions
1091 # TODO check for consistency among nodes
1092 else:
1093 main.log.error( "partitions() returned None" )
1094 except ( ValueError, TypeError ):
1095 main.log.exception( "Error parsing partitions" )
1096 main.log.error( repr( partitions ) )
1097 # Print Pending Map
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001098 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001099 try:
1100 if pendingMap :
1101 parsedPending = json.loads( pendingMap )
1102 main.log.warn( json.dumps( parsedPending,
1103 sort_keys=True,
1104 indent=4,
1105 separators=( ',', ': ' ) ) )
1106 # TODO check something here?
1107 else:
1108 main.log.error( "pendingMap() returned None" )
1109 except ( ValueError, TypeError ):
1110 main.log.exception( "Error parsing pending map" )
1111 main.log.error( repr( pendingMap ) )
1112
1113 if not installedCheck:
1114 main.log.info( "Waiting 60 seconds to see if the state of " +
1115 "intents change" )
1116 time.sleep( 60 )
1117 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001118 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001119 intentStates = []
1120 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1121 count = 0
1122 # Iter through intents of a node
1123 try:
1124 for intent in json.loads( intents ):
1125 state = intent.get( 'state', None )
1126 if "INSTALLED" not in state:
1127 installedCheck = False
1128 intentId = intent.get( 'id', None )
1129 intentStates.append( ( intentId, state ) )
1130 except ( ValueError, TypeError ):
1131 main.log.exception( "Error parsing intents." )
1132 intentStates.sort()
1133 for i, s in intentStates:
1134 count += 1
1135 main.log.info( "%-6s%-15s%-15s" %
1136 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001137 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001138 try:
1139 missing = False
1140 if leaders:
1141 parsedLeaders = json.loads( leaders )
1142 main.log.warn( json.dumps( parsedLeaders,
1143 sort_keys=True,
1144 indent=4,
1145 separators=( ',', ': ' ) ) )
1146 # check for all intent partitions
1147 # check for election
1148 topics = []
1149 for i in range( 14 ):
1150 topics.append( "intent-partition-" + str( i ) )
1151 # FIXME: this should only be after we start the app
1152 topics.append( "org.onosproject.election" )
1153 main.log.debug( topics )
1154 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1155 for topic in topics:
1156 if topic not in ONOStopics:
1157 main.log.error( "Error: " + topic +
1158 " not in leaders" )
1159 missing = True
1160 else:
1161 main.log.error( "leaders() returned None" )
1162 except ( ValueError, TypeError ):
1163 main.log.exception( "Error parsing leaders" )
1164 main.log.error( repr( leaders ) )
1165 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001166 for i in main.activeNodes:
1167 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001168 response = node.leaders( jsonFormat=False)
1169 main.log.warn( str( node.name ) + " leaders output: \n" +
1170 str( response ) )
1171
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001172 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001173 try:
1174 if partitions :
1175 parsedPartitions = json.loads( partitions )
1176 main.log.warn( json.dumps( parsedPartitions,
1177 sort_keys=True,
1178 indent=4,
1179 separators=( ',', ': ' ) ) )
1180 # TODO check for a leader in all paritions
1181 # TODO check for consistency among nodes
1182 else:
1183 main.log.error( "partitions() returned None" )
1184 except ( ValueError, TypeError ):
1185 main.log.exception( "Error parsing partitions" )
1186 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001187 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001188 try:
1189 if pendingMap :
1190 parsedPending = json.loads( pendingMap )
1191 main.log.warn( json.dumps( parsedPending,
1192 sort_keys=True,
1193 indent=4,
1194 separators=( ',', ': ' ) ) )
1195 # TODO check something here?
1196 else:
1197 main.log.error( "pendingMap() returned None" )
1198 except ( ValueError, TypeError ):
1199 main.log.exception( "Error parsing pending map" )
1200 main.log.error( repr( pendingMap ) )
1201 # Print flowrules
Jon Hall41d39f12016-04-11 22:54:35 -07001202 main.log.debug( onosCli.flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001203 main.step( "Wait a minute then ping again" )
1204 # the wait is above
1205 PingResult = main.TRUE
1206 for i in range( 8, 18 ):
1207 ping = main.Mininet1.pingHost( src="h" + str( i ),
1208 target="h" + str( i + 10 ) )
1209 PingResult = PingResult and ping
1210 if ping == main.FALSE:
1211 main.log.warn( "Ping failed between h" + str( i ) +
1212 " and h" + str( i + 10 ) )
1213 elif ping == main.TRUE:
1214 main.log.info( "Ping test passed!" )
1215 # Don't set PingResult or you'd override failures
1216 if PingResult == main.FALSE:
1217 main.log.error(
1218 "Intents have not been installed correctly, pings failed." )
1219 # TODO: pretty print
1220 main.log.warn( "ONOS1 intents: " )
1221 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001222 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001223 main.log.warn( json.dumps( json.loads( tmpIntents ),
1224 sort_keys=True,
1225 indent=4,
1226 separators=( ',', ': ' ) ) )
1227 except ( ValueError, TypeError ):
1228 main.log.warn( repr( tmpIntents ) )
1229 utilities.assert_equals(
1230 expect=main.TRUE,
1231 actual=PingResult,
1232 onpass="Intents have been installed correctly and pings work",
1233 onfail="Intents have not been installed correctly, pings failed." )
1234
1235 def CASE5( self, main ):
1236 """
1237 Reading state of ONOS
1238 """
1239 import json
1240 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001241 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001242 assert main, "main not defined"
1243 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001244 assert main.CLIs, "main.CLIs not defined"
1245 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001246
1247 main.case( "Setting up and gathering data for current state" )
1248 # The general idea for this test case is to pull the state of
1249 # ( intents,flows, topology,... ) from each ONOS node
1250 # We can then compare them with each other and also with past states
1251
1252 main.step( "Check that each switch has a master" )
1253 global mastershipState
1254 mastershipState = '[]'
1255
1256 # Assert that each device has a master
1257 rolesNotNull = main.TRUE
1258 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001259 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001260 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001261 name="rolesNotNull-" + str( i ),
1262 args=[] )
1263 threads.append( t )
1264 t.start()
1265
1266 for t in threads:
1267 t.join()
1268 rolesNotNull = rolesNotNull and t.result
1269 utilities.assert_equals(
1270 expect=main.TRUE,
1271 actual=rolesNotNull,
1272 onpass="Each device has a master",
1273 onfail="Some devices don't have a master assigned" )
1274
1275 main.step( "Get the Mastership of each switch from each controller" )
1276 ONOSMastership = []
1277 mastershipCheck = main.FALSE
1278 consistentMastership = True
1279 rolesResults = True
1280 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001281 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001282 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001283 name="roles-" + str( i ),
1284 args=[] )
1285 threads.append( t )
1286 t.start()
1287
1288 for t in threads:
1289 t.join()
1290 ONOSMastership.append( t.result )
1291
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001292 for i in range( len( ONOSMastership ) ):
1293 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001294 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001295 main.log.error( "Error in getting ONOS" + node + " roles" )
1296 main.log.warn( "ONOS" + node + " mastership response: " +
1297 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001298 rolesResults = False
1299 utilities.assert_equals(
1300 expect=True,
1301 actual=rolesResults,
1302 onpass="No error in reading roles output",
1303 onfail="Error in reading roles from ONOS" )
1304
1305 main.step( "Check for consistency in roles from each controller" )
1306 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1307 main.log.info(
1308 "Switch roles are consistent across all ONOS nodes" )
1309 else:
1310 consistentMastership = False
1311 utilities.assert_equals(
1312 expect=True,
1313 actual=consistentMastership,
1314 onpass="Switch roles are consistent across all ONOS nodes",
1315 onfail="ONOS nodes have different views of switch roles" )
1316
1317 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001318 for i in range( len( main.activeNodes ) ):
1319 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001320 try:
1321 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001322 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001323 json.dumps(
1324 json.loads( ONOSMastership[ i ] ),
1325 sort_keys=True,
1326 indent=4,
1327 separators=( ',', ': ' ) ) )
1328 except ( ValueError, TypeError ):
1329 main.log.warn( repr( ONOSMastership[ i ] ) )
1330 elif rolesResults and consistentMastership:
1331 mastershipCheck = main.TRUE
1332 mastershipState = ONOSMastership[ 0 ]
1333
1334 main.step( "Get the intents from each controller" )
1335 global intentState
1336 intentState = []
1337 ONOSIntents = []
1338 intentCheck = main.FALSE
1339 consistentIntents = True
1340 intentsResults = True
1341 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001342 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001343 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001344 name="intents-" + str( i ),
1345 args=[],
1346 kwargs={ 'jsonFormat': True } )
1347 threads.append( t )
1348 t.start()
1349
1350 for t in threads:
1351 t.join()
1352 ONOSIntents.append( t.result )
1353
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001354 for i in range( len( ONOSIntents ) ):
1355 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001356 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001357 main.log.error( "Error in getting ONOS" + node + " intents" )
1358 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001359 repr( ONOSIntents[ i ] ) )
1360 intentsResults = False
1361 utilities.assert_equals(
1362 expect=True,
1363 actual=intentsResults,
1364 onpass="No error in reading intents output",
1365 onfail="Error in reading intents from ONOS" )
1366
1367 main.step( "Check for consistency in Intents from each controller" )
1368 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1369 main.log.info( "Intents are consistent across all ONOS " +
1370 "nodes" )
1371 else:
1372 consistentIntents = False
1373 main.log.error( "Intents not consistent" )
1374 utilities.assert_equals(
1375 expect=True,
1376 actual=consistentIntents,
1377 onpass="Intents are consistent across all ONOS nodes",
1378 onfail="ONOS nodes have different views of intents" )
1379
1380 if intentsResults:
1381 # Try to make it easy to figure out what is happening
1382 #
1383 # Intent ONOS1 ONOS2 ...
1384 # 0x01 INSTALLED INSTALLING
1385 # ... ... ...
1386 # ... ... ...
1387 title = " Id"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001388 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001389 title += " " * 10 + "ONOS" + str( n + 1 )
1390 main.log.warn( title )
1391 # get all intent keys in the cluster
1392 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001393 try:
1394 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001395 for nodeStr in ONOSIntents:
1396 node = json.loads( nodeStr )
1397 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001398 keys.append( intent.get( 'id' ) )
1399 keys = set( keys )
1400 # For each intent key, print the state on each node
1401 for key in keys:
1402 row = "%-13s" % key
1403 for nodeStr in ONOSIntents:
1404 node = json.loads( nodeStr )
1405 for intent in node:
1406 if intent.get( 'id', "Error" ) == key:
1407 row += "%-15s" % intent.get( 'state' )
1408 main.log.warn( row )
1409 # End of intent state table
1410 except ValueError as e:
1411 main.log.exception( e )
1412 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001413
1414 if intentsResults and not consistentIntents:
1415 # print the json objects
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001416 n = str( main.activeNodes[-1] + 1 )
1417 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001418 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1419 sort_keys=True,
1420 indent=4,
1421 separators=( ',', ': ' ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001422 for i in range( len( ONOSIntents ) ):
1423 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001424 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001425 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001426 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1427 sort_keys=True,
1428 indent=4,
1429 separators=( ',', ': ' ) ) )
1430 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001431 main.log.debug( "ONOS" + node + " intents match ONOS" +
1432 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001433 elif intentsResults and consistentIntents:
1434 intentCheck = main.TRUE
1435 intentState = ONOSIntents[ 0 ]
1436
1437 main.step( "Get the flows from each controller" )
1438 global flowState
1439 flowState = []
1440 ONOSFlows = []
1441 ONOSFlowsJson = []
1442 flowCheck = main.FALSE
1443 consistentFlows = True
1444 flowsResults = True
1445 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001446 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001447 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001448 name="flows-" + str( i ),
1449 args=[],
1450 kwargs={ 'jsonFormat': True } )
1451 threads.append( t )
1452 t.start()
1453
1454 # NOTE: Flows command can take some time to run
1455 time.sleep(30)
1456 for t in threads:
1457 t.join()
1458 result = t.result
1459 ONOSFlows.append( result )
1460
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001461 for i in range( len( ONOSFlows ) ):
1462 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001463 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1464 main.log.error( "Error in getting ONOS" + num + " flows" )
1465 main.log.warn( "ONOS" + num + " flows response: " +
1466 repr( ONOSFlows[ i ] ) )
1467 flowsResults = False
1468 ONOSFlowsJson.append( None )
1469 else:
1470 try:
1471 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1472 except ( ValueError, TypeError ):
1473 # FIXME: change this to log.error?
1474 main.log.exception( "Error in parsing ONOS" + num +
1475 " response as json." )
1476 main.log.error( repr( ONOSFlows[ i ] ) )
1477 ONOSFlowsJson.append( None )
1478 flowsResults = False
1479 utilities.assert_equals(
1480 expect=True,
1481 actual=flowsResults,
1482 onpass="No error in reading flows output",
1483 onfail="Error in reading flows from ONOS" )
1484
1485 main.step( "Check for consistency in Flows from each controller" )
1486 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1487 if all( tmp ):
1488 main.log.info( "Flow count is consistent across all ONOS nodes" )
1489 else:
1490 consistentFlows = False
1491 utilities.assert_equals(
1492 expect=True,
1493 actual=consistentFlows,
1494 onpass="The flow count is consistent across all ONOS nodes",
1495 onfail="ONOS nodes have different flow counts" )
1496
1497 if flowsResults and not consistentFlows:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001498 for i in range( len( ONOSFlows ) ):
1499 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001500 try:
1501 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001502 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001503 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1504 indent=4, separators=( ',', ': ' ) ) )
1505 except ( ValueError, TypeError ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001506 main.log.warn( "ONOS" + node + " flows: " +
1507 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001508 elif flowsResults and consistentFlows:
1509 flowCheck = main.TRUE
1510 flowState = ONOSFlows[ 0 ]
1511
1512 main.step( "Get the OF Table entries" )
1513 global flows
1514 flows = []
1515 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001516 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001517 if flowCheck == main.FALSE:
1518 for table in flows:
1519 main.log.warn( table )
1520 # TODO: Compare switch flow tables with ONOS flow tables
1521
1522 main.step( "Start continuous pings" )
1523 main.Mininet2.pingLong(
1524 src=main.params[ 'PING' ][ 'source1' ],
1525 target=main.params[ 'PING' ][ 'target1' ],
1526 pingTime=500 )
1527 main.Mininet2.pingLong(
1528 src=main.params[ 'PING' ][ 'source2' ],
1529 target=main.params[ 'PING' ][ 'target2' ],
1530 pingTime=500 )
1531 main.Mininet2.pingLong(
1532 src=main.params[ 'PING' ][ 'source3' ],
1533 target=main.params[ 'PING' ][ 'target3' ],
1534 pingTime=500 )
1535 main.Mininet2.pingLong(
1536 src=main.params[ 'PING' ][ 'source4' ],
1537 target=main.params[ 'PING' ][ 'target4' ],
1538 pingTime=500 )
1539 main.Mininet2.pingLong(
1540 src=main.params[ 'PING' ][ 'source5' ],
1541 target=main.params[ 'PING' ][ 'target5' ],
1542 pingTime=500 )
1543 main.Mininet2.pingLong(
1544 src=main.params[ 'PING' ][ 'source6' ],
1545 target=main.params[ 'PING' ][ 'target6' ],
1546 pingTime=500 )
1547 main.Mininet2.pingLong(
1548 src=main.params[ 'PING' ][ 'source7' ],
1549 target=main.params[ 'PING' ][ 'target7' ],
1550 pingTime=500 )
1551 main.Mininet2.pingLong(
1552 src=main.params[ 'PING' ][ 'source8' ],
1553 target=main.params[ 'PING' ][ 'target8' ],
1554 pingTime=500 )
1555 main.Mininet2.pingLong(
1556 src=main.params[ 'PING' ][ 'source9' ],
1557 target=main.params[ 'PING' ][ 'target9' ],
1558 pingTime=500 )
1559 main.Mininet2.pingLong(
1560 src=main.params[ 'PING' ][ 'source10' ],
1561 target=main.params[ 'PING' ][ 'target10' ],
1562 pingTime=500 )
1563
1564 main.step( "Collecting topology information from ONOS" )
1565 devices = []
1566 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001567 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001568 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001569 name="devices-" + str( i ),
1570 args=[ ] )
1571 threads.append( t )
1572 t.start()
1573
1574 for t in threads:
1575 t.join()
1576 devices.append( t.result )
1577 hosts = []
1578 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001579 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001580 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001581 name="hosts-" + str( i ),
1582 args=[ ] )
1583 threads.append( t )
1584 t.start()
1585
1586 for t in threads:
1587 t.join()
1588 try:
1589 hosts.append( json.loads( t.result ) )
1590 except ( ValueError, TypeError ):
1591 # FIXME: better handling of this, print which node
1592 # Maybe use thread name?
1593 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001594 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001595 hosts.append( None )
1596
1597 ports = []
1598 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001599 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001600 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001601 name="ports-" + str( i ),
1602 args=[ ] )
1603 threads.append( t )
1604 t.start()
1605
1606 for t in threads:
1607 t.join()
1608 ports.append( t.result )
1609 links = []
1610 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001611 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001612 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001613 name="links-" + str( i ),
1614 args=[ ] )
1615 threads.append( t )
1616 t.start()
1617
1618 for t in threads:
1619 t.join()
1620 links.append( t.result )
1621 clusters = []
1622 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001623 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001624 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001625 name="clusters-" + str( i ),
1626 args=[ ] )
1627 threads.append( t )
1628 t.start()
1629
1630 for t in threads:
1631 t.join()
1632 clusters.append( t.result )
1633 # Compare json objects for hosts and dataplane clusters
1634
1635 # hosts
1636 main.step( "Host view is consistent across ONOS nodes" )
1637 consistentHostsResult = main.TRUE
1638 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001639 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001640 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001641 if hosts[ controller ] == hosts[ 0 ]:
1642 continue
1643 else: # hosts not consistent
1644 main.log.error( "hosts from ONOS" +
1645 controllerStr +
1646 " is inconsistent with ONOS1" )
1647 main.log.warn( repr( hosts[ controller ] ) )
1648 consistentHostsResult = main.FALSE
1649
1650 else:
1651 main.log.error( "Error in getting ONOS hosts from ONOS" +
1652 controllerStr )
1653 consistentHostsResult = main.FALSE
1654 main.log.warn( "ONOS" + controllerStr +
1655 " hosts response: " +
1656 repr( hosts[ controller ] ) )
1657 utilities.assert_equals(
1658 expect=main.TRUE,
1659 actual=consistentHostsResult,
1660 onpass="Hosts view is consistent across all ONOS nodes",
1661 onfail="ONOS nodes have different views of hosts" )
1662
1663 main.step( "Each host has an IP address" )
1664 ipResult = main.TRUE
1665 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001666 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001667 if hosts[ controller ]:
1668 for host in hosts[ controller ]:
1669 if not host.get( 'ipAddresses', [ ] ):
1670 main.log.error( "Error with host ips on controller" +
1671 controllerStr + ": " + str( host ) )
1672 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001673 utilities.assert_equals(
1674 expect=main.TRUE,
1675 actual=ipResult,
1676 onpass="The ips of the hosts aren't empty",
1677 onfail="The ip of at least one host is missing" )
1678
1679 # Strongly connected clusters of devices
1680 main.step( "Cluster view is consistent across ONOS nodes" )
1681 consistentClustersResult = main.TRUE
1682 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001683 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001684 if "Error" not in clusters[ controller ]:
1685 if clusters[ controller ] == clusters[ 0 ]:
1686 continue
1687 else: # clusters not consistent
1688 main.log.error( "clusters from ONOS" + controllerStr +
1689 " is inconsistent with ONOS1" )
1690 consistentClustersResult = main.FALSE
1691
1692 else:
1693 main.log.error( "Error in getting dataplane clusters " +
1694 "from ONOS" + controllerStr )
1695 consistentClustersResult = main.FALSE
1696 main.log.warn( "ONOS" + controllerStr +
1697 " clusters response: " +
1698 repr( clusters[ controller ] ) )
1699 utilities.assert_equals(
1700 expect=main.TRUE,
1701 actual=consistentClustersResult,
1702 onpass="Clusters view is consistent across all ONOS nodes",
1703 onfail="ONOS nodes have different views of clusters" )
Jon Hall172b7ba2016-04-07 18:12:20 -07001704 if consistentClustersResult != main.TRUE:
1705 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07001706 # there should always only be one cluster
1707 main.step( "Cluster view correct across ONOS nodes" )
1708 try:
1709 numClusters = len( json.loads( clusters[ 0 ] ) )
1710 except ( ValueError, TypeError ):
1711 main.log.exception( "Error parsing clusters[0]: " +
1712 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001713 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001714 clusterResults = main.FALSE
1715 if numClusters == 1:
1716 clusterResults = main.TRUE
1717 utilities.assert_equals(
1718 expect=1,
1719 actual=numClusters,
1720 onpass="ONOS shows 1 SCC",
1721 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1722
1723 main.step( "Comparing ONOS topology to MN" )
1724 devicesResults = main.TRUE
1725 linksResults = main.TRUE
1726 hostsResults = main.TRUE
1727 mnSwitches = main.Mininet1.getSwitches()
1728 mnLinks = main.Mininet1.getLinks()
1729 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001730 for controller in main.activeNodes:
1731 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001732 if devices[ controller ] and ports[ controller ] and\
1733 "Error" not in devices[ controller ] and\
1734 "Error" not in ports[ controller ]:
Jon Hall6e709752016-02-01 13:38:46 -08001735 currentDevicesResult = main.Mininet1.compareSwitches(
1736 mnSwitches,
1737 json.loads( devices[ controller ] ),
1738 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001739 else:
1740 currentDevicesResult = main.FALSE
1741 utilities.assert_equals( expect=main.TRUE,
1742 actual=currentDevicesResult,
1743 onpass="ONOS" + controllerStr +
1744 " Switches view is correct",
1745 onfail="ONOS" + controllerStr +
1746 " Switches view is incorrect" )
1747 if links[ controller ] and "Error" not in links[ controller ]:
1748 currentLinksResult = main.Mininet1.compareLinks(
1749 mnSwitches, mnLinks,
1750 json.loads( links[ controller ] ) )
1751 else:
1752 currentLinksResult = main.FALSE
1753 utilities.assert_equals( expect=main.TRUE,
1754 actual=currentLinksResult,
1755 onpass="ONOS" + controllerStr +
1756 " links view is correct",
1757 onfail="ONOS" + controllerStr +
1758 " links view is incorrect" )
1759
Jon Hall657cdf62015-12-17 14:40:51 -08001760 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001761 currentHostsResult = main.Mininet1.compareHosts(
1762 mnHosts,
1763 hosts[ controller ] )
1764 else:
1765 currentHostsResult = main.FALSE
1766 utilities.assert_equals( expect=main.TRUE,
1767 actual=currentHostsResult,
1768 onpass="ONOS" + controllerStr +
1769 " hosts exist in Mininet",
1770 onfail="ONOS" + controllerStr +
1771 " hosts don't match Mininet" )
1772
1773 devicesResults = devicesResults and currentDevicesResult
1774 linksResults = linksResults and currentLinksResult
1775 hostsResults = hostsResults and currentHostsResult
1776
1777 main.step( "Device information is correct" )
1778 utilities.assert_equals(
1779 expect=main.TRUE,
1780 actual=devicesResults,
1781 onpass="Device information is correct",
1782 onfail="Device information is incorrect" )
1783
1784 main.step( "Links are correct" )
1785 utilities.assert_equals(
1786 expect=main.TRUE,
1787 actual=linksResults,
1788 onpass="Link are correct",
1789 onfail="Links are incorrect" )
1790
1791 main.step( "Hosts are correct" )
1792 utilities.assert_equals(
1793 expect=main.TRUE,
1794 actual=hostsResults,
1795 onpass="Hosts are correct",
1796 onfail="Hosts are incorrect" )
1797
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001798 def CASE61( self, main ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001799 """
1800 The Failure case.
1801 """
Jon Halle1a3b752015-07-22 13:02:46 -07001802 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001803 assert main, "main not defined"
1804 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001805 assert main.CLIs, "main.CLIs not defined"
1806 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001807 main.case( "Kill minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -07001808
1809 main.step( "Checking ONOS Logs for errors" )
1810 for node in main.nodes:
1811 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1812 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1813
Jon Hall3b489db2015-10-05 14:38:37 -07001814 n = len( main.nodes ) # Number of nodes
1815 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
1816 main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
1817 if n > 3:
1818 main.kill.append( p - 1 )
1819 # NOTE: This only works for cluster sizes of 3,5, or 7.
1820
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001821 main.step( "Kill " + str( len( main.kill ) ) + " ONOS nodes" )
Jon Hall3b489db2015-10-05 14:38:37 -07001822 killResults = main.TRUE
1823 for i in main.kill:
1824 killResults = killResults and\
1825 main.ONOSbench.onosKill( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001826 main.activeNodes.remove( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001827 utilities.assert_equals( expect=main.TRUE, actual=killResults,
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001828 onpass="ONOS nodes killed successfully",
1829 onfail="ONOS nodes NOT successfully killed" )
1830
1831 def CASE62( self, main ):
1832 """
1833 The bring up stopped nodes
1834 """
1835 import time
1836 assert main.numCtrls, "main.numCtrls not defined"
1837 assert main, "main not defined"
1838 assert utilities.assert_equals, "utilities.assert_equals not defined"
1839 assert main.CLIs, "main.CLIs not defined"
1840 assert main.nodes, "main.nodes not defined"
1841 assert main.kill, "main.kill not defined"
1842 main.case( "Restart minority of ONOS nodes" )
1843
1844 main.step( "Restarting " + str( len( main.kill ) ) + " ONOS nodes" )
1845 startResults = main.TRUE
1846 restartTime = time.time()
1847 for i in main.kill:
1848 startResults = startResults and\
1849 main.ONOSbench.onosStart( main.nodes[i].ip_address )
1850 utilities.assert_equals( expect=main.TRUE, actual=startResults,
1851 onpass="ONOS nodes started successfully",
1852 onfail="ONOS nodes NOT successfully started" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001853
1854 main.step( "Checking if ONOS is up yet" )
1855 count = 0
1856 onosIsupResult = main.FALSE
1857 while onosIsupResult == main.FALSE and count < 10:
Jon Hall3b489db2015-10-05 14:38:37 -07001858 onosIsupResult = main.TRUE
1859 for i in main.kill:
1860 onosIsupResult = onosIsupResult and\
1861 main.ONOSbench.isup( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001862 count = count + 1
Jon Hall5cf14d52015-07-16 12:15:19 -07001863 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1864 onpass="ONOS restarted successfully",
1865 onfail="ONOS restart NOT successful" )
1866
Jon Halle1a3b752015-07-22 13:02:46 -07001867 main.step( "Restarting ONOS main.CLIs" )
Jon Hall3b489db2015-10-05 14:38:37 -07001868 cliResults = main.TRUE
1869 for i in main.kill:
1870 cliResults = cliResults and\
1871 main.CLIs[i].startOnosCli( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001872 main.activeNodes.append( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001873 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1874 onpass="ONOS cli restarted",
1875 onfail="ONOS cli did not restart" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001876 main.activeNodes.sort()
1877 try:
1878 assert list( set( main.activeNodes ) ) == main.activeNodes,\
1879 "List of active nodes has duplicates, this likely indicates something was run out of order"
1880 except AssertionError:
1881 main.log.exception( "" )
1882 main.cleanup()
1883 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -07001884
1885 # Grab the time of restart so we chan check how long the gossip
1886 # protocol has had time to work
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001887 main.restartTime = time.time() - restartTime
Jon Hall5cf14d52015-07-16 12:15:19 -07001888 main.log.debug( "Restart time: " + str( main.restartTime ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001889 # TODO: MAke this configurable. Also, we are breaking the above timer
1890 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001891 node = main.activeNodes[0]
1892 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1893 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1894 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001895
Jon Halla440e872016-03-31 15:15:50 -07001896 main.step( "Rerun for election on the node(s) that were killed" )
1897 runResults = main.TRUE
1898 for i in main.kill:
1899 runResults = runResults and\
1900 main.CLIs[i].electionTestRun()
1901 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1902 onpass="ONOS nodes reran for election topic",
1903 onfail="Errror rerunning for election" )
1904
Jon Hall5cf14d52015-07-16 12:15:19 -07001905 def CASE7( self, main ):
1906 """
1907 Check state after ONOS failure
1908 """
1909 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001910 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001911 assert main, "main not defined"
1912 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001913 assert main.CLIs, "main.CLIs not defined"
1914 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001915 try:
1916 main.kill
1917 except AttributeError:
1918 main.kill = []
1919
Jon Hall5cf14d52015-07-16 12:15:19 -07001920 main.case( "Running ONOS Constant State Tests" )
1921
1922 main.step( "Check that each switch has a master" )
1923 # Assert that each device has a master
1924 rolesNotNull = main.TRUE
1925 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001926 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001927 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001928 name="rolesNotNull-" + str( i ),
1929 args=[ ] )
1930 threads.append( t )
1931 t.start()
1932
1933 for t in threads:
1934 t.join()
1935 rolesNotNull = rolesNotNull and t.result
1936 utilities.assert_equals(
1937 expect=main.TRUE,
1938 actual=rolesNotNull,
1939 onpass="Each device has a master",
1940 onfail="Some devices don't have a master assigned" )
1941
1942 main.step( "Read device roles from ONOS" )
1943 ONOSMastership = []
Jon Halla440e872016-03-31 15:15:50 -07001944 mastershipCheck = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001945 consistentMastership = True
1946 rolesResults = True
1947 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001948 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001949 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001950 name="roles-" + str( i ),
1951 args=[] )
1952 threads.append( t )
1953 t.start()
1954
1955 for t in threads:
1956 t.join()
1957 ONOSMastership.append( t.result )
1958
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001959 for i in range( len( ONOSMastership ) ):
1960 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001961 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001962 main.log.error( "Error in getting ONOS" + node + " roles" )
1963 main.log.warn( "ONOS" + node + " mastership response: " +
1964 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001965 rolesResults = False
1966 utilities.assert_equals(
1967 expect=True,
1968 actual=rolesResults,
1969 onpass="No error in reading roles output",
1970 onfail="Error in reading roles from ONOS" )
1971
1972 main.step( "Check for consistency in roles from each controller" )
1973 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1974 main.log.info(
1975 "Switch roles are consistent across all ONOS nodes" )
1976 else:
1977 consistentMastership = False
1978 utilities.assert_equals(
1979 expect=True,
1980 actual=consistentMastership,
1981 onpass="Switch roles are consistent across all ONOS nodes",
1982 onfail="ONOS nodes have different views of switch roles" )
1983
1984 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001985 for i in range( len( ONOSMastership ) ):
1986 node = str( main.activeNodes[i] + 1 )
1987 main.log.warn( "ONOS" + node + " roles: ",
1988 json.dumps( json.loads( ONOSMastership[ i ] ),
1989 sort_keys=True,
1990 indent=4,
1991 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001992 elif rolesResults and consistentMastership:
1993 mastershipCheck = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07001994
1995 # NOTE: we expect mastership to change on controller failure
Jon Hall5cf14d52015-07-16 12:15:19 -07001996
1997 main.step( "Get the intents and compare across all nodes" )
1998 ONOSIntents = []
1999 intentCheck = main.FALSE
2000 consistentIntents = True
2001 intentsResults = True
2002 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002003 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002004 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07002005 name="intents-" + str( i ),
2006 args=[],
2007 kwargs={ 'jsonFormat': True } )
2008 threads.append( t )
2009 t.start()
2010
2011 for t in threads:
2012 t.join()
2013 ONOSIntents.append( t.result )
2014
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002015 for i in range( len( ONOSIntents) ):
2016 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002017 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002018 main.log.error( "Error in getting ONOS" + node + " intents" )
2019 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002020 repr( ONOSIntents[ i ] ) )
2021 intentsResults = False
2022 utilities.assert_equals(
2023 expect=True,
2024 actual=intentsResults,
2025 onpass="No error in reading intents output",
2026 onfail="Error in reading intents from ONOS" )
2027
2028 main.step( "Check for consistency in Intents from each controller" )
2029 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2030 main.log.info( "Intents are consistent across all ONOS " +
2031 "nodes" )
2032 else:
2033 consistentIntents = False
2034
2035 # Try to make it easy to figure out what is happening
2036 #
2037 # Intent ONOS1 ONOS2 ...
2038 # 0x01 INSTALLED INSTALLING
2039 # ... ... ...
2040 # ... ... ...
2041 title = " ID"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002042 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002043 title += " " * 10 + "ONOS" + str( n + 1 )
2044 main.log.warn( title )
2045 # get all intent keys in the cluster
2046 keys = []
2047 for nodeStr in ONOSIntents:
2048 node = json.loads( nodeStr )
2049 for intent in node:
2050 keys.append( intent.get( 'id' ) )
2051 keys = set( keys )
2052 for key in keys:
2053 row = "%-13s" % key
2054 for nodeStr in ONOSIntents:
2055 node = json.loads( nodeStr )
2056 for intent in node:
2057 if intent.get( 'id' ) == key:
2058 row += "%-15s" % intent.get( 'state' )
2059 main.log.warn( row )
2060 # End table view
2061
2062 utilities.assert_equals(
2063 expect=True,
2064 actual=consistentIntents,
2065 onpass="Intents are consistent across all ONOS nodes",
2066 onfail="ONOS nodes have different views of intents" )
2067 intentStates = []
2068 for node in ONOSIntents: # Iter through ONOS nodes
2069 nodeStates = []
2070 # Iter through intents of a node
2071 try:
2072 for intent in json.loads( node ):
2073 nodeStates.append( intent[ 'state' ] )
2074 except ( ValueError, TypeError ):
2075 main.log.exception( "Error in parsing intents" )
2076 main.log.error( repr( node ) )
2077 intentStates.append( nodeStates )
2078 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2079 main.log.info( dict( out ) )
2080
2081 if intentsResults and not consistentIntents:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002082 for i in range( len( main.activeNodes ) ):
2083 node = str( main.activeNodes[i] + 1 )
2084 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002085 main.log.warn( json.dumps(
2086 json.loads( ONOSIntents[ i ] ),
2087 sort_keys=True,
2088 indent=4,
2089 separators=( ',', ': ' ) ) )
2090 elif intentsResults and consistentIntents:
2091 intentCheck = main.TRUE
2092
2093 # NOTE: Store has no durability, so intents are lost across system
2094 # restarts
2095 main.step( "Compare current intents with intents before the failure" )
2096 # NOTE: this requires case 5 to pass for intentState to be set.
2097 # maybe we should stop the test if that fails?
2098 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002099 try:
2100 intentState
2101 except NameError:
2102 main.log.warn( "No previous intent state was saved" )
2103 else:
2104 if intentState and intentState == ONOSIntents[ 0 ]:
2105 sameIntents = main.TRUE
2106 main.log.info( "Intents are consistent with before failure" )
2107 # TODO: possibly the states have changed? we may need to figure out
2108 # what the acceptable states are
2109 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2110 sameIntents = main.TRUE
2111 try:
2112 before = json.loads( intentState )
2113 after = json.loads( ONOSIntents[ 0 ] )
2114 for intent in before:
2115 if intent not in after:
2116 sameIntents = main.FALSE
2117 main.log.debug( "Intent is not currently in ONOS " +
2118 "(at least in the same form):" )
2119 main.log.debug( json.dumps( intent ) )
2120 except ( ValueError, TypeError ):
2121 main.log.exception( "Exception printing intents" )
2122 main.log.debug( repr( ONOSIntents[0] ) )
2123 main.log.debug( repr( intentState ) )
2124 if sameIntents == main.FALSE:
2125 try:
2126 main.log.debug( "ONOS intents before: " )
2127 main.log.debug( json.dumps( json.loads( intentState ),
2128 sort_keys=True, indent=4,
2129 separators=( ',', ': ' ) ) )
2130 main.log.debug( "Current ONOS intents: " )
2131 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2132 sort_keys=True, indent=4,
2133 separators=( ',', ': ' ) ) )
2134 except ( ValueError, TypeError ):
2135 main.log.exception( "Exception printing intents" )
2136 main.log.debug( repr( ONOSIntents[0] ) )
2137 main.log.debug( repr( intentState ) )
2138 utilities.assert_equals(
2139 expect=main.TRUE,
2140 actual=sameIntents,
2141 onpass="Intents are consistent with before failure",
2142 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002143 intentCheck = intentCheck and sameIntents
2144
2145 main.step( "Get the OF Table entries and compare to before " +
2146 "component failure" )
2147 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002148 for i in range( 28 ):
2149 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002150 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002151 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2152 FlowTables = FlowTables and curSwitch
2153 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002154 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002155 utilities.assert_equals(
2156 expect=main.TRUE,
2157 actual=FlowTables,
2158 onpass="No changes were found in the flow tables",
2159 onfail="Changes were found in the flow tables" )
2160
2161 main.Mininet2.pingLongKill()
2162 '''
2163 main.step( "Check the continuous pings to ensure that no packets " +
2164 "were dropped during component failure" )
2165 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2166 main.params[ 'TESTONIP' ] )
2167 LossInPings = main.FALSE
2168 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2169 for i in range( 8, 18 ):
2170 main.log.info(
2171 "Checking for a loss in pings along flow from s" +
2172 str( i ) )
2173 LossInPings = main.Mininet2.checkForLoss(
2174 "/tmp/ping.h" +
2175 str( i ) ) or LossInPings
2176 if LossInPings == main.TRUE:
2177 main.log.info( "Loss in ping detected" )
2178 elif LossInPings == main.ERROR:
2179 main.log.info( "There are multiple mininet process running" )
2180 elif LossInPings == main.FALSE:
2181 main.log.info( "No Loss in the pings" )
2182 main.log.info( "No loss of dataplane connectivity" )
2183 utilities.assert_equals(
2184 expect=main.FALSE,
2185 actual=LossInPings,
2186 onpass="No Loss of connectivity",
2187 onfail="Loss of dataplane connectivity detected" )
2188 '''
2189
2190 main.step( "Leadership Election is still functional" )
2191 # Test of LeadershipElection
2192 leaderList = []
Jon Hall5cf14d52015-07-16 12:15:19 -07002193
Jon Hall3b489db2015-10-05 14:38:37 -07002194 restarted = []
2195 for i in main.kill:
2196 restarted.append( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002197 leaderResult = main.TRUE
Jon Hall3b489db2015-10-05 14:38:37 -07002198
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002199 for i in main.activeNodes:
2200 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002201 leaderN = cli.electionTestLeader()
2202 leaderList.append( leaderN )
2203 if leaderN == main.FALSE:
2204 # error in response
2205 main.log.error( "Something is wrong with " +
2206 "electionTestLeader function, check the" +
2207 " error logs" )
2208 leaderResult = main.FALSE
2209 elif leaderN is None:
2210 main.log.error( cli.name +
2211 " shows no leader for the election-app was" +
2212 " elected after the old one died" )
2213 leaderResult = main.FALSE
2214 elif leaderN in restarted:
2215 main.log.error( cli.name + " shows " + str( leaderN ) +
2216 " as leader for the election-app, but it " +
2217 "was restarted" )
2218 leaderResult = main.FALSE
2219 if len( set( leaderList ) ) != 1:
2220 leaderResult = main.FALSE
2221 main.log.error(
2222 "Inconsistent view of leader for the election test app" )
2223 # TODO: print the list
2224 utilities.assert_equals(
2225 expect=main.TRUE,
2226 actual=leaderResult,
2227 onpass="Leadership election passed",
2228 onfail="Something went wrong with Leadership election" )
2229
2230 def CASE8( self, main ):
2231 """
2232 Compare topo
2233 """
2234 import json
2235 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002236 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002237 assert main, "main not defined"
2238 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002239 assert main.CLIs, "main.CLIs not defined"
2240 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002241
2242 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002243 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002244 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002245 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002246 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002247 elapsed = 0
2248 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002249 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002250 startTime = time.time()
2251 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002252 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002253 devicesResults = main.TRUE
2254 linksResults = main.TRUE
2255 hostsResults = main.TRUE
2256 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002257 count += 1
2258 cliStart = time.time()
2259 devices = []
2260 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002261 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002262 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002263 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002264 args=[ main.CLIs[i].devices, [ None ] ],
2265 kwargs= { 'sleep': 5, 'attempts': 5,
2266 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002267 threads.append( t )
2268 t.start()
2269
2270 for t in threads:
2271 t.join()
2272 devices.append( t.result )
2273 hosts = []
2274 ipResult = main.TRUE
2275 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002276 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002277 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002278 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002279 args=[ main.CLIs[i].hosts, [ None ] ],
2280 kwargs= { 'sleep': 5, 'attempts': 5,
2281 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002282 threads.append( t )
2283 t.start()
2284
2285 for t in threads:
2286 t.join()
2287 try:
2288 hosts.append( json.loads( t.result ) )
2289 except ( ValueError, TypeError ):
2290 main.log.exception( "Error parsing hosts results" )
2291 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002292 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002293 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002294 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002295 if hosts[ controller ]:
2296 for host in hosts[ controller ]:
2297 if host is None or host.get( 'ipAddresses', [] ) == []:
2298 main.log.error(
2299 "Error with host ipAddresses on controller" +
2300 controllerStr + ": " + str( host ) )
2301 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002302 ports = []
2303 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002304 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002305 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002306 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002307 args=[ main.CLIs[i].ports, [ None ] ],
2308 kwargs= { 'sleep': 5, 'attempts': 5,
2309 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002310 threads.append( t )
2311 t.start()
2312
2313 for t in threads:
2314 t.join()
2315 ports.append( t.result )
2316 links = []
2317 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002318 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002319 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002320 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002321 args=[ main.CLIs[i].links, [ None ] ],
2322 kwargs= { 'sleep': 5, 'attempts': 5,
2323 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002324 threads.append( t )
2325 t.start()
2326
2327 for t in threads:
2328 t.join()
2329 links.append( t.result )
2330 clusters = []
2331 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002332 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002333 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002334 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002335 args=[ main.CLIs[i].clusters, [ None ] ],
2336 kwargs= { 'sleep': 5, 'attempts': 5,
2337 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002338 threads.append( t )
2339 t.start()
2340
2341 for t in threads:
2342 t.join()
2343 clusters.append( t.result )
2344
2345 elapsed = time.time() - startTime
2346 cliTime = time.time() - cliStart
2347 print "Elapsed time: " + str( elapsed )
2348 print "CLI time: " + str( cliTime )
2349
Jon Hall6e709752016-02-01 13:38:46 -08002350 if all( e is None for e in devices ) and\
2351 all( e is None for e in hosts ) and\
2352 all( e is None for e in ports ) and\
2353 all( e is None for e in links ) and\
2354 all( e is None for e in clusters ):
2355 topoFailMsg = "Could not get topology from ONOS"
2356 main.log.error( topoFailMsg )
2357 continue # Try again, No use trying to compare
2358
Jon Hall5cf14d52015-07-16 12:15:19 -07002359 mnSwitches = main.Mininet1.getSwitches()
2360 mnLinks = main.Mininet1.getLinks()
2361 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002362 for controller in range( len( main.activeNodes ) ):
2363 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002364 if devices[ controller ] and ports[ controller ] and\
2365 "Error" not in devices[ controller ] and\
2366 "Error" not in ports[ controller ]:
2367
Jon Hallc6793552016-01-19 14:18:37 -08002368 try:
2369 currentDevicesResult = main.Mininet1.compareSwitches(
2370 mnSwitches,
2371 json.loads( devices[ controller ] ),
2372 json.loads( ports[ controller ] ) )
2373 except ( TypeError, ValueError ) as e:
2374 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2375 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002376 else:
2377 currentDevicesResult = main.FALSE
2378 utilities.assert_equals( expect=main.TRUE,
2379 actual=currentDevicesResult,
2380 onpass="ONOS" + controllerStr +
2381 " Switches view is correct",
2382 onfail="ONOS" + controllerStr +
2383 " Switches view is incorrect" )
2384
2385 if links[ controller ] and "Error" not in links[ controller ]:
2386 currentLinksResult = main.Mininet1.compareLinks(
2387 mnSwitches, mnLinks,
2388 json.loads( links[ controller ] ) )
2389 else:
2390 currentLinksResult = main.FALSE
2391 utilities.assert_equals( expect=main.TRUE,
2392 actual=currentLinksResult,
2393 onpass="ONOS" + controllerStr +
2394 " links view is correct",
2395 onfail="ONOS" + controllerStr +
2396 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002397 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002398 currentHostsResult = main.Mininet1.compareHosts(
2399 mnHosts,
2400 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002401 elif hosts[ controller ] == []:
2402 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002403 else:
2404 currentHostsResult = main.FALSE
2405 utilities.assert_equals( expect=main.TRUE,
2406 actual=currentHostsResult,
2407 onpass="ONOS" + controllerStr +
2408 " hosts exist in Mininet",
2409 onfail="ONOS" + controllerStr +
2410 " hosts don't match Mininet" )
2411 # CHECKING HOST ATTACHMENT POINTS
2412 hostAttachment = True
2413 zeroHosts = False
2414 # FIXME: topo-HA/obelisk specific mappings:
2415 # key is mac and value is dpid
2416 mappings = {}
2417 for i in range( 1, 29 ): # hosts 1 through 28
2418 # set up correct variables:
2419 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2420 if i == 1:
2421 deviceId = "1000".zfill(16)
2422 elif i == 2:
2423 deviceId = "2000".zfill(16)
2424 elif i == 3:
2425 deviceId = "3000".zfill(16)
2426 elif i == 4:
2427 deviceId = "3004".zfill(16)
2428 elif i == 5:
2429 deviceId = "5000".zfill(16)
2430 elif i == 6:
2431 deviceId = "6000".zfill(16)
2432 elif i == 7:
2433 deviceId = "6007".zfill(16)
2434 elif i >= 8 and i <= 17:
2435 dpid = '3' + str( i ).zfill( 3 )
2436 deviceId = dpid.zfill(16)
2437 elif i >= 18 and i <= 27:
2438 dpid = '6' + str( i ).zfill( 3 )
2439 deviceId = dpid.zfill(16)
2440 elif i == 28:
2441 deviceId = "2800".zfill(16)
2442 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002443 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002444 if hosts[ controller ] == []:
2445 main.log.warn( "There are no hosts discovered" )
2446 zeroHosts = True
2447 else:
2448 for host in hosts[ controller ]:
2449 mac = None
2450 location = None
2451 device = None
2452 port = None
2453 try:
2454 mac = host.get( 'mac' )
2455 assert mac, "mac field could not be found for this host object"
2456
2457 location = host.get( 'location' )
2458 assert location, "location field could not be found for this host object"
2459
2460 # Trim the protocol identifier off deviceId
2461 device = str( location.get( 'elementId' ) ).split(':')[1]
2462 assert device, "elementId field could not be found for this host location object"
2463
2464 port = location.get( 'port' )
2465 assert port, "port field could not be found for this host location object"
2466
2467 # Now check if this matches where they should be
2468 if mac and device and port:
2469 if str( port ) != "1":
2470 main.log.error( "The attachment port is incorrect for " +
2471 "host " + str( mac ) +
2472 ". Expected: 1 Actual: " + str( port) )
2473 hostAttachment = False
2474 if device != mappings[ str( mac ) ]:
2475 main.log.error( "The attachment device is incorrect for " +
2476 "host " + str( mac ) +
2477 ". Expected: " + mappings[ str( mac ) ] +
2478 " Actual: " + device )
2479 hostAttachment = False
2480 else:
2481 hostAttachment = False
2482 except AssertionError:
2483 main.log.exception( "Json object not as expected" )
2484 main.log.error( repr( host ) )
2485 hostAttachment = False
2486 else:
2487 main.log.error( "No hosts json output or \"Error\"" +
2488 " in output. hosts = " +
2489 repr( hosts[ controller ] ) )
2490 if zeroHosts is False:
2491 hostAttachment = True
2492
2493 # END CHECKING HOST ATTACHMENT POINTS
2494 devicesResults = devicesResults and currentDevicesResult
2495 linksResults = linksResults and currentLinksResult
2496 hostsResults = hostsResults and currentHostsResult
2497 hostAttachmentResults = hostAttachmentResults and\
2498 hostAttachment
Jon Halla440e872016-03-31 15:15:50 -07002499 topoResult = ( devicesResults and linksResults
2500 and hostsResults and ipResult and
2501 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002502 utilities.assert_equals( expect=True,
2503 actual=topoResult,
2504 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002505 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002506 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002507
2508 # Compare json objects for hosts and dataplane clusters
2509
2510 # hosts
2511 main.step( "Hosts view is consistent across all ONOS nodes" )
2512 consistentHostsResult = main.TRUE
2513 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002514 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002515 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002516 if hosts[ controller ] == hosts[ 0 ]:
2517 continue
2518 else: # hosts not consistent
2519 main.log.error( "hosts from ONOS" + controllerStr +
2520 " is inconsistent with ONOS1" )
2521 main.log.warn( repr( hosts[ controller ] ) )
2522 consistentHostsResult = main.FALSE
2523
2524 else:
2525 main.log.error( "Error in getting ONOS hosts from ONOS" +
2526 controllerStr )
2527 consistentHostsResult = main.FALSE
2528 main.log.warn( "ONOS" + controllerStr +
2529 " hosts response: " +
2530 repr( hosts[ controller ] ) )
2531 utilities.assert_equals(
2532 expect=main.TRUE,
2533 actual=consistentHostsResult,
2534 onpass="Hosts view is consistent across all ONOS nodes",
2535 onfail="ONOS nodes have different views of hosts" )
2536
2537 main.step( "Hosts information is correct" )
2538 hostsResults = hostsResults and ipResult
2539 utilities.assert_equals(
2540 expect=main.TRUE,
2541 actual=hostsResults,
2542 onpass="Host information is correct",
2543 onfail="Host information is incorrect" )
2544
2545 main.step( "Host attachment points to the network" )
2546 utilities.assert_equals(
2547 expect=True,
2548 actual=hostAttachmentResults,
2549 onpass="Hosts are correctly attached to the network",
2550 onfail="ONOS did not correctly attach hosts to the network" )
2551
2552 # Strongly connected clusters of devices
2553 main.step( "Clusters view is consistent across all ONOS nodes" )
2554 consistentClustersResult = main.TRUE
2555 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002556 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002557 if "Error" not in clusters[ controller ]:
2558 if clusters[ controller ] == clusters[ 0 ]:
2559 continue
2560 else: # clusters not consistent
2561 main.log.error( "clusters from ONOS" +
2562 controllerStr +
2563 " is inconsistent with ONOS1" )
2564 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002565 else:
2566 main.log.error( "Error in getting dataplane clusters " +
2567 "from ONOS" + controllerStr )
2568 consistentClustersResult = main.FALSE
2569 main.log.warn( "ONOS" + controllerStr +
2570 " clusters response: " +
2571 repr( clusters[ controller ] ) )
2572 utilities.assert_equals(
2573 expect=main.TRUE,
2574 actual=consistentClustersResult,
2575 onpass="Clusters view is consistent across all ONOS nodes",
2576 onfail="ONOS nodes have different views of clusters" )
2577
2578 main.step( "There is only one SCC" )
2579 # there should always only be one cluster
2580 try:
2581 numClusters = len( json.loads( clusters[ 0 ] ) )
2582 except ( ValueError, TypeError ):
2583 main.log.exception( "Error parsing clusters[0]: " +
2584 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002585 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002586 clusterResults = main.FALSE
2587 if numClusters == 1:
2588 clusterResults = main.TRUE
2589 utilities.assert_equals(
2590 expect=1,
2591 actual=numClusters,
2592 onpass="ONOS shows 1 SCC",
2593 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2594
2595 topoResult = ( devicesResults and linksResults
2596 and hostsResults and consistentHostsResult
2597 and consistentClustersResult and clusterResults
2598 and ipResult and hostAttachmentResults )
2599
2600 topoResult = topoResult and int( count <= 2 )
2601 note = "note it takes about " + str( int( cliTime ) ) + \
2602 " seconds for the test to make all the cli calls to fetch " +\
2603 "the topology from each ONOS instance"
2604 main.log.info(
2605 "Very crass estimate for topology discovery/convergence( " +
2606 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2607 str( count ) + " tries" )
2608
2609 main.step( "Device information is correct" )
2610 utilities.assert_equals(
2611 expect=main.TRUE,
2612 actual=devicesResults,
2613 onpass="Device information is correct",
2614 onfail="Device information is incorrect" )
2615
2616 main.step( "Links are correct" )
2617 utilities.assert_equals(
2618 expect=main.TRUE,
2619 actual=linksResults,
2620 onpass="Link are correct",
2621 onfail="Links are incorrect" )
2622
Jon Halla440e872016-03-31 15:15:50 -07002623 main.step( "Hosts are correct" )
2624 utilities.assert_equals(
2625 expect=main.TRUE,
2626 actual=hostsResults,
2627 onpass="Hosts are correct",
2628 onfail="Hosts are incorrect" )
2629
Jon Hall5cf14d52015-07-16 12:15:19 -07002630 # FIXME: move this to an ONOS state case
2631 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002632 nodeResults = utilities.retry( main.HA.nodesCheck,
2633 False,
2634 args=[main.activeNodes],
2635 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002636
Jon Hall41d39f12016-04-11 22:54:35 -07002637 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002638 onpass="Nodes check successful",
2639 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002640 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002641 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002642 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002643 main.CLIs[i].name,
2644 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002645
2646 def CASE9( self, main ):
2647 """
2648 Link s3-s28 down
2649 """
2650 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002651 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002652 assert main, "main not defined"
2653 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002654 assert main.CLIs, "main.CLIs not defined"
2655 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002656 # NOTE: You should probably run a topology check after this
2657
2658 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2659
2660 description = "Turn off a link to ensure that Link Discovery " +\
2661 "is working properly"
2662 main.case( description )
2663
2664 main.step( "Kill Link between s3 and s28" )
2665 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2666 main.log.info( "Waiting " + str( linkSleep ) +
2667 " seconds for link down to be discovered" )
2668 time.sleep( linkSleep )
2669 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2670 onpass="Link down successful",
2671 onfail="Failed to bring link down" )
2672 # TODO do some sort of check here
2673
2674 def CASE10( self, main ):
2675 """
2676 Link s3-s28 up
2677 """
2678 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002679 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002680 assert main, "main not defined"
2681 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002682 assert main.CLIs, "main.CLIs not defined"
2683 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002684 # NOTE: You should probably run a topology check after this
2685
2686 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2687
2688 description = "Restore a link to ensure that Link Discovery is " + \
2689 "working properly"
2690 main.case( description )
2691
2692 main.step( "Bring link between s3 and s28 back up" )
2693 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2694 main.log.info( "Waiting " + str( linkSleep ) +
2695 " seconds for link up to be discovered" )
2696 time.sleep( linkSleep )
2697 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2698 onpass="Link up successful",
2699 onfail="Failed to bring link up" )
2700 # TODO do some sort of check here
2701
2702 def CASE11( self, main ):
2703 """
2704 Switch Down
2705 """
2706 # NOTE: You should probably run a topology check after this
2707 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002708 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002709 assert main, "main not defined"
2710 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002711 assert main.CLIs, "main.CLIs not defined"
2712 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002713
2714 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2715
2716 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002717 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002718 main.case( description )
2719 switch = main.params[ 'kill' ][ 'switch' ]
2720 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2721
2722 # TODO: Make this switch parameterizable
2723 main.step( "Kill " + switch )
2724 main.log.info( "Deleting " + switch )
2725 main.Mininet1.delSwitch( switch )
2726 main.log.info( "Waiting " + str( switchSleep ) +
2727 " seconds for switch down to be discovered" )
2728 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002729 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002730 # Peek at the deleted switch
2731 main.log.warn( str( device ) )
2732 result = main.FALSE
2733 if device and device[ 'available' ] is False:
2734 result = main.TRUE
2735 utilities.assert_equals( expect=main.TRUE, actual=result,
2736 onpass="Kill switch successful",
2737 onfail="Failed to kill switch?" )
2738
2739 def CASE12( self, main ):
2740 """
2741 Switch Up
2742 """
2743 # NOTE: You should probably run a topology check after this
2744 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002745 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002746 assert main, "main not defined"
2747 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002748 assert main.CLIs, "main.CLIs not defined"
2749 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002750 assert ONOS1Port, "ONOS1Port not defined"
2751 assert ONOS2Port, "ONOS2Port not defined"
2752 assert ONOS3Port, "ONOS3Port not defined"
2753 assert ONOS4Port, "ONOS4Port not defined"
2754 assert ONOS5Port, "ONOS5Port not defined"
2755 assert ONOS6Port, "ONOS6Port not defined"
2756 assert ONOS7Port, "ONOS7Port not defined"
2757
2758 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2759 switch = main.params[ 'kill' ][ 'switch' ]
2760 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2761 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002762 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002763 description = "Adding a switch to ensure it is discovered correctly"
2764 main.case( description )
2765
2766 main.step( "Add back " + switch )
2767 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2768 for peer in links:
2769 main.Mininet1.addLink( switch, peer )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002770 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002771 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2772 main.log.info( "Waiting " + str( switchSleep ) +
2773 " seconds for switch up to be discovered" )
2774 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002775 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002776 # Peek at the deleted switch
2777 main.log.warn( str( device ) )
2778 result = main.FALSE
2779 if device and device[ 'available' ]:
2780 result = main.TRUE
2781 utilities.assert_equals( expect=main.TRUE, actual=result,
2782 onpass="add switch successful",
2783 onfail="Failed to add switch?" )
2784
2785 def CASE13( self, main ):
2786 """
2787 Clean up
2788 """
2789 import os
2790 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002791 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002792 assert main, "main not defined"
2793 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002794 assert main.CLIs, "main.CLIs not defined"
2795 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002796
2797 # printing colors to terminal
2798 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2799 'blue': '\033[94m', 'green': '\033[92m',
2800 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2801 main.case( "Test Cleanup" )
2802 main.step( "Killing tcpdumps" )
2803 main.Mininet2.stopTcpdump()
2804
2805 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002806 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002807 main.step( "Copying MN pcap and ONOS log files to test station" )
2808 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2809 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002810 # NOTE: MN Pcap file is being saved to logdir.
2811 # We scp this file as MN and TestON aren't necessarily the same vm
2812
2813 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002814 # TODO: Load these from params
2815 # NOTE: must end in /
2816 logFolder = "/opt/onos/log/"
2817 logFiles = [ "karaf.log", "karaf.log.1" ]
2818 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002819 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002820 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002821 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002822 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2823 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002824 # std*.log's
2825 # NOTE: must end in /
2826 logFolder = "/opt/onos/var/"
2827 logFiles = [ "stderr.log", "stdout.log" ]
2828 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002829 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002830 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002831 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002832 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2833 logFolder + f, dstName )
2834 else:
2835 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002836
2837 main.step( "Stopping Mininet" )
2838 mnResult = main.Mininet1.stopNet()
2839 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2840 onpass="Mininet stopped",
2841 onfail="MN cleanup NOT successful" )
2842
2843 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002844 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002845 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2846 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002847
2848 try:
2849 timerLog = open( main.logdir + "/Timers.csv", 'w')
2850 # Overwrite with empty line and close
2851 labels = "Gossip Intents, Restart"
2852 data = str( gossipTime ) + ", " + str( main.restartTime )
2853 timerLog.write( labels + "\n" + data )
2854 timerLog.close()
2855 except NameError, e:
2856 main.log.exception(e)
2857
2858 def CASE14( self, main ):
2859 """
2860 start election app on all onos nodes
2861 """
Jon Halle1a3b752015-07-22 13:02:46 -07002862 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002863 assert main, "main not defined"
2864 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002865 assert main.CLIs, "main.CLIs not defined"
2866 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002867
2868 main.case("Start Leadership Election app")
2869 main.step( "Install leadership election app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002870 onosCli = main.CLIs[ main.activeNodes[0] ]
2871 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002872 utilities.assert_equals(
2873 expect=main.TRUE,
2874 actual=appResult,
2875 onpass="Election app installed",
2876 onfail="Something went wrong with installing Leadership election" )
2877
2878 main.step( "Run for election on each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002879 for i in main.activeNodes:
2880 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002881 time.sleep(5)
2882 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2883 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002884 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002885 expect=True,
2886 actual=sameResult,
2887 onpass="All nodes see the same leaderboards",
2888 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002889
Jon Hall25463a82016-04-13 14:03:52 -07002890 if sameResult:
2891 leader = leaders[ 0 ][ 0 ]
2892 if main.nodes[main.activeNodes[0]].ip_address in leader:
2893 correctLeader = True
2894 else:
2895 correctLeader = False
2896 main.step( "First node was elected leader" )
2897 utilities.assert_equals(
2898 expect=True,
2899 actual=correctLeader,
2900 onpass="Correct leader was elected",
2901 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002902
2903 def CASE15( self, main ):
2904 """
2905 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002906 15.1 Run election on each node
2907 15.2 Check that each node has the same leaders and candidates
2908 15.3 Find current leader and withdraw
2909 15.4 Check that a new node was elected leader
2910 15.5 Check that that new leader was the candidate of old leader
2911 15.6 Run for election on old leader
2912 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2913 15.8 Make sure that the old leader was added to the candidate list
2914
2915 old and new variable prefixes refer to data from before vs after
2916 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002917 """
2918 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002919 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002920 assert main, "main not defined"
2921 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002922 assert main.CLIs, "main.CLIs not defined"
2923 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002924
Jon Hall5cf14d52015-07-16 12:15:19 -07002925 description = "Check that Leadership Election is still functional"
2926 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002927 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002928
Jon Halla440e872016-03-31 15:15:50 -07002929 oldLeaders = [] # list of lists of each nodes' candidates before
2930 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002931 oldLeader = '' # the old leader from oldLeaders, None if not same
2932 newLeader = '' # the new leaders fron newLoeaders, None if not same
2933 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2934 expectNoLeader = False # True when there is only one leader
2935 if main.numCtrls == 1:
2936 expectNoLeader = True
2937
2938 main.step( "Run for election on each node" )
2939 electionResult = main.TRUE
2940
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002941 for i in main.activeNodes: # run test election on each node
2942 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002943 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002944 utilities.assert_equals(
2945 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002946 actual=electionResult,
2947 onpass="All nodes successfully ran for leadership",
2948 onfail="At least one node failed to run for leadership" )
2949
acsmars3a72bde2015-09-02 14:16:22 -07002950 if electionResult == main.FALSE:
2951 main.log.error(
2952 "Skipping Test Case because Election Test App isn't loaded" )
2953 main.skipCase()
2954
acsmars71adceb2015-08-31 15:09:26 -07002955 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002956 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002957 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002958 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002959 if sameResult:
2960 oldLeader = oldLeaders[ 0 ][ 0 ]
2961 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07002962 else:
Jon Halla440e872016-03-31 15:15:50 -07002963 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002964 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002965 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002966 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002967 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002968 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002969
2970 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002971 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002972 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002973 if oldLeader is None:
2974 main.log.error( "Leadership isn't consistent." )
2975 withdrawResult = main.FALSE
2976 # Get the CLI of the oldLeader
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002977 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002978 if oldLeader == main.nodes[ i ].ip_address:
2979 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002980 break
2981 else: # FOR/ELSE statement
2982 main.log.error( "Leader election, could not find current leader" )
2983 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002984 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002985 utilities.assert_equals(
2986 expect=main.TRUE,
2987 actual=withdrawResult,
2988 onpass="Node was withdrawn from election",
2989 onfail="Node was not withdrawn from election" )
2990
acsmars71adceb2015-08-31 15:09:26 -07002991 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002992 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07002993 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002994 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002995 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002996 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002997 if newLeaders[ 0 ][ 0 ] == 'none':
2998 main.log.error( "No leader was elected on at least 1 node" )
2999 if not expectNoLeader:
3000 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07003001 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07003002
3003 # Check that the new leader is not the older leader, which was withdrawn
3004 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07003005 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08003006 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07003007 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003008 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003009 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003010 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003011 onpass="Leadership election passed",
3012 onfail="Something went wrong with Leadership election" )
3013
Jon Halla440e872016-03-31 15:15:50 -07003014 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08003015 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07003016 correctCandidateResult = main.TRUE
3017 if expectNoLeader:
3018 if newLeader == 'none':
3019 main.log.info( "No leader expected. None found. Pass" )
3020 correctCandidateResult = main.TRUE
3021 else:
3022 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3023 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003024 elif len( oldLeaders[0] ) >= 3:
3025 if newLeader == oldLeaders[ 0 ][ 2 ]:
3026 # correct leader was elected
3027 correctCandidateResult = main.TRUE
3028 else:
3029 correctCandidateResult = main.FALSE
3030 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3031 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003032 else:
3033 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003034 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003035 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07003036 utilities.assert_equals(
3037 expect=main.TRUE,
3038 actual=correctCandidateResult,
3039 onpass="Correct Candidate Elected",
3040 onfail="Incorrect Candidate Elected" )
3041
Jon Hall5cf14d52015-07-16 12:15:19 -07003042 main.step( "Run for election on old leader( just so everyone " +
3043 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07003044 if oldLeaderCLI is not None:
3045 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003046 else:
acsmars71adceb2015-08-31 15:09:26 -07003047 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003048 runResult = main.FALSE
3049 utilities.assert_equals(
3050 expect=main.TRUE,
3051 actual=runResult,
3052 onpass="App re-ran for election",
3053 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003054
acsmars71adceb2015-08-31 15:09:26 -07003055 main.step(
3056 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003057 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003058 # Get new leaders and candidates
3059 reRunLeaders = []
3060 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003061 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003062
3063 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07003064 if not reRunLeaders[0]:
3065 positionResult = main.FALSE
3066 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07003067 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3068 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07003069 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003070 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003071 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003072 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003073 onpass="Old leader successfully re-ran for election",
3074 onfail="Something went wrong with Leadership election after " +
3075 "the old leader re-ran for election" )
3076
3077 def CASE16( self, main ):
3078 """
3079 Install Distributed Primitives app
3080 """
3081 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003082 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003083 assert main, "main not defined"
3084 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003085 assert main.CLIs, "main.CLIs not defined"
3086 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003087
3088 # Variables for the distributed primitives tests
3089 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003090 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003091 global onosSet
3092 global onosSetName
3093 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003094 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003095 onosSet = set([])
3096 onosSetName = "TestON-set"
3097
3098 description = "Install Primitives app"
3099 main.case( description )
3100 main.step( "Install Primitives app" )
3101 appName = "org.onosproject.distributedprimitives"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003102 node = main.activeNodes[0]
3103 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003104 utilities.assert_equals( expect=main.TRUE,
3105 actual=appResults,
3106 onpass="Primitives app activated",
3107 onfail="Primitives app not activated" )
3108 time.sleep( 5 ) # To allow all nodes to activate
3109
3110 def CASE17( self, main ):
3111 """
3112 Check for basic functionality with distributed primitives
3113 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003114 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003115 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003116 assert main, "main not defined"
3117 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003118 assert main.CLIs, "main.CLIs not defined"
3119 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003120 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003121 assert onosSetName, "onosSetName not defined"
3122 # NOTE: assert fails if value is 0/None/Empty/False
3123 try:
3124 pCounterValue
3125 except NameError:
3126 main.log.error( "pCounterValue not defined, setting to 0" )
3127 pCounterValue = 0
3128 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003129 onosSet
3130 except NameError:
3131 main.log.error( "onosSet not defined, setting to empty Set" )
3132 onosSet = set([])
3133 # Variables for the distributed primitives tests. These are local only
3134 addValue = "a"
3135 addAllValue = "a b c d e f"
3136 retainValue = "c d e f"
3137
3138 description = "Check for basic functionality with distributed " +\
3139 "primitives"
3140 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003141 main.caseExplanation = "Test the methods of the distributed " +\
3142 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003143 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003144 # Partitioned counters
3145 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003146 pCounters = []
3147 threads = []
3148 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003149 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003150 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3151 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003152 args=[ pCounterName ] )
3153 pCounterValue += 1
3154 addedPValues.append( pCounterValue )
3155 threads.append( t )
3156 t.start()
3157
3158 for t in threads:
3159 t.join()
3160 pCounters.append( t.result )
3161 # Check that counter incremented numController times
3162 pCounterResults = True
3163 for i in addedPValues:
3164 tmpResult = i in pCounters
3165 pCounterResults = pCounterResults and tmpResult
3166 if not tmpResult:
3167 main.log.error( str( i ) + " is not in partitioned "
3168 "counter incremented results" )
3169 utilities.assert_equals( expect=True,
3170 actual=pCounterResults,
3171 onpass="Default counter incremented",
3172 onfail="Error incrementing default" +
3173 " counter" )
3174
Jon Halle1a3b752015-07-22 13:02:46 -07003175 main.step( "Get then Increment a default counter on each node" )
3176 pCounters = []
3177 threads = []
3178 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003179 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003180 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3181 name="counterGetAndAdd-" + str( i ),
3182 args=[ pCounterName ] )
3183 addedPValues.append( pCounterValue )
3184 pCounterValue += 1
3185 threads.append( t )
3186 t.start()
3187
3188 for t in threads:
3189 t.join()
3190 pCounters.append( t.result )
3191 # Check that counter incremented numController times
3192 pCounterResults = True
3193 for i in addedPValues:
3194 tmpResult = i in pCounters
3195 pCounterResults = pCounterResults and tmpResult
3196 if not tmpResult:
3197 main.log.error( str( i ) + " is not in partitioned "
3198 "counter incremented results" )
3199 utilities.assert_equals( expect=True,
3200 actual=pCounterResults,
3201 onpass="Default counter incremented",
3202 onfail="Error incrementing default" +
3203 " counter" )
3204
3205 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003206 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003207 utilities.assert_equals( expect=main.TRUE,
3208 actual=incrementCheck,
3209 onpass="Added counters are correct",
3210 onfail="Added counters are incorrect" )
3211
3212 main.step( "Add -8 to then get a default counter on each node" )
3213 pCounters = []
3214 threads = []
3215 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003216 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003217 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3218 name="counterIncrement-" + str( i ),
3219 args=[ pCounterName ],
3220 kwargs={ "delta": -8 } )
3221 pCounterValue += -8
3222 addedPValues.append( pCounterValue )
3223 threads.append( t )
3224 t.start()
3225
3226 for t in threads:
3227 t.join()
3228 pCounters.append( t.result )
3229 # Check that counter incremented numController times
3230 pCounterResults = True
3231 for i in addedPValues:
3232 tmpResult = i in pCounters
3233 pCounterResults = pCounterResults and tmpResult
3234 if not tmpResult:
3235 main.log.error( str( i ) + " is not in partitioned "
3236 "counter incremented results" )
3237 utilities.assert_equals( expect=True,
3238 actual=pCounterResults,
3239 onpass="Default counter incremented",
3240 onfail="Error incrementing default" +
3241 " counter" )
3242
3243 main.step( "Add 5 to then get a default counter on each node" )
3244 pCounters = []
3245 threads = []
3246 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003247 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003248 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3249 name="counterIncrement-" + str( i ),
3250 args=[ pCounterName ],
3251 kwargs={ "delta": 5 } )
3252 pCounterValue += 5
3253 addedPValues.append( pCounterValue )
3254 threads.append( t )
3255 t.start()
3256
3257 for t in threads:
3258 t.join()
3259 pCounters.append( t.result )
3260 # Check that counter incremented numController times
3261 pCounterResults = True
3262 for i in addedPValues:
3263 tmpResult = i in pCounters
3264 pCounterResults = pCounterResults and tmpResult
3265 if not tmpResult:
3266 main.log.error( str( i ) + " is not in partitioned "
3267 "counter incremented results" )
3268 utilities.assert_equals( expect=True,
3269 actual=pCounterResults,
3270 onpass="Default counter incremented",
3271 onfail="Error incrementing default" +
3272 " counter" )
3273
3274 main.step( "Get then add 5 to a default counter on each node" )
3275 pCounters = []
3276 threads = []
3277 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003278 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003279 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3280 name="counterIncrement-" + str( i ),
3281 args=[ pCounterName ],
3282 kwargs={ "delta": 5 } )
3283 addedPValues.append( pCounterValue )
3284 pCounterValue += 5
3285 threads.append( t )
3286 t.start()
3287
3288 for t in threads:
3289 t.join()
3290 pCounters.append( t.result )
3291 # Check that counter incremented numController times
3292 pCounterResults = True
3293 for i in addedPValues:
3294 tmpResult = i in pCounters
3295 pCounterResults = pCounterResults and tmpResult
3296 if not tmpResult:
3297 main.log.error( str( i ) + " is not in partitioned "
3298 "counter incremented results" )
3299 utilities.assert_equals( expect=True,
3300 actual=pCounterResults,
3301 onpass="Default counter incremented",
3302 onfail="Error incrementing default" +
3303 " counter" )
3304
3305 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003306 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003307 utilities.assert_equals( expect=main.TRUE,
3308 actual=incrementCheck,
3309 onpass="Added counters are correct",
3310 onfail="Added counters are incorrect" )
3311
Jon Hall5cf14d52015-07-16 12:15:19 -07003312 # DISTRIBUTED SETS
3313 main.step( "Distributed Set get" )
3314 size = len( onosSet )
3315 getResponses = []
3316 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003317 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003318 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003319 name="setTestGet-" + str( i ),
3320 args=[ onosSetName ] )
3321 threads.append( t )
3322 t.start()
3323 for t in threads:
3324 t.join()
3325 getResponses.append( t.result )
3326
3327 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003328 for i in range( len( main.activeNodes ) ):
3329 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003330 if isinstance( getResponses[ i ], list):
3331 current = set( getResponses[ i ] )
3332 if len( current ) == len( getResponses[ i ] ):
3333 # no repeats
3334 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003335 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003336 " has incorrect view" +
3337 " of set " + onosSetName + ":\n" +
3338 str( getResponses[ i ] ) )
3339 main.log.debug( "Expected: " + str( onosSet ) )
3340 main.log.debug( "Actual: " + str( current ) )
3341 getResults = main.FALSE
3342 else:
3343 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003344 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003345 " has repeat elements in" +
3346 " set " + onosSetName + ":\n" +
3347 str( getResponses[ i ] ) )
3348 getResults = main.FALSE
3349 elif getResponses[ i ] == main.ERROR:
3350 getResults = main.FALSE
3351 utilities.assert_equals( expect=main.TRUE,
3352 actual=getResults,
3353 onpass="Set elements are correct",
3354 onfail="Set elements are incorrect" )
3355
3356 main.step( "Distributed Set size" )
3357 sizeResponses = []
3358 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003359 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003360 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003361 name="setTestSize-" + str( i ),
3362 args=[ onosSetName ] )
3363 threads.append( t )
3364 t.start()
3365 for t in threads:
3366 t.join()
3367 sizeResponses.append( t.result )
3368
3369 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003370 for i in range( len( main.activeNodes ) ):
3371 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003372 if size != sizeResponses[ i ]:
3373 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003374 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003375 " expected a size of " + str( size ) +
3376 " for set " + onosSetName +
3377 " but got " + str( sizeResponses[ i ] ) )
3378 utilities.assert_equals( expect=main.TRUE,
3379 actual=sizeResults,
3380 onpass="Set sizes are correct",
3381 onfail="Set sizes are incorrect" )
3382
3383 main.step( "Distributed Set add()" )
3384 onosSet.add( addValue )
3385 addResponses = []
3386 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003387 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003388 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003389 name="setTestAdd-" + str( i ),
3390 args=[ onosSetName, addValue ] )
3391 threads.append( t )
3392 t.start()
3393 for t in threads:
3394 t.join()
3395 addResponses.append( t.result )
3396
3397 # main.TRUE = successfully changed the set
3398 # main.FALSE = action resulted in no change in set
3399 # main.ERROR - Some error in executing the function
3400 addResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003401 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003402 if addResponses[ i ] == main.TRUE:
3403 # All is well
3404 pass
3405 elif addResponses[ i ] == main.FALSE:
3406 # Already in set, probably fine
3407 pass
3408 elif addResponses[ i ] == main.ERROR:
3409 # Error in execution
3410 addResults = main.FALSE
3411 else:
3412 # unexpected result
3413 addResults = main.FALSE
3414 if addResults != main.TRUE:
3415 main.log.error( "Error executing set add" )
3416
3417 # Check if set is still correct
3418 size = len( onosSet )
3419 getResponses = []
3420 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003421 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003422 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003423 name="setTestGet-" + str( i ),
3424 args=[ onosSetName ] )
3425 threads.append( t )
3426 t.start()
3427 for t in threads:
3428 t.join()
3429 getResponses.append( t.result )
3430 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003431 for i in range( len( main.activeNodes ) ):
3432 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003433 if isinstance( getResponses[ i ], list):
3434 current = set( getResponses[ i ] )
3435 if len( current ) == len( getResponses[ i ] ):
3436 # no repeats
3437 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003438 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003439 " of set " + onosSetName + ":\n" +
3440 str( getResponses[ i ] ) )
3441 main.log.debug( "Expected: " + str( onosSet ) )
3442 main.log.debug( "Actual: " + str( current ) )
3443 getResults = main.FALSE
3444 else:
3445 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003446 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003447 " set " + onosSetName + ":\n" +
3448 str( getResponses[ i ] ) )
3449 getResults = main.FALSE
3450 elif getResponses[ i ] == main.ERROR:
3451 getResults = main.FALSE
3452 sizeResponses = []
3453 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003454 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003455 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003456 name="setTestSize-" + str( i ),
3457 args=[ onosSetName ] )
3458 threads.append( t )
3459 t.start()
3460 for t in threads:
3461 t.join()
3462 sizeResponses.append( t.result )
3463 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003464 for i in range( len( main.activeNodes ) ):
3465 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003466 if size != sizeResponses[ i ]:
3467 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003468 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003469 " expected a size of " + str( size ) +
3470 " for set " + onosSetName +
3471 " but got " + str( sizeResponses[ i ] ) )
3472 addResults = addResults and getResults and sizeResults
3473 utilities.assert_equals( expect=main.TRUE,
3474 actual=addResults,
3475 onpass="Set add correct",
3476 onfail="Set add was incorrect" )
3477
3478 main.step( "Distributed Set addAll()" )
3479 onosSet.update( addAllValue.split() )
3480 addResponses = []
3481 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003482 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003483 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003484 name="setTestAddAll-" + str( i ),
3485 args=[ onosSetName, addAllValue ] )
3486 threads.append( t )
3487 t.start()
3488 for t in threads:
3489 t.join()
3490 addResponses.append( t.result )
3491
3492 # main.TRUE = successfully changed the set
3493 # main.FALSE = action resulted in no change in set
3494 # main.ERROR - Some error in executing the function
3495 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003496 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003497 if addResponses[ i ] == main.TRUE:
3498 # All is well
3499 pass
3500 elif addResponses[ i ] == main.FALSE:
3501 # Already in set, probably fine
3502 pass
3503 elif addResponses[ i ] == main.ERROR:
3504 # Error in execution
3505 addAllResults = main.FALSE
3506 else:
3507 # unexpected result
3508 addAllResults = main.FALSE
3509 if addAllResults != main.TRUE:
3510 main.log.error( "Error executing set addAll" )
3511
3512 # Check if set is still correct
3513 size = len( onosSet )
3514 getResponses = []
3515 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003516 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003517 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003518 name="setTestGet-" + str( i ),
3519 args=[ onosSetName ] )
3520 threads.append( t )
3521 t.start()
3522 for t in threads:
3523 t.join()
3524 getResponses.append( t.result )
3525 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003526 for i in range( len( main.activeNodes ) ):
3527 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003528 if isinstance( getResponses[ i ], list):
3529 current = set( getResponses[ i ] )
3530 if len( current ) == len( getResponses[ i ] ):
3531 # no repeats
3532 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003533 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003534 " has incorrect view" +
3535 " of set " + onosSetName + ":\n" +
3536 str( getResponses[ i ] ) )
3537 main.log.debug( "Expected: " + str( onosSet ) )
3538 main.log.debug( "Actual: " + str( current ) )
3539 getResults = main.FALSE
3540 else:
3541 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003542 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003543 " has repeat elements in" +
3544 " set " + onosSetName + ":\n" +
3545 str( getResponses[ i ] ) )
3546 getResults = main.FALSE
3547 elif getResponses[ i ] == main.ERROR:
3548 getResults = main.FALSE
3549 sizeResponses = []
3550 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003551 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003552 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003553 name="setTestSize-" + str( i ),
3554 args=[ onosSetName ] )
3555 threads.append( t )
3556 t.start()
3557 for t in threads:
3558 t.join()
3559 sizeResponses.append( t.result )
3560 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003561 for i in range( len( main.activeNodes ) ):
3562 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003563 if size != sizeResponses[ i ]:
3564 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003565 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003566 " expected a size of " + str( size ) +
3567 " for set " + onosSetName +
3568 " but got " + str( sizeResponses[ i ] ) )
3569 addAllResults = addAllResults and getResults and sizeResults
3570 utilities.assert_equals( expect=main.TRUE,
3571 actual=addAllResults,
3572 onpass="Set addAll correct",
3573 onfail="Set addAll was incorrect" )
3574
3575 main.step( "Distributed Set contains()" )
3576 containsResponses = []
3577 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003578 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003579 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003580 name="setContains-" + str( i ),
3581 args=[ onosSetName ],
3582 kwargs={ "values": addValue } )
3583 threads.append( t )
3584 t.start()
3585 for t in threads:
3586 t.join()
3587 # NOTE: This is the tuple
3588 containsResponses.append( t.result )
3589
3590 containsResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003591 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003592 if containsResponses[ i ] == main.ERROR:
3593 containsResults = main.FALSE
3594 else:
3595 containsResults = containsResults and\
3596 containsResponses[ i ][ 1 ]
3597 utilities.assert_equals( expect=main.TRUE,
3598 actual=containsResults,
3599 onpass="Set contains is functional",
3600 onfail="Set contains failed" )
3601
3602 main.step( "Distributed Set containsAll()" )
3603 containsAllResponses = []
3604 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003605 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003606 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003607 name="setContainsAll-" + str( i ),
3608 args=[ onosSetName ],
3609 kwargs={ "values": addAllValue } )
3610 threads.append( t )
3611 t.start()
3612 for t in threads:
3613 t.join()
3614 # NOTE: This is the tuple
3615 containsAllResponses.append( t.result )
3616
3617 containsAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003618 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003619 if containsResponses[ i ] == main.ERROR:
3620 containsResults = main.FALSE
3621 else:
3622 containsResults = containsResults and\
3623 containsResponses[ i ][ 1 ]
3624 utilities.assert_equals( expect=main.TRUE,
3625 actual=containsAllResults,
3626 onpass="Set containsAll is functional",
3627 onfail="Set containsAll failed" )
3628
3629 main.step( "Distributed Set remove()" )
3630 onosSet.remove( addValue )
3631 removeResponses = []
3632 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003633 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003634 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003635 name="setTestRemove-" + str( i ),
3636 args=[ onosSetName, addValue ] )
3637 threads.append( t )
3638 t.start()
3639 for t in threads:
3640 t.join()
3641 removeResponses.append( t.result )
3642
3643 # main.TRUE = successfully changed the set
3644 # main.FALSE = action resulted in no change in set
3645 # main.ERROR - Some error in executing the function
3646 removeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003647 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003648 if removeResponses[ i ] == main.TRUE:
3649 # All is well
3650 pass
3651 elif removeResponses[ i ] == main.FALSE:
3652 # not in set, probably fine
3653 pass
3654 elif removeResponses[ i ] == main.ERROR:
3655 # Error in execution
3656 removeResults = main.FALSE
3657 else:
3658 # unexpected result
3659 removeResults = main.FALSE
3660 if removeResults != main.TRUE:
3661 main.log.error( "Error executing set remove" )
3662
3663 # Check if set is still correct
3664 size = len( onosSet )
3665 getResponses = []
3666 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003667 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003668 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003669 name="setTestGet-" + str( i ),
3670 args=[ onosSetName ] )
3671 threads.append( t )
3672 t.start()
3673 for t in threads:
3674 t.join()
3675 getResponses.append( t.result )
3676 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003677 for i in range( len( main.activeNodes ) ):
3678 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003679 if isinstance( getResponses[ i ], list):
3680 current = set( getResponses[ i ] )
3681 if len( current ) == len( getResponses[ i ] ):
3682 # no repeats
3683 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003684 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003685 " has incorrect view" +
3686 " of set " + onosSetName + ":\n" +
3687 str( getResponses[ i ] ) )
3688 main.log.debug( "Expected: " + str( onosSet ) )
3689 main.log.debug( "Actual: " + str( current ) )
3690 getResults = main.FALSE
3691 else:
3692 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003693 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003694 " has repeat elements in" +
3695 " set " + onosSetName + ":\n" +
3696 str( getResponses[ i ] ) )
3697 getResults = main.FALSE
3698 elif getResponses[ i ] == main.ERROR:
3699 getResults = main.FALSE
3700 sizeResponses = []
3701 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003702 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003703 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003704 name="setTestSize-" + str( i ),
3705 args=[ onosSetName ] )
3706 threads.append( t )
3707 t.start()
3708 for t in threads:
3709 t.join()
3710 sizeResponses.append( t.result )
3711 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003712 for i in range( len( main.activeNodes ) ):
3713 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003714 if size != sizeResponses[ i ]:
3715 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003716 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003717 " expected a size of " + str( size ) +
3718 " for set " + onosSetName +
3719 " but got " + str( sizeResponses[ i ] ) )
3720 removeResults = removeResults and getResults and sizeResults
3721 utilities.assert_equals( expect=main.TRUE,
3722 actual=removeResults,
3723 onpass="Set remove correct",
3724 onfail="Set remove was incorrect" )
3725
3726 main.step( "Distributed Set removeAll()" )
3727 onosSet.difference_update( addAllValue.split() )
3728 removeAllResponses = []
3729 threads = []
3730 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003731 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003732 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003733 name="setTestRemoveAll-" + str( i ),
3734 args=[ onosSetName, addAllValue ] )
3735 threads.append( t )
3736 t.start()
3737 for t in threads:
3738 t.join()
3739 removeAllResponses.append( t.result )
3740 except Exception, e:
3741 main.log.exception(e)
3742
3743 # main.TRUE = successfully changed the set
3744 # main.FALSE = action resulted in no change in set
3745 # main.ERROR - Some error in executing the function
3746 removeAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003747 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003748 if removeAllResponses[ i ] == main.TRUE:
3749 # All is well
3750 pass
3751 elif removeAllResponses[ i ] == main.FALSE:
3752 # not in set, probably fine
3753 pass
3754 elif removeAllResponses[ i ] == main.ERROR:
3755 # Error in execution
3756 removeAllResults = main.FALSE
3757 else:
3758 # unexpected result
3759 removeAllResults = main.FALSE
3760 if removeAllResults != main.TRUE:
3761 main.log.error( "Error executing set removeAll" )
3762
3763 # Check if set is still correct
3764 size = len( onosSet )
3765 getResponses = []
3766 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003767 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003768 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003769 name="setTestGet-" + str( i ),
3770 args=[ onosSetName ] )
3771 threads.append( t )
3772 t.start()
3773 for t in threads:
3774 t.join()
3775 getResponses.append( t.result )
3776 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003777 for i in range( len( main.activeNodes ) ):
3778 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003779 if isinstance( getResponses[ i ], list):
3780 current = set( getResponses[ i ] )
3781 if len( current ) == len( getResponses[ i ] ):
3782 # no repeats
3783 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003784 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003785 " has incorrect view" +
3786 " of set " + onosSetName + ":\n" +
3787 str( getResponses[ i ] ) )
3788 main.log.debug( "Expected: " + str( onosSet ) )
3789 main.log.debug( "Actual: " + str( current ) )
3790 getResults = main.FALSE
3791 else:
3792 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003793 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003794 " has repeat elements in" +
3795 " set " + onosSetName + ":\n" +
3796 str( getResponses[ i ] ) )
3797 getResults = main.FALSE
3798 elif getResponses[ i ] == main.ERROR:
3799 getResults = main.FALSE
3800 sizeResponses = []
3801 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003802 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003803 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003804 name="setTestSize-" + str( i ),
3805 args=[ onosSetName ] )
3806 threads.append( t )
3807 t.start()
3808 for t in threads:
3809 t.join()
3810 sizeResponses.append( t.result )
3811 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003812 for i in range( len( main.activeNodes ) ):
3813 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003814 if size != sizeResponses[ i ]:
3815 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003816 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003817 " expected a size of " + str( size ) +
3818 " for set " + onosSetName +
3819 " but got " + str( sizeResponses[ i ] ) )
3820 removeAllResults = removeAllResults and getResults and sizeResults
3821 utilities.assert_equals( expect=main.TRUE,
3822 actual=removeAllResults,
3823 onpass="Set removeAll correct",
3824 onfail="Set removeAll was incorrect" )
3825
3826 main.step( "Distributed Set addAll()" )
3827 onosSet.update( addAllValue.split() )
3828 addResponses = []
3829 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003830 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003831 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003832 name="setTestAddAll-" + str( i ),
3833 args=[ onosSetName, addAllValue ] )
3834 threads.append( t )
3835 t.start()
3836 for t in threads:
3837 t.join()
3838 addResponses.append( t.result )
3839
3840 # main.TRUE = successfully changed the set
3841 # main.FALSE = action resulted in no change in set
3842 # main.ERROR - Some error in executing the function
3843 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003844 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003845 if addResponses[ i ] == main.TRUE:
3846 # All is well
3847 pass
3848 elif addResponses[ i ] == main.FALSE:
3849 # Already in set, probably fine
3850 pass
3851 elif addResponses[ i ] == main.ERROR:
3852 # Error in execution
3853 addAllResults = main.FALSE
3854 else:
3855 # unexpected result
3856 addAllResults = main.FALSE
3857 if addAllResults != main.TRUE:
3858 main.log.error( "Error executing set addAll" )
3859
3860 # Check if set is still correct
3861 size = len( onosSet )
3862 getResponses = []
3863 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003864 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003865 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003866 name="setTestGet-" + str( i ),
3867 args=[ onosSetName ] )
3868 threads.append( t )
3869 t.start()
3870 for t in threads:
3871 t.join()
3872 getResponses.append( t.result )
3873 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003874 for i in range( len( main.activeNodes ) ):
3875 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003876 if isinstance( getResponses[ i ], list):
3877 current = set( getResponses[ i ] )
3878 if len( current ) == len( getResponses[ i ] ):
3879 # no repeats
3880 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003881 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003882 " has incorrect view" +
3883 " of set " + onosSetName + ":\n" +
3884 str( getResponses[ i ] ) )
3885 main.log.debug( "Expected: " + str( onosSet ) )
3886 main.log.debug( "Actual: " + str( current ) )
3887 getResults = main.FALSE
3888 else:
3889 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003890 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003891 " has repeat elements in" +
3892 " set " + onosSetName + ":\n" +
3893 str( getResponses[ i ] ) )
3894 getResults = main.FALSE
3895 elif getResponses[ i ] == main.ERROR:
3896 getResults = main.FALSE
3897 sizeResponses = []
3898 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003899 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003900 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003901 name="setTestSize-" + str( i ),
3902 args=[ onosSetName ] )
3903 threads.append( t )
3904 t.start()
3905 for t in threads:
3906 t.join()
3907 sizeResponses.append( t.result )
3908 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003909 for i in range( len( main.activeNodes ) ):
3910 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003911 if size != sizeResponses[ i ]:
3912 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003913 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003914 " expected a size of " + str( size ) +
3915 " for set " + onosSetName +
3916 " but got " + str( sizeResponses[ i ] ) )
3917 addAllResults = addAllResults and getResults and sizeResults
3918 utilities.assert_equals( expect=main.TRUE,
3919 actual=addAllResults,
3920 onpass="Set addAll correct",
3921 onfail="Set addAll was incorrect" )
3922
3923 main.step( "Distributed Set clear()" )
3924 onosSet.clear()
3925 clearResponses = []
3926 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003927 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003928 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003929 name="setTestClear-" + str( i ),
3930 args=[ onosSetName, " "], # Values doesn't matter
3931 kwargs={ "clear": True } )
3932 threads.append( t )
3933 t.start()
3934 for t in threads:
3935 t.join()
3936 clearResponses.append( t.result )
3937
3938 # main.TRUE = successfully changed the set
3939 # main.FALSE = action resulted in no change in set
3940 # main.ERROR - Some error in executing the function
3941 clearResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003942 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003943 if clearResponses[ i ] == main.TRUE:
3944 # All is well
3945 pass
3946 elif clearResponses[ i ] == main.FALSE:
3947 # Nothing set, probably fine
3948 pass
3949 elif clearResponses[ i ] == main.ERROR:
3950 # Error in execution
3951 clearResults = main.FALSE
3952 else:
3953 # unexpected result
3954 clearResults = main.FALSE
3955 if clearResults != main.TRUE:
3956 main.log.error( "Error executing set clear" )
3957
3958 # Check if set is still correct
3959 size = len( onosSet )
3960 getResponses = []
3961 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003962 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003963 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003964 name="setTestGet-" + str( i ),
3965 args=[ onosSetName ] )
3966 threads.append( t )
3967 t.start()
3968 for t in threads:
3969 t.join()
3970 getResponses.append( t.result )
3971 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003972 for i in range( len( main.activeNodes ) ):
3973 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003974 if isinstance( getResponses[ i ], list):
3975 current = set( getResponses[ i ] )
3976 if len( current ) == len( getResponses[ i ] ):
3977 # no repeats
3978 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003979 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003980 " has incorrect view" +
3981 " of set " + onosSetName + ":\n" +
3982 str( getResponses[ i ] ) )
3983 main.log.debug( "Expected: " + str( onosSet ) )
3984 main.log.debug( "Actual: " + str( current ) )
3985 getResults = main.FALSE
3986 else:
3987 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003988 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003989 " has repeat elements in" +
3990 " set " + onosSetName + ":\n" +
3991 str( getResponses[ i ] ) )
3992 getResults = main.FALSE
3993 elif getResponses[ i ] == main.ERROR:
3994 getResults = main.FALSE
3995 sizeResponses = []
3996 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003997 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003998 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003999 name="setTestSize-" + str( i ),
4000 args=[ onosSetName ] )
4001 threads.append( t )
4002 t.start()
4003 for t in threads:
4004 t.join()
4005 sizeResponses.append( t.result )
4006 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004007 for i in range( len( main.activeNodes ) ):
4008 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004009 if size != sizeResponses[ i ]:
4010 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004011 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004012 " expected a size of " + str( size ) +
4013 " for set " + onosSetName +
4014 " but got " + str( sizeResponses[ i ] ) )
4015 clearResults = clearResults and getResults and sizeResults
4016 utilities.assert_equals( expect=main.TRUE,
4017 actual=clearResults,
4018 onpass="Set clear correct",
4019 onfail="Set clear was incorrect" )
4020
4021 main.step( "Distributed Set addAll()" )
4022 onosSet.update( addAllValue.split() )
4023 addResponses = []
4024 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004025 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004026 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004027 name="setTestAddAll-" + str( i ),
4028 args=[ onosSetName, addAllValue ] )
4029 threads.append( t )
4030 t.start()
4031 for t in threads:
4032 t.join()
4033 addResponses.append( t.result )
4034
4035 # main.TRUE = successfully changed the set
4036 # main.FALSE = action resulted in no change in set
4037 # main.ERROR - Some error in executing the function
4038 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004039 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004040 if addResponses[ i ] == main.TRUE:
4041 # All is well
4042 pass
4043 elif addResponses[ i ] == main.FALSE:
4044 # Already in set, probably fine
4045 pass
4046 elif addResponses[ i ] == main.ERROR:
4047 # Error in execution
4048 addAllResults = main.FALSE
4049 else:
4050 # unexpected result
4051 addAllResults = main.FALSE
4052 if addAllResults != main.TRUE:
4053 main.log.error( "Error executing set addAll" )
4054
4055 # Check if set is still correct
4056 size = len( onosSet )
4057 getResponses = []
4058 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004059 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004060 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004061 name="setTestGet-" + str( i ),
4062 args=[ onosSetName ] )
4063 threads.append( t )
4064 t.start()
4065 for t in threads:
4066 t.join()
4067 getResponses.append( t.result )
4068 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004069 for i in range( len( main.activeNodes ) ):
4070 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004071 if isinstance( getResponses[ i ], list):
4072 current = set( getResponses[ i ] )
4073 if len( current ) == len( getResponses[ i ] ):
4074 # no repeats
4075 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004076 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004077 " has incorrect view" +
4078 " of set " + onosSetName + ":\n" +
4079 str( getResponses[ i ] ) )
4080 main.log.debug( "Expected: " + str( onosSet ) )
4081 main.log.debug( "Actual: " + str( current ) )
4082 getResults = main.FALSE
4083 else:
4084 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004085 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004086 " has repeat elements in" +
4087 " set " + onosSetName + ":\n" +
4088 str( getResponses[ i ] ) )
4089 getResults = main.FALSE
4090 elif getResponses[ i ] == main.ERROR:
4091 getResults = main.FALSE
4092 sizeResponses = []
4093 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004094 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004095 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004096 name="setTestSize-" + str( i ),
4097 args=[ onosSetName ] )
4098 threads.append( t )
4099 t.start()
4100 for t in threads:
4101 t.join()
4102 sizeResponses.append( t.result )
4103 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004104 for i in range( len( main.activeNodes ) ):
4105 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004106 if size != sizeResponses[ i ]:
4107 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004108 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004109 " expected a size of " + str( size ) +
4110 " for set " + onosSetName +
4111 " but got " + str( sizeResponses[ i ] ) )
4112 addAllResults = addAllResults and getResults and sizeResults
4113 utilities.assert_equals( expect=main.TRUE,
4114 actual=addAllResults,
4115 onpass="Set addAll correct",
4116 onfail="Set addAll was incorrect" )
4117
4118 main.step( "Distributed Set retain()" )
4119 onosSet.intersection_update( retainValue.split() )
4120 retainResponses = []
4121 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004122 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004123 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004124 name="setTestRetain-" + str( i ),
4125 args=[ onosSetName, retainValue ],
4126 kwargs={ "retain": True } )
4127 threads.append( t )
4128 t.start()
4129 for t in threads:
4130 t.join()
4131 retainResponses.append( t.result )
4132
4133 # main.TRUE = successfully changed the set
4134 # main.FALSE = action resulted in no change in set
4135 # main.ERROR - Some error in executing the function
4136 retainResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004137 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004138 if retainResponses[ i ] == main.TRUE:
4139 # All is well
4140 pass
4141 elif retainResponses[ i ] == main.FALSE:
4142 # Already in set, probably fine
4143 pass
4144 elif retainResponses[ i ] == main.ERROR:
4145 # Error in execution
4146 retainResults = main.FALSE
4147 else:
4148 # unexpected result
4149 retainResults = main.FALSE
4150 if retainResults != main.TRUE:
4151 main.log.error( "Error executing set retain" )
4152
4153 # Check if set is still correct
4154 size = len( onosSet )
4155 getResponses = []
4156 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004157 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004158 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004159 name="setTestGet-" + str( i ),
4160 args=[ onosSetName ] )
4161 threads.append( t )
4162 t.start()
4163 for t in threads:
4164 t.join()
4165 getResponses.append( t.result )
4166 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004167 for i in range( len( main.activeNodes ) ):
4168 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004169 if isinstance( getResponses[ i ], list):
4170 current = set( getResponses[ i ] )
4171 if len( current ) == len( getResponses[ i ] ):
4172 # no repeats
4173 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004174 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004175 " has incorrect view" +
4176 " of set " + onosSetName + ":\n" +
4177 str( getResponses[ i ] ) )
4178 main.log.debug( "Expected: " + str( onosSet ) )
4179 main.log.debug( "Actual: " + str( current ) )
4180 getResults = main.FALSE
4181 else:
4182 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004183 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004184 " has repeat elements in" +
4185 " set " + onosSetName + ":\n" +
4186 str( getResponses[ i ] ) )
4187 getResults = main.FALSE
4188 elif getResponses[ i ] == main.ERROR:
4189 getResults = main.FALSE
4190 sizeResponses = []
4191 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004192 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004193 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004194 name="setTestSize-" + str( i ),
4195 args=[ onosSetName ] )
4196 threads.append( t )
4197 t.start()
4198 for t in threads:
4199 t.join()
4200 sizeResponses.append( t.result )
4201 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004202 for i in range( len( main.activeNodes ) ):
4203 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004204 if size != sizeResponses[ i ]:
4205 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004206 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004207 str( size ) + " for set " + onosSetName +
4208 " but got " + str( sizeResponses[ i ] ) )
4209 retainResults = retainResults and getResults and sizeResults
4210 utilities.assert_equals( expect=main.TRUE,
4211 actual=retainResults,
4212 onpass="Set retain correct",
4213 onfail="Set retain was incorrect" )
4214
Jon Hall2a5002c2015-08-21 16:49:11 -07004215 # Transactional maps
4216 main.step( "Partitioned Transactional maps put" )
4217 tMapValue = "Testing"
4218 numKeys = 100
4219 putResult = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004220 node = main.activeNodes[0]
4221 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004222 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004223 for i in putResponses:
4224 if putResponses[ i ][ 'value' ] != tMapValue:
4225 putResult = False
4226 else:
4227 putResult = False
4228 if not putResult:
4229 main.log.debug( "Put response values: " + str( putResponses ) )
4230 utilities.assert_equals( expect=True,
4231 actual=putResult,
4232 onpass="Partitioned Transactional Map put successful",
4233 onfail="Partitioned Transactional Map put values are incorrect" )
4234
4235 main.step( "Partitioned Transactional maps get" )
Jon Hall9bfadd22016-05-11 14:48:07 -07004236 # FIXME: is this sleep needed?
4237 time.sleep( 5 )
4238
Jon Hall2a5002c2015-08-21 16:49:11 -07004239 getCheck = True
4240 for n in range( 1, numKeys + 1 ):
4241 getResponses = []
4242 threads = []
4243 valueCheck = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004244 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004245 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4246 name="TMap-get-" + str( i ),
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004247 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004248 threads.append( t )
4249 t.start()
4250 for t in threads:
4251 t.join()
4252 getResponses.append( t.result )
4253 for node in getResponses:
4254 if node != tMapValue:
4255 valueCheck = False
4256 if not valueCheck:
4257 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4258 main.log.warn( getResponses )
4259 getCheck = getCheck and valueCheck
4260 utilities.assert_equals( expect=True,
4261 actual=getCheck,
4262 onpass="Partitioned Transactional Map get values were correct",
4263 onfail="Partitioned Transactional Map values incorrect" )