blob: 52a242cf9005bc87ab2288426be89f0fbdbd03b3 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
Jon Hallb3ed8ed2015-10-28 16:43:55 -070012CASE61: The Failure inducing case.
13CASE62: The Failure recovery case.
Jon Hall5cf14d52015-07-16 12:15:19 -070014CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
Jon Hallb3ed8ed2015-10-28 16:43:55 -070028class HAkillNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -070029
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall3b489db2015-10-05 14:38:37 -070052 import pexpect
Jon Hall6e709752016-02-01 13:38:46 -080053 import time
Jon Halla440e872016-03-31 15:15:50 -070054 import json
Jon Hall6e709752016-02-01 13:38:46 -080055 main.log.info( "ONOS HA test: Restart a minority of ONOS nodes - " +
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "initialization" )
57 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070058 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070059 "installing ONOS, starting Mininet and ONOS" +\
60 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070061
62 # load some variables from the params file
63 PULLCODE = False
64 if main.params[ 'Git' ] == 'True':
65 PULLCODE = True
66 gitBranch = main.params[ 'branch' ]
67 cellName = main.params[ 'ENV' ][ 'cellName' ]
68
Jon Halle1a3b752015-07-22 13:02:46 -070069 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070071 if main.ONOSbench.maxNodes < main.numCtrls:
72 main.numCtrls = int( main.ONOSbench.maxNodes )
73 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070074 global ONOS1Port
75 global ONOS2Port
76 global ONOS3Port
77 global ONOS4Port
78 global ONOS5Port
79 global ONOS6Port
80 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070081 # These are for csv plotting in jenkins
82 global labels
83 global data
84 labels = []
85 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070086
87 # FIXME: just get controller port from params?
88 # TODO: do we really need all these?
89 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
90 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
91 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
92 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
93 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
94 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
95 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
96
Jon Halle1a3b752015-07-22 13:02:46 -070097 try:
Jon Hall53c5e662016-04-13 16:06:56 -070098 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070099 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -0700100 except Exception as e:
101 main.log.exception( e )
102 main.cleanup()
103 main.exit()
104
105 main.CLIs = []
106 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700107 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700108 for i in range( 1, main.numCtrls + 1 ):
109 try:
110 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
111 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
112 ipList.append( main.nodes[ -1 ].ip_address )
113 except AttributeError:
114 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700115
116 main.step( "Create cell file" )
117 cellAppString = main.params[ 'ENV' ][ 'appString' ]
118 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
119 main.Mininet1.ip_address,
120 cellAppString, ipList )
121 main.step( "Applying cell variable to environment" )
122 cellResult = main.ONOSbench.setCell( cellName )
123 verifyResult = main.ONOSbench.verifyCell()
124
125 # FIXME:this is short term fix
126 main.log.info( "Removing raft logs" )
127 main.ONOSbench.onosRemoveRaftLogs()
128
129 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700130 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700131 main.ONOSbench.onosUninstall( node.ip_address )
132
133 # Make sure ONOS is DEAD
134 main.log.info( "Killing any ONOS processes" )
135 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700136 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700137 killed = main.ONOSbench.onosKill( node.ip_address )
138 killResults = killResults and killed
139
140 cleanInstallResult = main.TRUE
141 gitPullResult = main.TRUE
142
143 main.step( "Starting Mininet" )
144 # scp topo file to mininet
145 # TODO: move to params?
146 topoName = "obelisk.py"
147 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700148 main.ONOSbench.scp( main.Mininet1,
149 filePath + topoName,
150 main.Mininet1.home,
151 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700152 mnResult = main.Mininet1.startNet( )
153 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
154 onpass="Mininet Started",
155 onfail="Error starting Mininet" )
156
157 main.step( "Git checkout and pull " + gitBranch )
158 if PULLCODE:
159 main.ONOSbench.gitCheckout( gitBranch )
160 gitPullResult = main.ONOSbench.gitPull()
161 # values of 1 or 3 are good
162 utilities.assert_lesser( expect=0, actual=gitPullResult,
163 onpass="Git pull successful",
164 onfail="Git pull failed" )
165 main.ONOSbench.getVersion( report=True )
166
167 main.step( "Using mvn clean install" )
168 cleanInstallResult = main.TRUE
169 if PULLCODE and gitPullResult == main.TRUE:
170 cleanInstallResult = main.ONOSbench.cleanInstall()
171 else:
172 main.log.warn( "Did not pull new code so skipping mvn " +
173 "clean install" )
174 utilities.assert_equals( expect=main.TRUE,
175 actual=cleanInstallResult,
176 onpass="MCI successful",
177 onfail="MCI failed" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700178
179 main.step( "Make sure ONOS service doesn't automatically respawn" )
180 handle = main.ONOSbench.handle
181 handle.sendline( "sed -i -e 's/^respawn$/#respawn/g' tools/package/init/onos.conf" )
182 handle.expect( "\$" ) # $ from the command
183 handle.expect( "\$" ) # $ from the prompt
184
Jon Hall5cf14d52015-07-16 12:15:19 -0700185 # GRAPHS
186 # NOTE: important params here:
187 # job = name of Jenkins job
188 # Plot Name = Plot-HA, only can be used if multiple plots
189 # index = The number of the graph under plot name
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700190 job = "HAkillNodes"
Jon Hall5cf14d52015-07-16 12:15:19 -0700191 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700192 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700193 graphs = '<ac:structured-macro ac:name="html">\n'
194 graphs += '<ac:plain-text-body><![CDATA[\n'
195 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800196 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700197 '&width=500&height=300"' +\
198 'noborder="0" width="500" height="300" scrolling="yes" ' +\
199 'seamless="seamless"></iframe>\n'
200 graphs += ']]></ac:plain-text-body>\n'
201 graphs += '</ac:structured-macro>\n'
202 main.log.wiki(graphs)
203
204 main.step( "Creating ONOS package" )
Jon Hall3b489db2015-10-05 14:38:37 -0700205 # copy gen-partions file to ONOS
206 # NOTE: this assumes TestON and ONOS are on the same machine
Jon Hall53c5e662016-04-13 16:06:56 -0700207 srcFile = main.testDir + "/HA/dependencies/onos-gen-partitions"
Jon Hall3b489db2015-10-05 14:38:37 -0700208 dstDir = main.ONOSbench.home + "/tools/test/bin/onos-gen-partitions"
209 cpResult = main.ONOSbench.secureCopy( main.ONOSbench.user_name,
210 main.ONOSbench.ip_address,
211 srcFile,
212 dstDir,
213 pwd=main.ONOSbench.pwd,
214 direction="from" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700215 packageResult = main.ONOSbench.onosPackage()
216 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
217 onpass="ONOS package successful",
218 onfail="ONOS package failed" )
219
220 main.step( "Installing ONOS package" )
221 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700222 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700223 tmpResult = main.ONOSbench.onosInstall( options="-f",
224 node=node.ip_address )
225 onosInstallResult = onosInstallResult and tmpResult
226 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
227 onpass="ONOS install successful",
228 onfail="ONOS install failed" )
Jon Hall3b489db2015-10-05 14:38:37 -0700229 # clean up gen-partitions file
230 try:
231 main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
232 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
233 main.ONOSbench.handle.sendline( "git checkout -- tools/test/bin/onos-gen-partitions" )
234 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
235 main.log.info( " Cleaning custom gen partitions file, response was: \n" +
236 str( main.ONOSbench.handle.before ) )
237 except ( pexpect.TIMEOUT, pexpect.EOF ):
238 main.log.exception( "ONOSbench: pexpect exception found:" +
239 main.ONOSbench.handle.before )
240 main.cleanup()
241 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700242
243 main.step( "Checking if ONOS is up yet" )
244 for i in range( 2 ):
245 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700246 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700247 started = main.ONOSbench.isup( node.ip_address )
248 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800249 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700250 onosIsupResult = onosIsupResult and started
251 if onosIsupResult == main.TRUE:
252 break
253 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
254 onpass="ONOS startup successful",
255 onfail="ONOS startup failed" )
256
257 main.log.step( "Starting ONOS CLI sessions" )
258 cliResults = main.TRUE
259 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700260 for i in range( main.numCtrls ):
261 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700262 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700263 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700264 threads.append( t )
265 t.start()
266
267 for t in threads:
268 t.join()
269 cliResults = cliResults and t.result
270 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
271 onpass="ONOS cli startup successful",
272 onfail="ONOS cli startup failed" )
273
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700274 # Create a list of active nodes for use when some nodes are stopped
275 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
276
Jon Hall5cf14d52015-07-16 12:15:19 -0700277 if main.params[ 'tcpdump' ].lower() == "true":
278 main.step( "Start Packet Capture MN" )
279 main.Mininet2.startTcpdump(
280 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
281 + "-MN.pcap",
282 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
283 port=main.params[ 'MNtcpdump' ][ 'port' ] )
284
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700285 main.step( "Clean up ONOS service changes" )
286 handle.sendline( "git checkout -- tools/package/init/onos.conf" )
287 handle.expect( "\$" )
288
Jon Halla440e872016-03-31 15:15:50 -0700289 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700290 nodeResults = utilities.retry( main.HA.nodesCheck,
291 False,
292 args=[main.activeNodes],
293 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700294
Jon Hall41d39f12016-04-11 22:54:35 -0700295 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700296 onpass="Nodes check successful",
297 onfail="Nodes check NOT successful" )
298
299 if not nodeResults:
300 for cli in main.CLIs:
301 main.log.debug( "{} components not ACTIVE: \n{}".format(
302 cli.name,
303 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
304
Jon Hall5cf14d52015-07-16 12:15:19 -0700305 if cliResults == main.FALSE:
306 main.log.error( "Failed to start ONOS, stopping test" )
307 main.cleanup()
308 main.exit()
309
Jon Hall172b7ba2016-04-07 18:12:20 -0700310 main.step( "Activate apps defined in the params file" )
311 # get data from the params
312 apps = main.params.get( 'apps' )
313 if apps:
314 apps = apps.split(',')
315 main.log.warn( apps )
316 activateResult = True
317 for app in apps:
318 main.CLIs[ 0 ].app( app, "Activate" )
319 # TODO: check this worked
320 time.sleep( 10 ) # wait for apps to activate
321 for app in apps:
322 state = main.CLIs[ 0 ].appStatus( app )
323 if state == "ACTIVE":
324 activateResult = activeResult and True
325 else:
326 main.log.error( "{} is in {} state".format( app, state ) )
327 activeResult = False
328 utilities.assert_equals( expect=True,
329 actual=activateResult,
330 onpass="Successfully activated apps",
331 onfail="Failed to activate apps" )
332 else:
333 main.log.warn( "No apps were specified to be loaded after startup" )
334
335 main.step( "Set ONOS configurations" )
336 config = main.params.get( 'ONOS_Configuration' )
337 if config:
338 main.log.debug( config )
339 checkResult = main.TRUE
340 for component in config:
341 for setting in config[component]:
342 value = config[component][setting]
343 check = main.CLIs[ 0 ].setCfg( component, setting, value )
344 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
345 checkResult = check and checkResult
346 utilities.assert_equals( expect=main.TRUE,
347 actual=checkResult,
348 onpass="Successfully set config",
349 onfail="Failed to set config" )
350 else:
351 main.log.warn( "No configurations were specified to be changed after startup" )
352
Jon Hall9d2dcad2016-04-08 10:15:20 -0700353 main.step( "App Ids check" )
354 appCheck = main.TRUE
355 threads = []
356 for i in main.activeNodes:
357 t = main.Thread( target=main.CLIs[i].appToIDCheck,
358 name="appToIDCheck-" + str( i ),
359 args=[] )
360 threads.append( t )
361 t.start()
362
363 for t in threads:
364 t.join()
365 appCheck = appCheck and t.result
366 if appCheck != main.TRUE:
367 node = main.activeNodes[0]
368 main.log.warn( main.CLIs[node].apps() )
369 main.log.warn( main.CLIs[node].appIDs() )
370 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
371 onpass="App Ids seem to be correct",
372 onfail="Something is wrong with app Ids" )
373
Jon Hall5cf14d52015-07-16 12:15:19 -0700374 def CASE2( self, main ):
375 """
376 Assign devices to controllers
377 """
378 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700379 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700380 assert main, "main not defined"
381 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700382 assert main.CLIs, "main.CLIs not defined"
383 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700384 assert ONOS1Port, "ONOS1Port not defined"
385 assert ONOS2Port, "ONOS2Port not defined"
386 assert ONOS3Port, "ONOS3Port not defined"
387 assert ONOS4Port, "ONOS4Port not defined"
388 assert ONOS5Port, "ONOS5Port not defined"
389 assert ONOS6Port, "ONOS6Port not defined"
390 assert ONOS7Port, "ONOS7Port not defined"
391
392 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700393 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700394 "and check that an ONOS node becomes the " +\
395 "master of the device."
396 main.step( "Assign switches to controllers" )
397
398 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700399 for i in range( main.numCtrls ):
400 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700401 swList = []
402 for i in range( 1, 29 ):
403 swList.append( "s" + str( i ) )
404 main.Mininet1.assignSwController( sw=swList, ip=ipList )
405
406 mastershipCheck = main.TRUE
407 for i in range( 1, 29 ):
408 response = main.Mininet1.getSwController( "s" + str( i ) )
409 try:
410 main.log.info( str( response ) )
411 except Exception:
412 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700413 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700414 if re.search( "tcp:" + node.ip_address, response ):
415 mastershipCheck = mastershipCheck and main.TRUE
416 else:
417 main.log.error( "Error, node " + node.ip_address + " is " +
418 "not in the list of controllers s" +
419 str( i ) + " is connecting to." )
420 mastershipCheck = main.FALSE
421 utilities.assert_equals(
422 expect=main.TRUE,
423 actual=mastershipCheck,
424 onpass="Switch mastership assigned correctly",
425 onfail="Switches not assigned correctly to controllers" )
426
427 def CASE21( self, main ):
428 """
429 Assign mastership to controllers
430 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700431 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700432 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700433 assert main, "main not defined"
434 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700435 assert main.CLIs, "main.CLIs not defined"
436 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700437 assert ONOS1Port, "ONOS1Port not defined"
438 assert ONOS2Port, "ONOS2Port not defined"
439 assert ONOS3Port, "ONOS3Port not defined"
440 assert ONOS4Port, "ONOS4Port not defined"
441 assert ONOS5Port, "ONOS5Port not defined"
442 assert ONOS6Port, "ONOS6Port not defined"
443 assert ONOS7Port, "ONOS7Port not defined"
444
445 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700446 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700447 "device. Then manually assign" +\
448 " mastership to specific ONOS nodes using" +\
449 " 'device-role'"
450 main.step( "Assign mastership of switches to specific controllers" )
451 # Manually assign mastership to the controller we want
452 roleCall = main.TRUE
453
454 ipList = [ ]
455 deviceList = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700456 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700457 try:
458 # Assign mastership to specific controllers. This assignment was
459 # determined for a 7 node cluser, but will work with any sized
460 # cluster
461 for i in range( 1, 29 ): # switches 1 through 28
462 # set up correct variables:
463 if i == 1:
464 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700465 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700466 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700467 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700468 c = 1 % main.numCtrls
469 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700470 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700471 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700472 c = 1 % main.numCtrls
473 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700474 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700475 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700476 c = 3 % main.numCtrls
477 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700478 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700479 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700480 c = 2 % main.numCtrls
481 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700482 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700483 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700484 c = 2 % main.numCtrls
485 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700486 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700487 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700488 c = 5 % main.numCtrls
489 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700490 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700491 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700492 c = 4 % main.numCtrls
493 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700494 dpid = '3' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700495 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700496 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700497 c = 6 % main.numCtrls
498 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700499 dpid = '6' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700500 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700501 elif i == 28:
502 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700503 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700504 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700505 else:
506 main.log.error( "You didn't write an else statement for " +
507 "switch s" + str( i ) )
508 roleCall = main.FALSE
509 # Assign switch
510 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
511 # TODO: make this controller dynamic
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700512 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700513 ipList.append( ip )
514 deviceList.append( deviceId )
515 except ( AttributeError, AssertionError ):
516 main.log.exception( "Something is wrong with ONOS device view" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700517 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700518 utilities.assert_equals(
519 expect=main.TRUE,
520 actual=roleCall,
521 onpass="Re-assigned switch mastership to designated controller",
522 onfail="Something wrong with deviceRole calls" )
523
524 main.step( "Check mastership was correctly assigned" )
525 roleCheck = main.TRUE
526 # NOTE: This is due to the fact that device mastership change is not
527 # atomic and is actually a multi step process
528 time.sleep( 5 )
529 for i in range( len( ipList ) ):
530 ip = ipList[i]
531 deviceId = deviceList[i]
532 # Check assignment
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700533 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700534 if ip in master:
535 roleCheck = roleCheck and main.TRUE
536 else:
537 roleCheck = roleCheck and main.FALSE
538 main.log.error( "Error, controller " + ip + " is not" +
539 " master " + "of device " +
540 str( deviceId ) + ". Master is " +
541 repr( master ) + "." )
542 utilities.assert_equals(
543 expect=main.TRUE,
544 actual=roleCheck,
545 onpass="Switches were successfully reassigned to designated " +
546 "controller",
547 onfail="Switches were not successfully reassigned" )
548
549 def CASE3( self, main ):
550 """
551 Assign intents
552 """
553 import time
554 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700555 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700556 assert main, "main not defined"
557 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700558 assert main.CLIs, "main.CLIs not defined"
559 assert main.nodes, "main.nodes not defined"
Jon Halla440e872016-03-31 15:15:50 -0700560 try:
561 labels
562 except NameError:
563 main.log.error( "labels not defined, setting to []" )
564 labels = []
565 try:
566 data
567 except NameError:
568 main.log.error( "data not defined, setting to []" )
569 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700570 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700571 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700572 "assign predetermined host-to-host intents." +\
573 " After installation, check that the intent" +\
574 " is distributed to all nodes and the state" +\
575 " is INSTALLED"
576
577 # install onos-app-fwd
578 main.step( "Install reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700579 onosCli = main.CLIs[ main.activeNodes[0] ]
580 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700581 utilities.assert_equals( expect=main.TRUE, actual=installResults,
582 onpass="Install fwd successful",
583 onfail="Install fwd failed" )
584
585 main.step( "Check app ids" )
586 appCheck = main.TRUE
587 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700588 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700589 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700590 name="appToIDCheck-" + str( i ),
591 args=[] )
592 threads.append( t )
593 t.start()
594
595 for t in threads:
596 t.join()
597 appCheck = appCheck and t.result
598 if appCheck != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700599 main.log.warn( onosCli.apps() )
600 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700601 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
602 onpass="App Ids seem to be correct",
603 onfail="Something is wrong with app Ids" )
604
605 main.step( "Discovering Hosts( Via pingall for now )" )
606 # FIXME: Once we have a host discovery mechanism, use that instead
607 # REACTIVE FWD test
608 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700609 passMsg = "Reactive Pingall test passed"
610 time1 = time.time()
611 pingResult = main.Mininet1.pingall()
612 time2 = time.time()
613 if not pingResult:
614 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700615 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700616 passMsg += " on the second try"
617 utilities.assert_equals(
618 expect=main.TRUE,
619 actual=pingResult,
620 onpass= passMsg,
621 onfail="Reactive Pingall failed, " +
622 "one or more ping pairs failed" )
623 main.log.info( "Time for pingall: %2f seconds" %
624 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700625 # timeout for fwd flows
626 time.sleep( 11 )
627 # uninstall onos-app-fwd
628 main.step( "Uninstall reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700629 node = main.activeNodes[0]
630 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700631 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
632 onpass="Uninstall fwd successful",
633 onfail="Uninstall fwd failed" )
634
635 main.step( "Check app ids" )
636 threads = []
637 appCheck2 = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700638 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700639 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700640 name="appToIDCheck-" + str( i ),
641 args=[] )
642 threads.append( t )
643 t.start()
644
645 for t in threads:
646 t.join()
647 appCheck2 = appCheck2 and t.result
648 if appCheck2 != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700649 node = main.activeNodes[0]
650 main.log.warn( main.CLIs[node].apps() )
651 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700652 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
653 onpass="App Ids seem to be correct",
654 onfail="Something is wrong with app Ids" )
655
656 main.step( "Add host intents via cli" )
657 intentIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700658 # TODO: move the host numbers to params
659 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700660 intentAddResult = True
661 hostResult = main.TRUE
662 for i in range( 8, 18 ):
663 main.log.info( "Adding host intent between h" + str( i ) +
664 " and h" + str( i + 10 ) )
665 host1 = "00:00:00:00:00:" + \
666 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
667 host2 = "00:00:00:00:00:" + \
668 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
669 # NOTE: getHost can return None
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700670 host1Dict = onosCli.getHost( host1 )
671 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700672 host1Id = None
673 host2Id = None
674 if host1Dict and host2Dict:
675 host1Id = host1Dict.get( 'id', None )
676 host2Id = host2Dict.get( 'id', None )
677 if host1Id and host2Id:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700678 nodeNum = ( i % len( main.activeNodes ) )
679 node = main.activeNodes[nodeNum]
680 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700681 if tmpId:
682 main.log.info( "Added intent with id: " + tmpId )
683 intentIds.append( tmpId )
684 else:
685 main.log.error( "addHostIntent returned: " +
686 repr( tmpId ) )
687 else:
688 main.log.error( "Error, getHost() failed for h" + str( i ) +
689 " and/or h" + str( i + 10 ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700690 node = main.activeNodes[0]
691 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700692 main.log.warn( "Hosts output: " )
693 try:
694 main.log.warn( json.dumps( json.loads( hosts ),
695 sort_keys=True,
696 indent=4,
697 separators=( ',', ': ' ) ) )
698 except ( ValueError, TypeError ):
699 main.log.warn( repr( hosts ) )
700 hostResult = main.FALSE
701 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
702 onpass="Found a host id for each host",
703 onfail="Error looking up host ids" )
704
705 intentStart = time.time()
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700706 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700707 main.log.info( "Submitted intents: " + str( intentIds ) )
708 main.log.info( "Intents in ONOS: " + str( onosIds ) )
709 for intent in intentIds:
710 if intent in onosIds:
711 pass # intent submitted is in onos
712 else:
713 intentAddResult = False
714 if intentAddResult:
715 intentStop = time.time()
716 else:
717 intentStop = None
718 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700719 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700720 intentStates = []
721 installedCheck = True
722 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
723 count = 0
724 try:
725 for intent in json.loads( intents ):
726 state = intent.get( 'state', None )
727 if "INSTALLED" not in state:
728 installedCheck = False
729 intentId = intent.get( 'id', None )
730 intentStates.append( ( intentId, state ) )
731 except ( ValueError, TypeError ):
732 main.log.exception( "Error parsing intents" )
733 # add submitted intents not in the store
734 tmplist = [ i for i, s in intentStates ]
735 missingIntents = False
736 for i in intentIds:
737 if i not in tmplist:
738 intentStates.append( ( i, " - " ) )
739 missingIntents = True
740 intentStates.sort()
741 for i, s in intentStates:
742 count += 1
743 main.log.info( "%-6s%-15s%-15s" %
744 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700745 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700746 try:
747 missing = False
748 if leaders:
749 parsedLeaders = json.loads( leaders )
750 main.log.warn( json.dumps( parsedLeaders,
751 sort_keys=True,
752 indent=4,
753 separators=( ',', ': ' ) ) )
754 # check for all intent partitions
755 topics = []
756 for i in range( 14 ):
757 topics.append( "intent-partition-" + str( i ) )
758 main.log.debug( topics )
759 ONOStopics = [ j['topic'] for j in parsedLeaders ]
760 for topic in topics:
761 if topic not in ONOStopics:
762 main.log.error( "Error: " + topic +
763 " not in leaders" )
764 missing = True
765 else:
766 main.log.error( "leaders() returned None" )
767 except ( ValueError, TypeError ):
768 main.log.exception( "Error parsing leaders" )
769 main.log.error( repr( leaders ) )
770 # Check all nodes
771 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700772 for i in main.activeNodes:
773 response = main.CLIs[i].leaders( jsonFormat=False)
774 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700775 str( response ) )
776
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700777 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700778 try:
779 if partitions :
780 parsedPartitions = json.loads( partitions )
781 main.log.warn( json.dumps( parsedPartitions,
782 sort_keys=True,
783 indent=4,
784 separators=( ',', ': ' ) ) )
785 # TODO check for a leader in all paritions
786 # TODO check for consistency among nodes
787 else:
788 main.log.error( "partitions() returned None" )
789 except ( ValueError, TypeError ):
790 main.log.exception( "Error parsing partitions" )
791 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700792 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700793 try:
794 if pendingMap :
795 parsedPending = json.loads( pendingMap )
796 main.log.warn( json.dumps( parsedPending,
797 sort_keys=True,
798 indent=4,
799 separators=( ',', ': ' ) ) )
800 # TODO check something here?
801 else:
802 main.log.error( "pendingMap() returned None" )
803 except ( ValueError, TypeError ):
804 main.log.exception( "Error parsing pending map" )
805 main.log.error( repr( pendingMap ) )
806
807 intentAddResult = bool( intentAddResult and not missingIntents and
808 installedCheck )
809 if not intentAddResult:
810 main.log.error( "Error in pushing host intents to ONOS" )
811
812 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700813 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700814 correct = True
815 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700816 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700817 onosIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700818 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700819 onosIds.append( ids )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700820 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700821 str( sorted( onosIds ) ) )
822 if sorted( ids ) != sorted( intentIds ):
823 main.log.warn( "Set of intent IDs doesn't match" )
824 correct = False
825 break
826 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700827 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700828 for intent in intents:
829 if intent[ 'state' ] != "INSTALLED":
830 main.log.warn( "Intent " + intent[ 'id' ] +
831 " is " + intent[ 'state' ] )
832 correct = False
833 break
834 if correct:
835 break
836 else:
837 time.sleep(1)
838 if not intentStop:
839 intentStop = time.time()
840 global gossipTime
841 gossipTime = intentStop - intentStart
842 main.log.info( "It took about " + str( gossipTime ) +
843 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700844 gossipPeriod = int( main.params['timers']['gossip'] )
845 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700846 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700847 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700848 onpass="ECM anti-entropy for intents worked within " +
849 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700850 onfail="Intent ECM anti-entropy took too long. " +
851 "Expected time:{}, Actual time:{}".format( maxGossipTime,
852 gossipTime ) )
853 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700854 intentAddResult = True
855
856 if not intentAddResult or "key" in pendingMap:
857 import time
858 installedCheck = True
859 main.log.info( "Sleeping 60 seconds to see if intents are found" )
860 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700861 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700862 main.log.info( "Submitted intents: " + str( intentIds ) )
863 main.log.info( "Intents in ONOS: " + str( onosIds ) )
864 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700865 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700866 intentStates = []
867 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
868 count = 0
869 try:
870 for intent in json.loads( intents ):
871 # Iter through intents of a node
872 state = intent.get( 'state', None )
873 if "INSTALLED" not in state:
874 installedCheck = False
875 intentId = intent.get( 'id', None )
876 intentStates.append( ( intentId, state ) )
877 except ( ValueError, TypeError ):
878 main.log.exception( "Error parsing intents" )
879 # add submitted intents not in the store
880 tmplist = [ i for i, s in intentStates ]
881 for i in intentIds:
882 if i not in tmplist:
883 intentStates.append( ( i, " - " ) )
884 intentStates.sort()
885 for i, s in intentStates:
886 count += 1
887 main.log.info( "%-6s%-15s%-15s" %
888 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700889 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700890 try:
891 missing = False
892 if leaders:
893 parsedLeaders = json.loads( leaders )
894 main.log.warn( json.dumps( parsedLeaders,
895 sort_keys=True,
896 indent=4,
897 separators=( ',', ': ' ) ) )
898 # check for all intent partitions
899 # check for election
900 topics = []
901 for i in range( 14 ):
902 topics.append( "intent-partition-" + str( i ) )
903 # FIXME: this should only be after we start the app
904 topics.append( "org.onosproject.election" )
905 main.log.debug( topics )
906 ONOStopics = [ j['topic'] for j in parsedLeaders ]
907 for topic in topics:
908 if topic not in ONOStopics:
909 main.log.error( "Error: " + topic +
910 " not in leaders" )
911 missing = True
912 else:
913 main.log.error( "leaders() returned None" )
914 except ( ValueError, TypeError ):
915 main.log.exception( "Error parsing leaders" )
916 main.log.error( repr( leaders ) )
917 # Check all nodes
918 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700919 for i in main.activeNodes:
920 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700921 response = node.leaders( jsonFormat=False)
922 main.log.warn( str( node.name ) + " leaders output: \n" +
923 str( response ) )
924
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700925 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700926 try:
927 if partitions :
928 parsedPartitions = json.loads( partitions )
929 main.log.warn( json.dumps( parsedPartitions,
930 sort_keys=True,
931 indent=4,
932 separators=( ',', ': ' ) ) )
933 # TODO check for a leader in all paritions
934 # TODO check for consistency among nodes
935 else:
936 main.log.error( "partitions() returned None" )
937 except ( ValueError, TypeError ):
938 main.log.exception( "Error parsing partitions" )
939 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700940 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700941 try:
942 if pendingMap :
943 parsedPending = json.loads( pendingMap )
944 main.log.warn( json.dumps( parsedPending,
945 sort_keys=True,
946 indent=4,
947 separators=( ',', ': ' ) ) )
948 # TODO check something here?
949 else:
950 main.log.error( "pendingMap() returned None" )
951 except ( ValueError, TypeError ):
952 main.log.exception( "Error parsing pending map" )
953 main.log.error( repr( pendingMap ) )
954
955 def CASE4( self, main ):
956 """
957 Ping across added host intents
958 """
959 import json
960 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700961 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700962 assert main, "main not defined"
963 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700964 assert main.CLIs, "main.CLIs not defined"
965 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700966 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700967 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700968 "functionality and check the state of " +\
969 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700970
Jon Hall41d39f12016-04-11 22:54:35 -0700971 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700972 main.step( "Check Intent state" )
973 installedCheck = False
974 loopCount = 0
975 while not installedCheck and loopCount < 40:
976 installedCheck = True
977 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700978 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700979 intentStates = []
980 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
981 count = 0
982 # Iter through intents of a node
983 try:
984 for intent in json.loads( intents ):
985 state = intent.get( 'state', None )
986 if "INSTALLED" not in state:
987 installedCheck = False
988 intentId = intent.get( 'id', None )
989 intentStates.append( ( intentId, state ) )
990 except ( ValueError, TypeError ):
991 main.log.exception( "Error parsing intents." )
992 # Print states
993 intentStates.sort()
994 for i, s in intentStates:
995 count += 1
996 main.log.info( "%-6s%-15s%-15s" %
997 ( str( count ), str( i ), str( s ) ) )
998 if not installedCheck:
999 time.sleep( 1 )
1000 loopCount += 1
1001 utilities.assert_equals( expect=True, actual=installedCheck,
1002 onpass="Intents are all INSTALLED",
1003 onfail="Intents are not all in " +
1004 "INSTALLED state" )
1005
Jon Hall9d2dcad2016-04-08 10:15:20 -07001006 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -07001007 PingResult = main.TRUE
1008 for i in range( 8, 18 ):
1009 ping = main.Mininet1.pingHost( src="h" + str( i ),
1010 target="h" + str( i + 10 ) )
1011 PingResult = PingResult and ping
1012 if ping == main.FALSE:
1013 main.log.warn( "Ping failed between h" + str( i ) +
1014 " and h" + str( i + 10 ) )
1015 elif ping == main.TRUE:
1016 main.log.info( "Ping test passed!" )
1017 # Don't set PingResult or you'd override failures
1018 if PingResult == main.FALSE:
1019 main.log.error(
1020 "Intents have not been installed correctly, pings failed." )
1021 # TODO: pretty print
1022 main.log.warn( "ONOS1 intents: " )
1023 try:
1024 tmpIntents = onosCli.intents()
1025 main.log.warn( json.dumps( json.loads( tmpIntents ),
1026 sort_keys=True,
1027 indent=4,
1028 separators=( ',', ': ' ) ) )
1029 except ( ValueError, TypeError ):
1030 main.log.warn( repr( tmpIntents ) )
1031 utilities.assert_equals(
1032 expect=main.TRUE,
1033 actual=PingResult,
1034 onpass="Intents have been installed correctly and pings work",
1035 onfail="Intents have not been installed correctly, pings failed." )
1036
Jon Hall5cf14d52015-07-16 12:15:19 -07001037 main.step( "Check leadership of topics" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001038 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001039 topicCheck = main.TRUE
1040 try:
1041 if leaders:
1042 parsedLeaders = json.loads( leaders )
1043 main.log.warn( json.dumps( parsedLeaders,
1044 sort_keys=True,
1045 indent=4,
1046 separators=( ',', ': ' ) ) )
1047 # check for all intent partitions
1048 # check for election
1049 # TODO: Look at Devices as topics now that it uses this system
1050 topics = []
1051 for i in range( 14 ):
1052 topics.append( "intent-partition-" + str( i ) )
1053 # FIXME: this should only be after we start the app
1054 # FIXME: topics.append( "org.onosproject.election" )
1055 # Print leaders output
1056 main.log.debug( topics )
1057 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1058 for topic in topics:
1059 if topic not in ONOStopics:
1060 main.log.error( "Error: " + topic +
1061 " not in leaders" )
1062 topicCheck = main.FALSE
1063 else:
1064 main.log.error( "leaders() returned None" )
1065 topicCheck = main.FALSE
1066 except ( ValueError, TypeError ):
1067 topicCheck = main.FALSE
1068 main.log.exception( "Error parsing leaders" )
1069 main.log.error( repr( leaders ) )
1070 # TODO: Check for a leader of these topics
1071 # Check all nodes
1072 if topicCheck:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001073 for i in main.activeNodes:
1074 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001075 response = node.leaders( jsonFormat=False)
1076 main.log.warn( str( node.name ) + " leaders output: \n" +
1077 str( response ) )
1078
1079 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1080 onpass="intent Partitions is in leaders",
1081 onfail="Some topics were lost " )
1082 # Print partitions
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001083 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001084 try:
1085 if partitions :
1086 parsedPartitions = json.loads( partitions )
1087 main.log.warn( json.dumps( parsedPartitions,
1088 sort_keys=True,
1089 indent=4,
1090 separators=( ',', ': ' ) ) )
1091 # TODO check for a leader in all paritions
1092 # TODO check for consistency among nodes
1093 else:
1094 main.log.error( "partitions() returned None" )
1095 except ( ValueError, TypeError ):
1096 main.log.exception( "Error parsing partitions" )
1097 main.log.error( repr( partitions ) )
1098 # Print Pending Map
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001099 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001100 try:
1101 if pendingMap :
1102 parsedPending = json.loads( pendingMap )
1103 main.log.warn( json.dumps( parsedPending,
1104 sort_keys=True,
1105 indent=4,
1106 separators=( ',', ': ' ) ) )
1107 # TODO check something here?
1108 else:
1109 main.log.error( "pendingMap() returned None" )
1110 except ( ValueError, TypeError ):
1111 main.log.exception( "Error parsing pending map" )
1112 main.log.error( repr( pendingMap ) )
1113
1114 if not installedCheck:
1115 main.log.info( "Waiting 60 seconds to see if the state of " +
1116 "intents change" )
1117 time.sleep( 60 )
1118 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001119 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001120 intentStates = []
1121 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1122 count = 0
1123 # Iter through intents of a node
1124 try:
1125 for intent in json.loads( intents ):
1126 state = intent.get( 'state', None )
1127 if "INSTALLED" not in state:
1128 installedCheck = False
1129 intentId = intent.get( 'id', None )
1130 intentStates.append( ( intentId, state ) )
1131 except ( ValueError, TypeError ):
1132 main.log.exception( "Error parsing intents." )
1133 intentStates.sort()
1134 for i, s in intentStates:
1135 count += 1
1136 main.log.info( "%-6s%-15s%-15s" %
1137 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001138 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001139 try:
1140 missing = False
1141 if leaders:
1142 parsedLeaders = json.loads( leaders )
1143 main.log.warn( json.dumps( parsedLeaders,
1144 sort_keys=True,
1145 indent=4,
1146 separators=( ',', ': ' ) ) )
1147 # check for all intent partitions
1148 # check for election
1149 topics = []
1150 for i in range( 14 ):
1151 topics.append( "intent-partition-" + str( i ) )
1152 # FIXME: this should only be after we start the app
1153 topics.append( "org.onosproject.election" )
1154 main.log.debug( topics )
1155 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1156 for topic in topics:
1157 if topic not in ONOStopics:
1158 main.log.error( "Error: " + topic +
1159 " not in leaders" )
1160 missing = True
1161 else:
1162 main.log.error( "leaders() returned None" )
1163 except ( ValueError, TypeError ):
1164 main.log.exception( "Error parsing leaders" )
1165 main.log.error( repr( leaders ) )
1166 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001167 for i in main.activeNodes:
1168 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001169 response = node.leaders( jsonFormat=False)
1170 main.log.warn( str( node.name ) + " leaders output: \n" +
1171 str( response ) )
1172
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001173 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001174 try:
1175 if partitions :
1176 parsedPartitions = json.loads( partitions )
1177 main.log.warn( json.dumps( parsedPartitions,
1178 sort_keys=True,
1179 indent=4,
1180 separators=( ',', ': ' ) ) )
1181 # TODO check for a leader in all paritions
1182 # TODO check for consistency among nodes
1183 else:
1184 main.log.error( "partitions() returned None" )
1185 except ( ValueError, TypeError ):
1186 main.log.exception( "Error parsing partitions" )
1187 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001188 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001189 try:
1190 if pendingMap :
1191 parsedPending = json.loads( pendingMap )
1192 main.log.warn( json.dumps( parsedPending,
1193 sort_keys=True,
1194 indent=4,
1195 separators=( ',', ': ' ) ) )
1196 # TODO check something here?
1197 else:
1198 main.log.error( "pendingMap() returned None" )
1199 except ( ValueError, TypeError ):
1200 main.log.exception( "Error parsing pending map" )
1201 main.log.error( repr( pendingMap ) )
1202 # Print flowrules
Jon Hall41d39f12016-04-11 22:54:35 -07001203 main.log.debug( onosCli.flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001204 main.step( "Wait a minute then ping again" )
1205 # the wait is above
1206 PingResult = main.TRUE
1207 for i in range( 8, 18 ):
1208 ping = main.Mininet1.pingHost( src="h" + str( i ),
1209 target="h" + str( i + 10 ) )
1210 PingResult = PingResult and ping
1211 if ping == main.FALSE:
1212 main.log.warn( "Ping failed between h" + str( i ) +
1213 " and h" + str( i + 10 ) )
1214 elif ping == main.TRUE:
1215 main.log.info( "Ping test passed!" )
1216 # Don't set PingResult or you'd override failures
1217 if PingResult == main.FALSE:
1218 main.log.error(
1219 "Intents have not been installed correctly, pings failed." )
1220 # TODO: pretty print
1221 main.log.warn( "ONOS1 intents: " )
1222 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001223 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001224 main.log.warn( json.dumps( json.loads( tmpIntents ),
1225 sort_keys=True,
1226 indent=4,
1227 separators=( ',', ': ' ) ) )
1228 except ( ValueError, TypeError ):
1229 main.log.warn( repr( tmpIntents ) )
1230 utilities.assert_equals(
1231 expect=main.TRUE,
1232 actual=PingResult,
1233 onpass="Intents have been installed correctly and pings work",
1234 onfail="Intents have not been installed correctly, pings failed." )
1235
1236 def CASE5( self, main ):
1237 """
1238 Reading state of ONOS
1239 """
1240 import json
1241 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001242 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001243 assert main, "main not defined"
1244 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001245 assert main.CLIs, "main.CLIs not defined"
1246 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001247
1248 main.case( "Setting up and gathering data for current state" )
1249 # The general idea for this test case is to pull the state of
1250 # ( intents,flows, topology,... ) from each ONOS node
1251 # We can then compare them with each other and also with past states
1252
1253 main.step( "Check that each switch has a master" )
1254 global mastershipState
1255 mastershipState = '[]'
1256
1257 # Assert that each device has a master
1258 rolesNotNull = main.TRUE
1259 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001260 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001261 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001262 name="rolesNotNull-" + str( i ),
1263 args=[] )
1264 threads.append( t )
1265 t.start()
1266
1267 for t in threads:
1268 t.join()
1269 rolesNotNull = rolesNotNull and t.result
1270 utilities.assert_equals(
1271 expect=main.TRUE,
1272 actual=rolesNotNull,
1273 onpass="Each device has a master",
1274 onfail="Some devices don't have a master assigned" )
1275
1276 main.step( "Get the Mastership of each switch from each controller" )
1277 ONOSMastership = []
1278 mastershipCheck = main.FALSE
1279 consistentMastership = True
1280 rolesResults = True
1281 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001282 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001283 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001284 name="roles-" + str( i ),
1285 args=[] )
1286 threads.append( t )
1287 t.start()
1288
1289 for t in threads:
1290 t.join()
1291 ONOSMastership.append( t.result )
1292
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001293 for i in range( len( ONOSMastership ) ):
1294 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001295 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001296 main.log.error( "Error in getting ONOS" + node + " roles" )
1297 main.log.warn( "ONOS" + node + " mastership response: " +
1298 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001299 rolesResults = False
1300 utilities.assert_equals(
1301 expect=True,
1302 actual=rolesResults,
1303 onpass="No error in reading roles output",
1304 onfail="Error in reading roles from ONOS" )
1305
1306 main.step( "Check for consistency in roles from each controller" )
1307 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1308 main.log.info(
1309 "Switch roles are consistent across all ONOS nodes" )
1310 else:
1311 consistentMastership = False
1312 utilities.assert_equals(
1313 expect=True,
1314 actual=consistentMastership,
1315 onpass="Switch roles are consistent across all ONOS nodes",
1316 onfail="ONOS nodes have different views of switch roles" )
1317
1318 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001319 for i in range( len( main.activeNodes ) ):
1320 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001321 try:
1322 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001323 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001324 json.dumps(
1325 json.loads( ONOSMastership[ i ] ),
1326 sort_keys=True,
1327 indent=4,
1328 separators=( ',', ': ' ) ) )
1329 except ( ValueError, TypeError ):
1330 main.log.warn( repr( ONOSMastership[ i ] ) )
1331 elif rolesResults and consistentMastership:
1332 mastershipCheck = main.TRUE
1333 mastershipState = ONOSMastership[ 0 ]
1334
1335 main.step( "Get the intents from each controller" )
1336 global intentState
1337 intentState = []
1338 ONOSIntents = []
1339 intentCheck = main.FALSE
1340 consistentIntents = True
1341 intentsResults = True
1342 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001343 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001344 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001345 name="intents-" + str( i ),
1346 args=[],
1347 kwargs={ 'jsonFormat': True } )
1348 threads.append( t )
1349 t.start()
1350
1351 for t in threads:
1352 t.join()
1353 ONOSIntents.append( t.result )
1354
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001355 for i in range( len( ONOSIntents ) ):
1356 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001357 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001358 main.log.error( "Error in getting ONOS" + node + " intents" )
1359 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001360 repr( ONOSIntents[ i ] ) )
1361 intentsResults = False
1362 utilities.assert_equals(
1363 expect=True,
1364 actual=intentsResults,
1365 onpass="No error in reading intents output",
1366 onfail="Error in reading intents from ONOS" )
1367
1368 main.step( "Check for consistency in Intents from each controller" )
1369 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1370 main.log.info( "Intents are consistent across all ONOS " +
1371 "nodes" )
1372 else:
1373 consistentIntents = False
1374 main.log.error( "Intents not consistent" )
1375 utilities.assert_equals(
1376 expect=True,
1377 actual=consistentIntents,
1378 onpass="Intents are consistent across all ONOS nodes",
1379 onfail="ONOS nodes have different views of intents" )
1380
1381 if intentsResults:
1382 # Try to make it easy to figure out what is happening
1383 #
1384 # Intent ONOS1 ONOS2 ...
1385 # 0x01 INSTALLED INSTALLING
1386 # ... ... ...
1387 # ... ... ...
1388 title = " Id"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001389 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001390 title += " " * 10 + "ONOS" + str( n + 1 )
1391 main.log.warn( title )
1392 # get all intent keys in the cluster
1393 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001394 try:
1395 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001396 for nodeStr in ONOSIntents:
1397 node = json.loads( nodeStr )
1398 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001399 keys.append( intent.get( 'id' ) )
1400 keys = set( keys )
1401 # For each intent key, print the state on each node
1402 for key in keys:
1403 row = "%-13s" % key
1404 for nodeStr in ONOSIntents:
1405 node = json.loads( nodeStr )
1406 for intent in node:
1407 if intent.get( 'id', "Error" ) == key:
1408 row += "%-15s" % intent.get( 'state' )
1409 main.log.warn( row )
1410 # End of intent state table
1411 except ValueError as e:
1412 main.log.exception( e )
1413 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001414
1415 if intentsResults and not consistentIntents:
1416 # print the json objects
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001417 n = str( main.activeNodes[-1] + 1 )
1418 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001419 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1420 sort_keys=True,
1421 indent=4,
1422 separators=( ',', ': ' ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001423 for i in range( len( ONOSIntents ) ):
1424 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001425 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001426 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001427 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1428 sort_keys=True,
1429 indent=4,
1430 separators=( ',', ': ' ) ) )
1431 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001432 main.log.debug( "ONOS" + node + " intents match ONOS" +
1433 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001434 elif intentsResults and consistentIntents:
1435 intentCheck = main.TRUE
1436 intentState = ONOSIntents[ 0 ]
1437
1438 main.step( "Get the flows from each controller" )
1439 global flowState
1440 flowState = []
1441 ONOSFlows = []
1442 ONOSFlowsJson = []
1443 flowCheck = main.FALSE
1444 consistentFlows = True
1445 flowsResults = True
1446 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001447 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001448 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001449 name="flows-" + str( i ),
1450 args=[],
1451 kwargs={ 'jsonFormat': True } )
1452 threads.append( t )
1453 t.start()
1454
1455 # NOTE: Flows command can take some time to run
1456 time.sleep(30)
1457 for t in threads:
1458 t.join()
1459 result = t.result
1460 ONOSFlows.append( result )
1461
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001462 for i in range( len( ONOSFlows ) ):
1463 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001464 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1465 main.log.error( "Error in getting ONOS" + num + " flows" )
1466 main.log.warn( "ONOS" + num + " flows response: " +
1467 repr( ONOSFlows[ i ] ) )
1468 flowsResults = False
1469 ONOSFlowsJson.append( None )
1470 else:
1471 try:
1472 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1473 except ( ValueError, TypeError ):
1474 # FIXME: change this to log.error?
1475 main.log.exception( "Error in parsing ONOS" + num +
1476 " response as json." )
1477 main.log.error( repr( ONOSFlows[ i ] ) )
1478 ONOSFlowsJson.append( None )
1479 flowsResults = False
1480 utilities.assert_equals(
1481 expect=True,
1482 actual=flowsResults,
1483 onpass="No error in reading flows output",
1484 onfail="Error in reading flows from ONOS" )
1485
1486 main.step( "Check for consistency in Flows from each controller" )
1487 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1488 if all( tmp ):
1489 main.log.info( "Flow count is consistent across all ONOS nodes" )
1490 else:
1491 consistentFlows = False
1492 utilities.assert_equals(
1493 expect=True,
1494 actual=consistentFlows,
1495 onpass="The flow count is consistent across all ONOS nodes",
1496 onfail="ONOS nodes have different flow counts" )
1497
1498 if flowsResults and not consistentFlows:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001499 for i in range( len( ONOSFlows ) ):
1500 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001501 try:
1502 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001503 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001504 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1505 indent=4, separators=( ',', ': ' ) ) )
1506 except ( ValueError, TypeError ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001507 main.log.warn( "ONOS" + node + " flows: " +
1508 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001509 elif flowsResults and consistentFlows:
1510 flowCheck = main.TRUE
1511 flowState = ONOSFlows[ 0 ]
1512
1513 main.step( "Get the OF Table entries" )
1514 global flows
1515 flows = []
1516 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001517 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001518 if flowCheck == main.FALSE:
1519 for table in flows:
1520 main.log.warn( table )
1521 # TODO: Compare switch flow tables with ONOS flow tables
1522
1523 main.step( "Start continuous pings" )
1524 main.Mininet2.pingLong(
1525 src=main.params[ 'PING' ][ 'source1' ],
1526 target=main.params[ 'PING' ][ 'target1' ],
1527 pingTime=500 )
1528 main.Mininet2.pingLong(
1529 src=main.params[ 'PING' ][ 'source2' ],
1530 target=main.params[ 'PING' ][ 'target2' ],
1531 pingTime=500 )
1532 main.Mininet2.pingLong(
1533 src=main.params[ 'PING' ][ 'source3' ],
1534 target=main.params[ 'PING' ][ 'target3' ],
1535 pingTime=500 )
1536 main.Mininet2.pingLong(
1537 src=main.params[ 'PING' ][ 'source4' ],
1538 target=main.params[ 'PING' ][ 'target4' ],
1539 pingTime=500 )
1540 main.Mininet2.pingLong(
1541 src=main.params[ 'PING' ][ 'source5' ],
1542 target=main.params[ 'PING' ][ 'target5' ],
1543 pingTime=500 )
1544 main.Mininet2.pingLong(
1545 src=main.params[ 'PING' ][ 'source6' ],
1546 target=main.params[ 'PING' ][ 'target6' ],
1547 pingTime=500 )
1548 main.Mininet2.pingLong(
1549 src=main.params[ 'PING' ][ 'source7' ],
1550 target=main.params[ 'PING' ][ 'target7' ],
1551 pingTime=500 )
1552 main.Mininet2.pingLong(
1553 src=main.params[ 'PING' ][ 'source8' ],
1554 target=main.params[ 'PING' ][ 'target8' ],
1555 pingTime=500 )
1556 main.Mininet2.pingLong(
1557 src=main.params[ 'PING' ][ 'source9' ],
1558 target=main.params[ 'PING' ][ 'target9' ],
1559 pingTime=500 )
1560 main.Mininet2.pingLong(
1561 src=main.params[ 'PING' ][ 'source10' ],
1562 target=main.params[ 'PING' ][ 'target10' ],
1563 pingTime=500 )
1564
1565 main.step( "Collecting topology information from ONOS" )
1566 devices = []
1567 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001568 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001569 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001570 name="devices-" + str( i ),
1571 args=[ ] )
1572 threads.append( t )
1573 t.start()
1574
1575 for t in threads:
1576 t.join()
1577 devices.append( t.result )
1578 hosts = []
1579 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001580 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001581 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001582 name="hosts-" + str( i ),
1583 args=[ ] )
1584 threads.append( t )
1585 t.start()
1586
1587 for t in threads:
1588 t.join()
1589 try:
1590 hosts.append( json.loads( t.result ) )
1591 except ( ValueError, TypeError ):
1592 # FIXME: better handling of this, print which node
1593 # Maybe use thread name?
1594 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001595 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001596 hosts.append( None )
1597
1598 ports = []
1599 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001600 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001601 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001602 name="ports-" + str( i ),
1603 args=[ ] )
1604 threads.append( t )
1605 t.start()
1606
1607 for t in threads:
1608 t.join()
1609 ports.append( t.result )
1610 links = []
1611 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001612 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001613 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001614 name="links-" + str( i ),
1615 args=[ ] )
1616 threads.append( t )
1617 t.start()
1618
1619 for t in threads:
1620 t.join()
1621 links.append( t.result )
1622 clusters = []
1623 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001624 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001625 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001626 name="clusters-" + str( i ),
1627 args=[ ] )
1628 threads.append( t )
1629 t.start()
1630
1631 for t in threads:
1632 t.join()
1633 clusters.append( t.result )
1634 # Compare json objects for hosts and dataplane clusters
1635
1636 # hosts
1637 main.step( "Host view is consistent across ONOS nodes" )
1638 consistentHostsResult = main.TRUE
1639 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001640 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001641 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001642 if hosts[ controller ] == hosts[ 0 ]:
1643 continue
1644 else: # hosts not consistent
1645 main.log.error( "hosts from ONOS" +
1646 controllerStr +
1647 " is inconsistent with ONOS1" )
1648 main.log.warn( repr( hosts[ controller ] ) )
1649 consistentHostsResult = main.FALSE
1650
1651 else:
1652 main.log.error( "Error in getting ONOS hosts from ONOS" +
1653 controllerStr )
1654 consistentHostsResult = main.FALSE
1655 main.log.warn( "ONOS" + controllerStr +
1656 " hosts response: " +
1657 repr( hosts[ controller ] ) )
1658 utilities.assert_equals(
1659 expect=main.TRUE,
1660 actual=consistentHostsResult,
1661 onpass="Hosts view is consistent across all ONOS nodes",
1662 onfail="ONOS nodes have different views of hosts" )
1663
1664 main.step( "Each host has an IP address" )
1665 ipResult = main.TRUE
1666 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001667 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001668 if hosts[ controller ]:
1669 for host in hosts[ controller ]:
1670 if not host.get( 'ipAddresses', [ ] ):
1671 main.log.error( "Error with host ips on controller" +
1672 controllerStr + ": " + str( host ) )
1673 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001674 utilities.assert_equals(
1675 expect=main.TRUE,
1676 actual=ipResult,
1677 onpass="The ips of the hosts aren't empty",
1678 onfail="The ip of at least one host is missing" )
1679
1680 # Strongly connected clusters of devices
1681 main.step( "Cluster view is consistent across ONOS nodes" )
1682 consistentClustersResult = main.TRUE
1683 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001684 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001685 if "Error" not in clusters[ controller ]:
1686 if clusters[ controller ] == clusters[ 0 ]:
1687 continue
1688 else: # clusters not consistent
1689 main.log.error( "clusters from ONOS" + controllerStr +
1690 " is inconsistent with ONOS1" )
1691 consistentClustersResult = main.FALSE
1692
1693 else:
1694 main.log.error( "Error in getting dataplane clusters " +
1695 "from ONOS" + controllerStr )
1696 consistentClustersResult = main.FALSE
1697 main.log.warn( "ONOS" + controllerStr +
1698 " clusters response: " +
1699 repr( clusters[ controller ] ) )
1700 utilities.assert_equals(
1701 expect=main.TRUE,
1702 actual=consistentClustersResult,
1703 onpass="Clusters view is consistent across all ONOS nodes",
1704 onfail="ONOS nodes have different views of clusters" )
Jon Hall172b7ba2016-04-07 18:12:20 -07001705 if consistentClustersResult != main.TRUE:
1706 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07001707 # there should always only be one cluster
1708 main.step( "Cluster view correct across ONOS nodes" )
1709 try:
1710 numClusters = len( json.loads( clusters[ 0 ] ) )
1711 except ( ValueError, TypeError ):
1712 main.log.exception( "Error parsing clusters[0]: " +
1713 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001714 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001715 clusterResults = main.FALSE
1716 if numClusters == 1:
1717 clusterResults = main.TRUE
1718 utilities.assert_equals(
1719 expect=1,
1720 actual=numClusters,
1721 onpass="ONOS shows 1 SCC",
1722 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1723
1724 main.step( "Comparing ONOS topology to MN" )
1725 devicesResults = main.TRUE
1726 linksResults = main.TRUE
1727 hostsResults = main.TRUE
1728 mnSwitches = main.Mininet1.getSwitches()
1729 mnLinks = main.Mininet1.getLinks()
1730 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001731 for controller in main.activeNodes:
1732 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001733 if devices[ controller ] and ports[ controller ] and\
1734 "Error" not in devices[ controller ] and\
1735 "Error" not in ports[ controller ]:
Jon Hall6e709752016-02-01 13:38:46 -08001736 currentDevicesResult = main.Mininet1.compareSwitches(
1737 mnSwitches,
1738 json.loads( devices[ controller ] ),
1739 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001740 else:
1741 currentDevicesResult = main.FALSE
1742 utilities.assert_equals( expect=main.TRUE,
1743 actual=currentDevicesResult,
1744 onpass="ONOS" + controllerStr +
1745 " Switches view is correct",
1746 onfail="ONOS" + controllerStr +
1747 " Switches view is incorrect" )
1748 if links[ controller ] and "Error" not in links[ controller ]:
1749 currentLinksResult = main.Mininet1.compareLinks(
1750 mnSwitches, mnLinks,
1751 json.loads( links[ controller ] ) )
1752 else:
1753 currentLinksResult = main.FALSE
1754 utilities.assert_equals( expect=main.TRUE,
1755 actual=currentLinksResult,
1756 onpass="ONOS" + controllerStr +
1757 " links view is correct",
1758 onfail="ONOS" + controllerStr +
1759 " links view is incorrect" )
1760
Jon Hall657cdf62015-12-17 14:40:51 -08001761 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001762 currentHostsResult = main.Mininet1.compareHosts(
1763 mnHosts,
1764 hosts[ controller ] )
1765 else:
1766 currentHostsResult = main.FALSE
1767 utilities.assert_equals( expect=main.TRUE,
1768 actual=currentHostsResult,
1769 onpass="ONOS" + controllerStr +
1770 " hosts exist in Mininet",
1771 onfail="ONOS" + controllerStr +
1772 " hosts don't match Mininet" )
1773
1774 devicesResults = devicesResults and currentDevicesResult
1775 linksResults = linksResults and currentLinksResult
1776 hostsResults = hostsResults and currentHostsResult
1777
1778 main.step( "Device information is correct" )
1779 utilities.assert_equals(
1780 expect=main.TRUE,
1781 actual=devicesResults,
1782 onpass="Device information is correct",
1783 onfail="Device information is incorrect" )
1784
1785 main.step( "Links are correct" )
1786 utilities.assert_equals(
1787 expect=main.TRUE,
1788 actual=linksResults,
1789 onpass="Link are correct",
1790 onfail="Links are incorrect" )
1791
1792 main.step( "Hosts are correct" )
1793 utilities.assert_equals(
1794 expect=main.TRUE,
1795 actual=hostsResults,
1796 onpass="Hosts are correct",
1797 onfail="Hosts are incorrect" )
1798
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001799 def CASE61( self, main ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001800 """
1801 The Failure case.
1802 """
Jon Halle1a3b752015-07-22 13:02:46 -07001803 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001804 assert main, "main not defined"
1805 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001806 assert main.CLIs, "main.CLIs not defined"
1807 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001808 main.case( "Kill minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -07001809
1810 main.step( "Checking ONOS Logs for errors" )
1811 for node in main.nodes:
1812 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1813 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1814
Jon Hall3b489db2015-10-05 14:38:37 -07001815 n = len( main.nodes ) # Number of nodes
1816 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
1817 main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
1818 if n > 3:
1819 main.kill.append( p - 1 )
1820 # NOTE: This only works for cluster sizes of 3,5, or 7.
1821
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001822 main.step( "Kill " + str( len( main.kill ) ) + " ONOS nodes" )
Jon Hall3b489db2015-10-05 14:38:37 -07001823 killResults = main.TRUE
1824 for i in main.kill:
1825 killResults = killResults and\
1826 main.ONOSbench.onosKill( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001827 main.activeNodes.remove( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001828 utilities.assert_equals( expect=main.TRUE, actual=killResults,
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001829 onpass="ONOS nodes killed successfully",
1830 onfail="ONOS nodes NOT successfully killed" )
1831
1832 def CASE62( self, main ):
1833 """
1834 The bring up stopped nodes
1835 """
1836 import time
1837 assert main.numCtrls, "main.numCtrls not defined"
1838 assert main, "main not defined"
1839 assert utilities.assert_equals, "utilities.assert_equals not defined"
1840 assert main.CLIs, "main.CLIs not defined"
1841 assert main.nodes, "main.nodes not defined"
1842 assert main.kill, "main.kill not defined"
1843 main.case( "Restart minority of ONOS nodes" )
1844
1845 main.step( "Restarting " + str( len( main.kill ) ) + " ONOS nodes" )
1846 startResults = main.TRUE
1847 restartTime = time.time()
1848 for i in main.kill:
1849 startResults = startResults and\
1850 main.ONOSbench.onosStart( main.nodes[i].ip_address )
1851 utilities.assert_equals( expect=main.TRUE, actual=startResults,
1852 onpass="ONOS nodes started successfully",
1853 onfail="ONOS nodes NOT successfully started" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001854
1855 main.step( "Checking if ONOS is up yet" )
1856 count = 0
1857 onosIsupResult = main.FALSE
1858 while onosIsupResult == main.FALSE and count < 10:
Jon Hall3b489db2015-10-05 14:38:37 -07001859 onosIsupResult = main.TRUE
1860 for i in main.kill:
1861 onosIsupResult = onosIsupResult and\
1862 main.ONOSbench.isup( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001863 count = count + 1
Jon Hall5cf14d52015-07-16 12:15:19 -07001864 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1865 onpass="ONOS restarted successfully",
1866 onfail="ONOS restart NOT successful" )
1867
Jon Halle1a3b752015-07-22 13:02:46 -07001868 main.step( "Restarting ONOS main.CLIs" )
Jon Hall3b489db2015-10-05 14:38:37 -07001869 cliResults = main.TRUE
1870 for i in main.kill:
1871 cliResults = cliResults and\
1872 main.CLIs[i].startOnosCli( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001873 main.activeNodes.append( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001874 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1875 onpass="ONOS cli restarted",
1876 onfail="ONOS cli did not restart" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001877 main.activeNodes.sort()
1878 try:
1879 assert list( set( main.activeNodes ) ) == main.activeNodes,\
1880 "List of active nodes has duplicates, this likely indicates something was run out of order"
1881 except AssertionError:
1882 main.log.exception( "" )
1883 main.cleanup()
1884 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -07001885
1886 # Grab the time of restart so we chan check how long the gossip
1887 # protocol has had time to work
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001888 main.restartTime = time.time() - restartTime
Jon Hall5cf14d52015-07-16 12:15:19 -07001889 main.log.debug( "Restart time: " + str( main.restartTime ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001890 # TODO: MAke this configurable. Also, we are breaking the above timer
1891 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001892 node = main.activeNodes[0]
1893 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1894 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1895 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001896
Jon Halla440e872016-03-31 15:15:50 -07001897 main.step( "Rerun for election on the node(s) that were killed" )
1898 runResults = main.TRUE
1899 for i in main.kill:
1900 runResults = runResults and\
1901 main.CLIs[i].electionTestRun()
1902 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1903 onpass="ONOS nodes reran for election topic",
1904 onfail="Errror rerunning for election" )
1905
Jon Hall5cf14d52015-07-16 12:15:19 -07001906 def CASE7( self, main ):
1907 """
1908 Check state after ONOS failure
1909 """
1910 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001911 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001912 assert main, "main not defined"
1913 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001914 assert main.CLIs, "main.CLIs not defined"
1915 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001916 try:
1917 main.kill
1918 except AttributeError:
1919 main.kill = []
1920
Jon Hall5cf14d52015-07-16 12:15:19 -07001921 main.case( "Running ONOS Constant State Tests" )
1922
1923 main.step( "Check that each switch has a master" )
1924 # Assert that each device has a master
1925 rolesNotNull = main.TRUE
1926 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001927 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001928 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001929 name="rolesNotNull-" + str( i ),
1930 args=[ ] )
1931 threads.append( t )
1932 t.start()
1933
1934 for t in threads:
1935 t.join()
1936 rolesNotNull = rolesNotNull and t.result
1937 utilities.assert_equals(
1938 expect=main.TRUE,
1939 actual=rolesNotNull,
1940 onpass="Each device has a master",
1941 onfail="Some devices don't have a master assigned" )
1942
1943 main.step( "Read device roles from ONOS" )
1944 ONOSMastership = []
Jon Halla440e872016-03-31 15:15:50 -07001945 mastershipCheck = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001946 consistentMastership = True
1947 rolesResults = True
1948 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001949 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001950 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001951 name="roles-" + str( i ),
1952 args=[] )
1953 threads.append( t )
1954 t.start()
1955
1956 for t in threads:
1957 t.join()
1958 ONOSMastership.append( t.result )
1959
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001960 for i in range( len( ONOSMastership ) ):
1961 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001962 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001963 main.log.error( "Error in getting ONOS" + node + " roles" )
1964 main.log.warn( "ONOS" + node + " mastership response: " +
1965 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001966 rolesResults = False
1967 utilities.assert_equals(
1968 expect=True,
1969 actual=rolesResults,
1970 onpass="No error in reading roles output",
1971 onfail="Error in reading roles from ONOS" )
1972
1973 main.step( "Check for consistency in roles from each controller" )
1974 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1975 main.log.info(
1976 "Switch roles are consistent across all ONOS nodes" )
1977 else:
1978 consistentMastership = False
1979 utilities.assert_equals(
1980 expect=True,
1981 actual=consistentMastership,
1982 onpass="Switch roles are consistent across all ONOS nodes",
1983 onfail="ONOS nodes have different views of switch roles" )
1984
1985 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001986 for i in range( len( ONOSMastership ) ):
1987 node = str( main.activeNodes[i] + 1 )
1988 main.log.warn( "ONOS" + node + " roles: ",
1989 json.dumps( json.loads( ONOSMastership[ i ] ),
1990 sort_keys=True,
1991 indent=4,
1992 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001993 elif rolesResults and consistentMastership:
1994 mastershipCheck = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07001995
1996 # NOTE: we expect mastership to change on controller failure
Jon Hall5cf14d52015-07-16 12:15:19 -07001997
1998 main.step( "Get the intents and compare across all nodes" )
1999 ONOSIntents = []
2000 intentCheck = main.FALSE
2001 consistentIntents = True
2002 intentsResults = True
2003 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002004 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002005 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07002006 name="intents-" + str( i ),
2007 args=[],
2008 kwargs={ 'jsonFormat': True } )
2009 threads.append( t )
2010 t.start()
2011
2012 for t in threads:
2013 t.join()
2014 ONOSIntents.append( t.result )
2015
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002016 for i in range( len( ONOSIntents) ):
2017 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002018 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002019 main.log.error( "Error in getting ONOS" + node + " intents" )
2020 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002021 repr( ONOSIntents[ i ] ) )
2022 intentsResults = False
2023 utilities.assert_equals(
2024 expect=True,
2025 actual=intentsResults,
2026 onpass="No error in reading intents output",
2027 onfail="Error in reading intents from ONOS" )
2028
2029 main.step( "Check for consistency in Intents from each controller" )
2030 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2031 main.log.info( "Intents are consistent across all ONOS " +
2032 "nodes" )
2033 else:
2034 consistentIntents = False
2035
2036 # Try to make it easy to figure out what is happening
2037 #
2038 # Intent ONOS1 ONOS2 ...
2039 # 0x01 INSTALLED INSTALLING
2040 # ... ... ...
2041 # ... ... ...
2042 title = " ID"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002043 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002044 title += " " * 10 + "ONOS" + str( n + 1 )
2045 main.log.warn( title )
2046 # get all intent keys in the cluster
2047 keys = []
2048 for nodeStr in ONOSIntents:
2049 node = json.loads( nodeStr )
2050 for intent in node:
2051 keys.append( intent.get( 'id' ) )
2052 keys = set( keys )
2053 for key in keys:
2054 row = "%-13s" % key
2055 for nodeStr in ONOSIntents:
2056 node = json.loads( nodeStr )
2057 for intent in node:
2058 if intent.get( 'id' ) == key:
2059 row += "%-15s" % intent.get( 'state' )
2060 main.log.warn( row )
2061 # End table view
2062
2063 utilities.assert_equals(
2064 expect=True,
2065 actual=consistentIntents,
2066 onpass="Intents are consistent across all ONOS nodes",
2067 onfail="ONOS nodes have different views of intents" )
2068 intentStates = []
2069 for node in ONOSIntents: # Iter through ONOS nodes
2070 nodeStates = []
2071 # Iter through intents of a node
2072 try:
2073 for intent in json.loads( node ):
2074 nodeStates.append( intent[ 'state' ] )
2075 except ( ValueError, TypeError ):
2076 main.log.exception( "Error in parsing intents" )
2077 main.log.error( repr( node ) )
2078 intentStates.append( nodeStates )
2079 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2080 main.log.info( dict( out ) )
2081
2082 if intentsResults and not consistentIntents:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002083 for i in range( len( main.activeNodes ) ):
2084 node = str( main.activeNodes[i] + 1 )
2085 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002086 main.log.warn( json.dumps(
2087 json.loads( ONOSIntents[ i ] ),
2088 sort_keys=True,
2089 indent=4,
2090 separators=( ',', ': ' ) ) )
2091 elif intentsResults and consistentIntents:
2092 intentCheck = main.TRUE
2093
2094 # NOTE: Store has no durability, so intents are lost across system
2095 # restarts
2096 main.step( "Compare current intents with intents before the failure" )
2097 # NOTE: this requires case 5 to pass for intentState to be set.
2098 # maybe we should stop the test if that fails?
2099 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002100 try:
2101 intentState
2102 except NameError:
2103 main.log.warn( "No previous intent state was saved" )
2104 else:
2105 if intentState and intentState == ONOSIntents[ 0 ]:
2106 sameIntents = main.TRUE
2107 main.log.info( "Intents are consistent with before failure" )
2108 # TODO: possibly the states have changed? we may need to figure out
2109 # what the acceptable states are
2110 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2111 sameIntents = main.TRUE
2112 try:
2113 before = json.loads( intentState )
2114 after = json.loads( ONOSIntents[ 0 ] )
2115 for intent in before:
2116 if intent not in after:
2117 sameIntents = main.FALSE
2118 main.log.debug( "Intent is not currently in ONOS " +
2119 "(at least in the same form):" )
2120 main.log.debug( json.dumps( intent ) )
2121 except ( ValueError, TypeError ):
2122 main.log.exception( "Exception printing intents" )
2123 main.log.debug( repr( ONOSIntents[0] ) )
2124 main.log.debug( repr( intentState ) )
2125 if sameIntents == main.FALSE:
2126 try:
2127 main.log.debug( "ONOS intents before: " )
2128 main.log.debug( json.dumps( json.loads( intentState ),
2129 sort_keys=True, indent=4,
2130 separators=( ',', ': ' ) ) )
2131 main.log.debug( "Current ONOS intents: " )
2132 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2133 sort_keys=True, indent=4,
2134 separators=( ',', ': ' ) ) )
2135 except ( ValueError, TypeError ):
2136 main.log.exception( "Exception printing intents" )
2137 main.log.debug( repr( ONOSIntents[0] ) )
2138 main.log.debug( repr( intentState ) )
2139 utilities.assert_equals(
2140 expect=main.TRUE,
2141 actual=sameIntents,
2142 onpass="Intents are consistent with before failure",
2143 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002144 intentCheck = intentCheck and sameIntents
2145
2146 main.step( "Get the OF Table entries and compare to before " +
2147 "component failure" )
2148 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002149 for i in range( 28 ):
2150 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002151 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002152 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2153 FlowTables = FlowTables and curSwitch
2154 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002155 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002156 utilities.assert_equals(
2157 expect=main.TRUE,
2158 actual=FlowTables,
2159 onpass="No changes were found in the flow tables",
2160 onfail="Changes were found in the flow tables" )
2161
2162 main.Mininet2.pingLongKill()
2163 '''
2164 main.step( "Check the continuous pings to ensure that no packets " +
2165 "were dropped during component failure" )
2166 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2167 main.params[ 'TESTONIP' ] )
2168 LossInPings = main.FALSE
2169 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2170 for i in range( 8, 18 ):
2171 main.log.info(
2172 "Checking for a loss in pings along flow from s" +
2173 str( i ) )
2174 LossInPings = main.Mininet2.checkForLoss(
2175 "/tmp/ping.h" +
2176 str( i ) ) or LossInPings
2177 if LossInPings == main.TRUE:
2178 main.log.info( "Loss in ping detected" )
2179 elif LossInPings == main.ERROR:
2180 main.log.info( "There are multiple mininet process running" )
2181 elif LossInPings == main.FALSE:
2182 main.log.info( "No Loss in the pings" )
2183 main.log.info( "No loss of dataplane connectivity" )
2184 utilities.assert_equals(
2185 expect=main.FALSE,
2186 actual=LossInPings,
2187 onpass="No Loss of connectivity",
2188 onfail="Loss of dataplane connectivity detected" )
2189 '''
2190
2191 main.step( "Leadership Election is still functional" )
2192 # Test of LeadershipElection
2193 leaderList = []
Jon Hall5cf14d52015-07-16 12:15:19 -07002194
Jon Hall3b489db2015-10-05 14:38:37 -07002195 restarted = []
2196 for i in main.kill:
2197 restarted.append( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002198 leaderResult = main.TRUE
Jon Hall3b489db2015-10-05 14:38:37 -07002199
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002200 for i in main.activeNodes:
2201 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002202 leaderN = cli.electionTestLeader()
2203 leaderList.append( leaderN )
2204 if leaderN == main.FALSE:
2205 # error in response
2206 main.log.error( "Something is wrong with " +
2207 "electionTestLeader function, check the" +
2208 " error logs" )
2209 leaderResult = main.FALSE
2210 elif leaderN is None:
2211 main.log.error( cli.name +
2212 " shows no leader for the election-app was" +
2213 " elected after the old one died" )
2214 leaderResult = main.FALSE
2215 elif leaderN in restarted:
2216 main.log.error( cli.name + " shows " + str( leaderN ) +
2217 " as leader for the election-app, but it " +
2218 "was restarted" )
2219 leaderResult = main.FALSE
2220 if len( set( leaderList ) ) != 1:
2221 leaderResult = main.FALSE
2222 main.log.error(
2223 "Inconsistent view of leader for the election test app" )
2224 # TODO: print the list
2225 utilities.assert_equals(
2226 expect=main.TRUE,
2227 actual=leaderResult,
2228 onpass="Leadership election passed",
2229 onfail="Something went wrong with Leadership election" )
2230
2231 def CASE8( self, main ):
2232 """
2233 Compare topo
2234 """
2235 import json
2236 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002237 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002238 assert main, "main not defined"
2239 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002240 assert main.CLIs, "main.CLIs not defined"
2241 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002242
2243 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002244 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002245 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002246 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002247 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002248 elapsed = 0
2249 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002250 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002251 startTime = time.time()
2252 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002253 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002254 devicesResults = main.TRUE
2255 linksResults = main.TRUE
2256 hostsResults = main.TRUE
2257 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002258 count += 1
2259 cliStart = time.time()
2260 devices = []
2261 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002262 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002263 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002264 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002265 args=[ main.CLIs[i].devices, [ None ] ],
2266 kwargs= { 'sleep': 5, 'attempts': 5,
2267 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002268 threads.append( t )
2269 t.start()
2270
2271 for t in threads:
2272 t.join()
2273 devices.append( t.result )
2274 hosts = []
2275 ipResult = main.TRUE
2276 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002277 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002278 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002279 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002280 args=[ main.CLIs[i].hosts, [ None ] ],
2281 kwargs= { 'sleep': 5, 'attempts': 5,
2282 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002283 threads.append( t )
2284 t.start()
2285
2286 for t in threads:
2287 t.join()
2288 try:
2289 hosts.append( json.loads( t.result ) )
2290 except ( ValueError, TypeError ):
2291 main.log.exception( "Error parsing hosts results" )
2292 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002293 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002294 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002295 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002296 if hosts[ controller ]:
2297 for host in hosts[ controller ]:
2298 if host is None or host.get( 'ipAddresses', [] ) == []:
2299 main.log.error(
2300 "Error with host ipAddresses on controller" +
2301 controllerStr + ": " + str( host ) )
2302 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002303 ports = []
2304 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002305 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002306 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002307 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002308 args=[ main.CLIs[i].ports, [ None ] ],
2309 kwargs= { 'sleep': 5, 'attempts': 5,
2310 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002311 threads.append( t )
2312 t.start()
2313
2314 for t in threads:
2315 t.join()
2316 ports.append( t.result )
2317 links = []
2318 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002319 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002320 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002321 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002322 args=[ main.CLIs[i].links, [ None ] ],
2323 kwargs= { 'sleep': 5, 'attempts': 5,
2324 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002325 threads.append( t )
2326 t.start()
2327
2328 for t in threads:
2329 t.join()
2330 links.append( t.result )
2331 clusters = []
2332 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002333 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002334 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002335 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002336 args=[ main.CLIs[i].clusters, [ None ] ],
2337 kwargs= { 'sleep': 5, 'attempts': 5,
2338 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002339 threads.append( t )
2340 t.start()
2341
2342 for t in threads:
2343 t.join()
2344 clusters.append( t.result )
2345
2346 elapsed = time.time() - startTime
2347 cliTime = time.time() - cliStart
2348 print "Elapsed time: " + str( elapsed )
2349 print "CLI time: " + str( cliTime )
2350
Jon Hall6e709752016-02-01 13:38:46 -08002351 if all( e is None for e in devices ) and\
2352 all( e is None for e in hosts ) and\
2353 all( e is None for e in ports ) and\
2354 all( e is None for e in links ) and\
2355 all( e is None for e in clusters ):
2356 topoFailMsg = "Could not get topology from ONOS"
2357 main.log.error( topoFailMsg )
2358 continue # Try again, No use trying to compare
2359
Jon Hall5cf14d52015-07-16 12:15:19 -07002360 mnSwitches = main.Mininet1.getSwitches()
2361 mnLinks = main.Mininet1.getLinks()
2362 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002363 for controller in range( len( main.activeNodes ) ):
2364 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002365 if devices[ controller ] and ports[ controller ] and\
2366 "Error" not in devices[ controller ] and\
2367 "Error" not in ports[ controller ]:
2368
Jon Hallc6793552016-01-19 14:18:37 -08002369 try:
2370 currentDevicesResult = main.Mininet1.compareSwitches(
2371 mnSwitches,
2372 json.loads( devices[ controller ] ),
2373 json.loads( ports[ controller ] ) )
2374 except ( TypeError, ValueError ) as e:
2375 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2376 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002377 else:
2378 currentDevicesResult = main.FALSE
2379 utilities.assert_equals( expect=main.TRUE,
2380 actual=currentDevicesResult,
2381 onpass="ONOS" + controllerStr +
2382 " Switches view is correct",
2383 onfail="ONOS" + controllerStr +
2384 " Switches view is incorrect" )
2385
2386 if links[ controller ] and "Error" not in links[ controller ]:
2387 currentLinksResult = main.Mininet1.compareLinks(
2388 mnSwitches, mnLinks,
2389 json.loads( links[ controller ] ) )
2390 else:
2391 currentLinksResult = main.FALSE
2392 utilities.assert_equals( expect=main.TRUE,
2393 actual=currentLinksResult,
2394 onpass="ONOS" + controllerStr +
2395 " links view is correct",
2396 onfail="ONOS" + controllerStr +
2397 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002398 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002399 currentHostsResult = main.Mininet1.compareHosts(
2400 mnHosts,
2401 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002402 elif hosts[ controller ] == []:
2403 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002404 else:
2405 currentHostsResult = main.FALSE
2406 utilities.assert_equals( expect=main.TRUE,
2407 actual=currentHostsResult,
2408 onpass="ONOS" + controllerStr +
2409 " hosts exist in Mininet",
2410 onfail="ONOS" + controllerStr +
2411 " hosts don't match Mininet" )
2412 # CHECKING HOST ATTACHMENT POINTS
2413 hostAttachment = True
2414 zeroHosts = False
2415 # FIXME: topo-HA/obelisk specific mappings:
2416 # key is mac and value is dpid
2417 mappings = {}
2418 for i in range( 1, 29 ): # hosts 1 through 28
2419 # set up correct variables:
2420 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2421 if i == 1:
2422 deviceId = "1000".zfill(16)
2423 elif i == 2:
2424 deviceId = "2000".zfill(16)
2425 elif i == 3:
2426 deviceId = "3000".zfill(16)
2427 elif i == 4:
2428 deviceId = "3004".zfill(16)
2429 elif i == 5:
2430 deviceId = "5000".zfill(16)
2431 elif i == 6:
2432 deviceId = "6000".zfill(16)
2433 elif i == 7:
2434 deviceId = "6007".zfill(16)
2435 elif i >= 8 and i <= 17:
2436 dpid = '3' + str( i ).zfill( 3 )
2437 deviceId = dpid.zfill(16)
2438 elif i >= 18 and i <= 27:
2439 dpid = '6' + str( i ).zfill( 3 )
2440 deviceId = dpid.zfill(16)
2441 elif i == 28:
2442 deviceId = "2800".zfill(16)
2443 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002444 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002445 if hosts[ controller ] == []:
2446 main.log.warn( "There are no hosts discovered" )
2447 zeroHosts = True
2448 else:
2449 for host in hosts[ controller ]:
2450 mac = None
2451 location = None
2452 device = None
2453 port = None
2454 try:
2455 mac = host.get( 'mac' )
2456 assert mac, "mac field could not be found for this host object"
2457
2458 location = host.get( 'location' )
2459 assert location, "location field could not be found for this host object"
2460
2461 # Trim the protocol identifier off deviceId
2462 device = str( location.get( 'elementId' ) ).split(':')[1]
2463 assert device, "elementId field could not be found for this host location object"
2464
2465 port = location.get( 'port' )
2466 assert port, "port field could not be found for this host location object"
2467
2468 # Now check if this matches where they should be
2469 if mac and device and port:
2470 if str( port ) != "1":
2471 main.log.error( "The attachment port is incorrect for " +
2472 "host " + str( mac ) +
2473 ". Expected: 1 Actual: " + str( port) )
2474 hostAttachment = False
2475 if device != mappings[ str( mac ) ]:
2476 main.log.error( "The attachment device is incorrect for " +
2477 "host " + str( mac ) +
2478 ". Expected: " + mappings[ str( mac ) ] +
2479 " Actual: " + device )
2480 hostAttachment = False
2481 else:
2482 hostAttachment = False
2483 except AssertionError:
2484 main.log.exception( "Json object not as expected" )
2485 main.log.error( repr( host ) )
2486 hostAttachment = False
2487 else:
2488 main.log.error( "No hosts json output or \"Error\"" +
2489 " in output. hosts = " +
2490 repr( hosts[ controller ] ) )
2491 if zeroHosts is False:
2492 hostAttachment = True
2493
2494 # END CHECKING HOST ATTACHMENT POINTS
2495 devicesResults = devicesResults and currentDevicesResult
2496 linksResults = linksResults and currentLinksResult
2497 hostsResults = hostsResults and currentHostsResult
2498 hostAttachmentResults = hostAttachmentResults and\
2499 hostAttachment
Jon Halla440e872016-03-31 15:15:50 -07002500 topoResult = ( devicesResults and linksResults
2501 and hostsResults and ipResult and
2502 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002503 utilities.assert_equals( expect=True,
2504 actual=topoResult,
2505 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002506 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002507 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002508
2509 # Compare json objects for hosts and dataplane clusters
2510
2511 # hosts
2512 main.step( "Hosts view is consistent across all ONOS nodes" )
2513 consistentHostsResult = main.TRUE
2514 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002515 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002516 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002517 if hosts[ controller ] == hosts[ 0 ]:
2518 continue
2519 else: # hosts not consistent
2520 main.log.error( "hosts from ONOS" + controllerStr +
2521 " is inconsistent with ONOS1" )
2522 main.log.warn( repr( hosts[ controller ] ) )
2523 consistentHostsResult = main.FALSE
2524
2525 else:
2526 main.log.error( "Error in getting ONOS hosts from ONOS" +
2527 controllerStr )
2528 consistentHostsResult = main.FALSE
2529 main.log.warn( "ONOS" + controllerStr +
2530 " hosts response: " +
2531 repr( hosts[ controller ] ) )
2532 utilities.assert_equals(
2533 expect=main.TRUE,
2534 actual=consistentHostsResult,
2535 onpass="Hosts view is consistent across all ONOS nodes",
2536 onfail="ONOS nodes have different views of hosts" )
2537
2538 main.step( "Hosts information is correct" )
2539 hostsResults = hostsResults and ipResult
2540 utilities.assert_equals(
2541 expect=main.TRUE,
2542 actual=hostsResults,
2543 onpass="Host information is correct",
2544 onfail="Host information is incorrect" )
2545
2546 main.step( "Host attachment points to the network" )
2547 utilities.assert_equals(
2548 expect=True,
2549 actual=hostAttachmentResults,
2550 onpass="Hosts are correctly attached to the network",
2551 onfail="ONOS did not correctly attach hosts to the network" )
2552
2553 # Strongly connected clusters of devices
2554 main.step( "Clusters view is consistent across all ONOS nodes" )
2555 consistentClustersResult = main.TRUE
2556 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002557 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002558 if "Error" not in clusters[ controller ]:
2559 if clusters[ controller ] == clusters[ 0 ]:
2560 continue
2561 else: # clusters not consistent
2562 main.log.error( "clusters from ONOS" +
2563 controllerStr +
2564 " is inconsistent with ONOS1" )
2565 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002566 else:
2567 main.log.error( "Error in getting dataplane clusters " +
2568 "from ONOS" + controllerStr )
2569 consistentClustersResult = main.FALSE
2570 main.log.warn( "ONOS" + controllerStr +
2571 " clusters response: " +
2572 repr( clusters[ controller ] ) )
2573 utilities.assert_equals(
2574 expect=main.TRUE,
2575 actual=consistentClustersResult,
2576 onpass="Clusters view is consistent across all ONOS nodes",
2577 onfail="ONOS nodes have different views of clusters" )
2578
2579 main.step( "There is only one SCC" )
2580 # there should always only be one cluster
2581 try:
2582 numClusters = len( json.loads( clusters[ 0 ] ) )
2583 except ( ValueError, TypeError ):
2584 main.log.exception( "Error parsing clusters[0]: " +
2585 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002586 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002587 clusterResults = main.FALSE
2588 if numClusters == 1:
2589 clusterResults = main.TRUE
2590 utilities.assert_equals(
2591 expect=1,
2592 actual=numClusters,
2593 onpass="ONOS shows 1 SCC",
2594 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2595
2596 topoResult = ( devicesResults and linksResults
2597 and hostsResults and consistentHostsResult
2598 and consistentClustersResult and clusterResults
2599 and ipResult and hostAttachmentResults )
2600
2601 topoResult = topoResult and int( count <= 2 )
2602 note = "note it takes about " + str( int( cliTime ) ) + \
2603 " seconds for the test to make all the cli calls to fetch " +\
2604 "the topology from each ONOS instance"
2605 main.log.info(
2606 "Very crass estimate for topology discovery/convergence( " +
2607 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2608 str( count ) + " tries" )
2609
2610 main.step( "Device information is correct" )
2611 utilities.assert_equals(
2612 expect=main.TRUE,
2613 actual=devicesResults,
2614 onpass="Device information is correct",
2615 onfail="Device information is incorrect" )
2616
2617 main.step( "Links are correct" )
2618 utilities.assert_equals(
2619 expect=main.TRUE,
2620 actual=linksResults,
2621 onpass="Link are correct",
2622 onfail="Links are incorrect" )
2623
Jon Halla440e872016-03-31 15:15:50 -07002624 main.step( "Hosts are correct" )
2625 utilities.assert_equals(
2626 expect=main.TRUE,
2627 actual=hostsResults,
2628 onpass="Hosts are correct",
2629 onfail="Hosts are incorrect" )
2630
Jon Hall5cf14d52015-07-16 12:15:19 -07002631 # FIXME: move this to an ONOS state case
2632 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002633 nodeResults = utilities.retry( main.HA.nodesCheck,
2634 False,
2635 args=[main.activeNodes],
2636 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002637
Jon Hall41d39f12016-04-11 22:54:35 -07002638 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002639 onpass="Nodes check successful",
2640 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002641 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002642 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002643 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002644 main.CLIs[i].name,
2645 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002646
2647 def CASE9( self, main ):
2648 """
2649 Link s3-s28 down
2650 """
2651 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002652 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002653 assert main, "main not defined"
2654 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002655 assert main.CLIs, "main.CLIs not defined"
2656 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002657 # NOTE: You should probably run a topology check after this
2658
2659 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2660
2661 description = "Turn off a link to ensure that Link Discovery " +\
2662 "is working properly"
2663 main.case( description )
2664
2665 main.step( "Kill Link between s3 and s28" )
2666 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2667 main.log.info( "Waiting " + str( linkSleep ) +
2668 " seconds for link down to be discovered" )
2669 time.sleep( linkSleep )
2670 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2671 onpass="Link down successful",
2672 onfail="Failed to bring link down" )
2673 # TODO do some sort of check here
2674
2675 def CASE10( self, main ):
2676 """
2677 Link s3-s28 up
2678 """
2679 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002680 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002681 assert main, "main not defined"
2682 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002683 assert main.CLIs, "main.CLIs not defined"
2684 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002685 # NOTE: You should probably run a topology check after this
2686
2687 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2688
2689 description = "Restore a link to ensure that Link Discovery is " + \
2690 "working properly"
2691 main.case( description )
2692
2693 main.step( "Bring link between s3 and s28 back up" )
2694 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2695 main.log.info( "Waiting " + str( linkSleep ) +
2696 " seconds for link up to be discovered" )
2697 time.sleep( linkSleep )
2698 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2699 onpass="Link up successful",
2700 onfail="Failed to bring link up" )
2701 # TODO do some sort of check here
2702
2703 def CASE11( self, main ):
2704 """
2705 Switch Down
2706 """
2707 # NOTE: You should probably run a topology check after this
2708 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002709 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002710 assert main, "main not defined"
2711 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002712 assert main.CLIs, "main.CLIs not defined"
2713 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002714
2715 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2716
2717 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002718 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002719 main.case( description )
2720 switch = main.params[ 'kill' ][ 'switch' ]
2721 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2722
2723 # TODO: Make this switch parameterizable
2724 main.step( "Kill " + switch )
2725 main.log.info( "Deleting " + switch )
2726 main.Mininet1.delSwitch( switch )
2727 main.log.info( "Waiting " + str( switchSleep ) +
2728 " seconds for switch down to be discovered" )
2729 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002730 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002731 # Peek at the deleted switch
2732 main.log.warn( str( device ) )
2733 result = main.FALSE
2734 if device and device[ 'available' ] is False:
2735 result = main.TRUE
2736 utilities.assert_equals( expect=main.TRUE, actual=result,
2737 onpass="Kill switch successful",
2738 onfail="Failed to kill switch?" )
2739
2740 def CASE12( self, main ):
2741 """
2742 Switch Up
2743 """
2744 # NOTE: You should probably run a topology check after this
2745 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002746 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002747 assert main, "main not defined"
2748 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002749 assert main.CLIs, "main.CLIs not defined"
2750 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002751 assert ONOS1Port, "ONOS1Port not defined"
2752 assert ONOS2Port, "ONOS2Port not defined"
2753 assert ONOS3Port, "ONOS3Port not defined"
2754 assert ONOS4Port, "ONOS4Port not defined"
2755 assert ONOS5Port, "ONOS5Port not defined"
2756 assert ONOS6Port, "ONOS6Port not defined"
2757 assert ONOS7Port, "ONOS7Port not defined"
2758
2759 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2760 switch = main.params[ 'kill' ][ 'switch' ]
2761 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2762 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002763 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002764 description = "Adding a switch to ensure it is discovered correctly"
2765 main.case( description )
2766
2767 main.step( "Add back " + switch )
2768 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2769 for peer in links:
2770 main.Mininet1.addLink( switch, peer )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002771 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002772 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2773 main.log.info( "Waiting " + str( switchSleep ) +
2774 " seconds for switch up to be discovered" )
2775 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002776 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002777 # Peek at the deleted switch
2778 main.log.warn( str( device ) )
2779 result = main.FALSE
2780 if device and device[ 'available' ]:
2781 result = main.TRUE
2782 utilities.assert_equals( expect=main.TRUE, actual=result,
2783 onpass="add switch successful",
2784 onfail="Failed to add switch?" )
2785
2786 def CASE13( self, main ):
2787 """
2788 Clean up
2789 """
2790 import os
2791 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002792 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002793 assert main, "main not defined"
2794 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002795 assert main.CLIs, "main.CLIs not defined"
2796 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002797
2798 # printing colors to terminal
2799 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2800 'blue': '\033[94m', 'green': '\033[92m',
2801 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2802 main.case( "Test Cleanup" )
2803 main.step( "Killing tcpdumps" )
2804 main.Mininet2.stopTcpdump()
2805
2806 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002807 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002808 main.step( "Copying MN pcap and ONOS log files to test station" )
2809 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2810 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002811 # NOTE: MN Pcap file is being saved to logdir.
2812 # We scp this file as MN and TestON aren't necessarily the same vm
2813
2814 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002815 # TODO: Load these from params
2816 # NOTE: must end in /
2817 logFolder = "/opt/onos/log/"
2818 logFiles = [ "karaf.log", "karaf.log.1" ]
2819 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002820 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002821 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002822 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002823 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2824 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002825 # std*.log's
2826 # NOTE: must end in /
2827 logFolder = "/opt/onos/var/"
2828 logFiles = [ "stderr.log", "stdout.log" ]
2829 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002830 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002831 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002832 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002833 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2834 logFolder + f, dstName )
2835 else:
2836 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002837
2838 main.step( "Stopping Mininet" )
2839 mnResult = main.Mininet1.stopNet()
2840 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2841 onpass="Mininet stopped",
2842 onfail="MN cleanup NOT successful" )
2843
2844 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002845 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002846 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2847 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002848
2849 try:
2850 timerLog = open( main.logdir + "/Timers.csv", 'w')
2851 # Overwrite with empty line and close
2852 labels = "Gossip Intents, Restart"
2853 data = str( gossipTime ) + ", " + str( main.restartTime )
2854 timerLog.write( labels + "\n" + data )
2855 timerLog.close()
2856 except NameError, e:
2857 main.log.exception(e)
2858
2859 def CASE14( self, main ):
2860 """
2861 start election app on all onos nodes
2862 """
Jon Halle1a3b752015-07-22 13:02:46 -07002863 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002864 assert main, "main not defined"
2865 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002866 assert main.CLIs, "main.CLIs not defined"
2867 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002868
2869 main.case("Start Leadership Election app")
2870 main.step( "Install leadership election app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002871 onosCli = main.CLIs[ main.activeNodes[0] ]
2872 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002873 utilities.assert_equals(
2874 expect=main.TRUE,
2875 actual=appResult,
2876 onpass="Election app installed",
2877 onfail="Something went wrong with installing Leadership election" )
2878
2879 main.step( "Run for election on each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002880 for i in main.activeNodes:
2881 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002882 time.sleep(5)
2883 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2884 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002885 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002886 expect=True,
2887 actual=sameResult,
2888 onpass="All nodes see the same leaderboards",
2889 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002890
Jon Hall25463a82016-04-13 14:03:52 -07002891 if sameResult:
2892 leader = leaders[ 0 ][ 0 ]
2893 if main.nodes[main.activeNodes[0]].ip_address in leader:
2894 correctLeader = True
2895 else:
2896 correctLeader = False
2897 main.step( "First node was elected leader" )
2898 utilities.assert_equals(
2899 expect=True,
2900 actual=correctLeader,
2901 onpass="Correct leader was elected",
2902 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002903
2904 def CASE15( self, main ):
2905 """
2906 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002907 15.1 Run election on each node
2908 15.2 Check that each node has the same leaders and candidates
2909 15.3 Find current leader and withdraw
2910 15.4 Check that a new node was elected leader
2911 15.5 Check that that new leader was the candidate of old leader
2912 15.6 Run for election on old leader
2913 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2914 15.8 Make sure that the old leader was added to the candidate list
2915
2916 old and new variable prefixes refer to data from before vs after
2917 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002918 """
2919 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002920 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002921 assert main, "main not defined"
2922 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002923 assert main.CLIs, "main.CLIs not defined"
2924 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002925
Jon Hall5cf14d52015-07-16 12:15:19 -07002926 description = "Check that Leadership Election is still functional"
2927 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002928 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002929
Jon Halla440e872016-03-31 15:15:50 -07002930 oldLeaders = [] # list of lists of each nodes' candidates before
2931 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002932 oldLeader = '' # the old leader from oldLeaders, None if not same
2933 newLeader = '' # the new leaders fron newLoeaders, None if not same
2934 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2935 expectNoLeader = False # True when there is only one leader
2936 if main.numCtrls == 1:
2937 expectNoLeader = True
2938
2939 main.step( "Run for election on each node" )
2940 electionResult = main.TRUE
2941
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002942 for i in main.activeNodes: # run test election on each node
2943 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002944 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002945 utilities.assert_equals(
2946 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002947 actual=electionResult,
2948 onpass="All nodes successfully ran for leadership",
2949 onfail="At least one node failed to run for leadership" )
2950
acsmars3a72bde2015-09-02 14:16:22 -07002951 if electionResult == main.FALSE:
2952 main.log.error(
2953 "Skipping Test Case because Election Test App isn't loaded" )
2954 main.skipCase()
2955
acsmars71adceb2015-08-31 15:09:26 -07002956 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002957 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002958 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002959 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002960 if sameResult:
2961 oldLeader = oldLeaders[ 0 ][ 0 ]
2962 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07002963 else:
Jon Halla440e872016-03-31 15:15:50 -07002964 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002965 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002966 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002967 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002968 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002969 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002970
2971 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002972 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002973 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002974 if oldLeader is None:
2975 main.log.error( "Leadership isn't consistent." )
2976 withdrawResult = main.FALSE
2977 # Get the CLI of the oldLeader
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002978 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002979 if oldLeader == main.nodes[ i ].ip_address:
2980 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002981 break
2982 else: # FOR/ELSE statement
2983 main.log.error( "Leader election, could not find current leader" )
2984 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002985 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002986 utilities.assert_equals(
2987 expect=main.TRUE,
2988 actual=withdrawResult,
2989 onpass="Node was withdrawn from election",
2990 onfail="Node was not withdrawn from election" )
2991
acsmars71adceb2015-08-31 15:09:26 -07002992 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002993 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07002994 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002995 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002996 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002997 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002998 if newLeaders[ 0 ][ 0 ] == 'none':
2999 main.log.error( "No leader was elected on at least 1 node" )
3000 if not expectNoLeader:
3001 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07003002 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07003003
3004 # Check that the new leader is not the older leader, which was withdrawn
3005 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07003006 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08003007 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07003008 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003009 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003010 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003011 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003012 onpass="Leadership election passed",
3013 onfail="Something went wrong with Leadership election" )
3014
Jon Halla440e872016-03-31 15:15:50 -07003015 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08003016 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07003017 correctCandidateResult = main.TRUE
3018 if expectNoLeader:
3019 if newLeader == 'none':
3020 main.log.info( "No leader expected. None found. Pass" )
3021 correctCandidateResult = main.TRUE
3022 else:
3023 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3024 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003025 elif len( oldLeaders[0] ) >= 3:
3026 if newLeader == oldLeaders[ 0 ][ 2 ]:
3027 # correct leader was elected
3028 correctCandidateResult = main.TRUE
3029 else:
3030 correctCandidateResult = main.FALSE
3031 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3032 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003033 else:
3034 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003035 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003036 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07003037 utilities.assert_equals(
3038 expect=main.TRUE,
3039 actual=correctCandidateResult,
3040 onpass="Correct Candidate Elected",
3041 onfail="Incorrect Candidate Elected" )
3042
Jon Hall5cf14d52015-07-16 12:15:19 -07003043 main.step( "Run for election on old leader( just so everyone " +
3044 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07003045 if oldLeaderCLI is not None:
3046 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003047 else:
acsmars71adceb2015-08-31 15:09:26 -07003048 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003049 runResult = main.FALSE
3050 utilities.assert_equals(
3051 expect=main.TRUE,
3052 actual=runResult,
3053 onpass="App re-ran for election",
3054 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003055
acsmars71adceb2015-08-31 15:09:26 -07003056 main.step(
3057 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003058 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003059 # Get new leaders and candidates
3060 reRunLeaders = []
3061 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003062 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003063
3064 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07003065 if not reRunLeaders[0]:
3066 positionResult = main.FALSE
3067 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07003068 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3069 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07003070 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003071 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003072 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003073 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003074 onpass="Old leader successfully re-ran for election",
3075 onfail="Something went wrong with Leadership election after " +
3076 "the old leader re-ran for election" )
3077
3078 def CASE16( self, main ):
3079 """
3080 Install Distributed Primitives app
3081 """
3082 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003083 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003084 assert main, "main not defined"
3085 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003086 assert main.CLIs, "main.CLIs not defined"
3087 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003088
3089 # Variables for the distributed primitives tests
3090 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003091 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003092 global onosSet
3093 global onosSetName
3094 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003095 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003096 onosSet = set([])
3097 onosSetName = "TestON-set"
3098
3099 description = "Install Primitives app"
3100 main.case( description )
3101 main.step( "Install Primitives app" )
3102 appName = "org.onosproject.distributedprimitives"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003103 node = main.activeNodes[0]
3104 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003105 utilities.assert_equals( expect=main.TRUE,
3106 actual=appResults,
3107 onpass="Primitives app activated",
3108 onfail="Primitives app not activated" )
3109 time.sleep( 5 ) # To allow all nodes to activate
3110
3111 def CASE17( self, main ):
3112 """
3113 Check for basic functionality with distributed primitives
3114 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003115 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003116 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003117 assert main, "main not defined"
3118 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003119 assert main.CLIs, "main.CLIs not defined"
3120 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003121 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003122 assert onosSetName, "onosSetName not defined"
3123 # NOTE: assert fails if value is 0/None/Empty/False
3124 try:
3125 pCounterValue
3126 except NameError:
3127 main.log.error( "pCounterValue not defined, setting to 0" )
3128 pCounterValue = 0
3129 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003130 onosSet
3131 except NameError:
3132 main.log.error( "onosSet not defined, setting to empty Set" )
3133 onosSet = set([])
3134 # Variables for the distributed primitives tests. These are local only
3135 addValue = "a"
3136 addAllValue = "a b c d e f"
3137 retainValue = "c d e f"
3138
3139 description = "Check for basic functionality with distributed " +\
3140 "primitives"
3141 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003142 main.caseExplanation = "Test the methods of the distributed " +\
3143 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003144 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003145 # Partitioned counters
3146 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003147 pCounters = []
3148 threads = []
3149 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003150 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003151 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3152 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003153 args=[ pCounterName ] )
3154 pCounterValue += 1
3155 addedPValues.append( pCounterValue )
3156 threads.append( t )
3157 t.start()
3158
3159 for t in threads:
3160 t.join()
3161 pCounters.append( t.result )
3162 # Check that counter incremented numController times
3163 pCounterResults = True
3164 for i in addedPValues:
3165 tmpResult = i in pCounters
3166 pCounterResults = pCounterResults and tmpResult
3167 if not tmpResult:
3168 main.log.error( str( i ) + " is not in partitioned "
3169 "counter incremented results" )
3170 utilities.assert_equals( expect=True,
3171 actual=pCounterResults,
3172 onpass="Default counter incremented",
3173 onfail="Error incrementing default" +
3174 " counter" )
3175
Jon Halle1a3b752015-07-22 13:02:46 -07003176 main.step( "Get then Increment a default counter on each node" )
3177 pCounters = []
3178 threads = []
3179 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003180 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003181 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3182 name="counterGetAndAdd-" + str( i ),
3183 args=[ pCounterName ] )
3184 addedPValues.append( pCounterValue )
3185 pCounterValue += 1
3186 threads.append( t )
3187 t.start()
3188
3189 for t in threads:
3190 t.join()
3191 pCounters.append( t.result )
3192 # Check that counter incremented numController times
3193 pCounterResults = True
3194 for i in addedPValues:
3195 tmpResult = i in pCounters
3196 pCounterResults = pCounterResults and tmpResult
3197 if not tmpResult:
3198 main.log.error( str( i ) + " is not in partitioned "
3199 "counter incremented results" )
3200 utilities.assert_equals( expect=True,
3201 actual=pCounterResults,
3202 onpass="Default counter incremented",
3203 onfail="Error incrementing default" +
3204 " counter" )
3205
3206 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003207 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003208 utilities.assert_equals( expect=main.TRUE,
3209 actual=incrementCheck,
3210 onpass="Added counters are correct",
3211 onfail="Added counters are incorrect" )
3212
3213 main.step( "Add -8 to then get a default counter on each node" )
3214 pCounters = []
3215 threads = []
3216 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003217 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003218 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3219 name="counterIncrement-" + str( i ),
3220 args=[ pCounterName ],
3221 kwargs={ "delta": -8 } )
3222 pCounterValue += -8
3223 addedPValues.append( pCounterValue )
3224 threads.append( t )
3225 t.start()
3226
3227 for t in threads:
3228 t.join()
3229 pCounters.append( t.result )
3230 # Check that counter incremented numController times
3231 pCounterResults = True
3232 for i in addedPValues:
3233 tmpResult = i in pCounters
3234 pCounterResults = pCounterResults and tmpResult
3235 if not tmpResult:
3236 main.log.error( str( i ) + " is not in partitioned "
3237 "counter incremented results" )
3238 utilities.assert_equals( expect=True,
3239 actual=pCounterResults,
3240 onpass="Default counter incremented",
3241 onfail="Error incrementing default" +
3242 " counter" )
3243
3244 main.step( "Add 5 to then get a default counter on each node" )
3245 pCounters = []
3246 threads = []
3247 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003248 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003249 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3250 name="counterIncrement-" + str( i ),
3251 args=[ pCounterName ],
3252 kwargs={ "delta": 5 } )
3253 pCounterValue += 5
3254 addedPValues.append( pCounterValue )
3255 threads.append( t )
3256 t.start()
3257
3258 for t in threads:
3259 t.join()
3260 pCounters.append( t.result )
3261 # Check that counter incremented numController times
3262 pCounterResults = True
3263 for i in addedPValues:
3264 tmpResult = i in pCounters
3265 pCounterResults = pCounterResults and tmpResult
3266 if not tmpResult:
3267 main.log.error( str( i ) + " is not in partitioned "
3268 "counter incremented results" )
3269 utilities.assert_equals( expect=True,
3270 actual=pCounterResults,
3271 onpass="Default counter incremented",
3272 onfail="Error incrementing default" +
3273 " counter" )
3274
3275 main.step( "Get then add 5 to a default counter on each node" )
3276 pCounters = []
3277 threads = []
3278 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003279 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003280 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3281 name="counterIncrement-" + str( i ),
3282 args=[ pCounterName ],
3283 kwargs={ "delta": 5 } )
3284 addedPValues.append( pCounterValue )
3285 pCounterValue += 5
3286 threads.append( t )
3287 t.start()
3288
3289 for t in threads:
3290 t.join()
3291 pCounters.append( t.result )
3292 # Check that counter incremented numController times
3293 pCounterResults = True
3294 for i in addedPValues:
3295 tmpResult = i in pCounters
3296 pCounterResults = pCounterResults and tmpResult
3297 if not tmpResult:
3298 main.log.error( str( i ) + " is not in partitioned "
3299 "counter incremented results" )
3300 utilities.assert_equals( expect=True,
3301 actual=pCounterResults,
3302 onpass="Default counter incremented",
3303 onfail="Error incrementing default" +
3304 " counter" )
3305
3306 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003307 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003308 utilities.assert_equals( expect=main.TRUE,
3309 actual=incrementCheck,
3310 onpass="Added counters are correct",
3311 onfail="Added counters are incorrect" )
3312
Jon Hall5cf14d52015-07-16 12:15:19 -07003313 # DISTRIBUTED SETS
3314 main.step( "Distributed Set get" )
3315 size = len( onosSet )
3316 getResponses = []
3317 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003318 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003319 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003320 name="setTestGet-" + str( i ),
3321 args=[ onosSetName ] )
3322 threads.append( t )
3323 t.start()
3324 for t in threads:
3325 t.join()
3326 getResponses.append( t.result )
3327
3328 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003329 for i in range( len( main.activeNodes ) ):
3330 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003331 if isinstance( getResponses[ i ], list):
3332 current = set( getResponses[ i ] )
3333 if len( current ) == len( getResponses[ i ] ):
3334 # no repeats
3335 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003336 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003337 " has incorrect view" +
3338 " of set " + onosSetName + ":\n" +
3339 str( getResponses[ i ] ) )
3340 main.log.debug( "Expected: " + str( onosSet ) )
3341 main.log.debug( "Actual: " + str( current ) )
3342 getResults = main.FALSE
3343 else:
3344 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003345 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003346 " has repeat elements in" +
3347 " set " + onosSetName + ":\n" +
3348 str( getResponses[ i ] ) )
3349 getResults = main.FALSE
3350 elif getResponses[ i ] == main.ERROR:
3351 getResults = main.FALSE
3352 utilities.assert_equals( expect=main.TRUE,
3353 actual=getResults,
3354 onpass="Set elements are correct",
3355 onfail="Set elements are incorrect" )
3356
3357 main.step( "Distributed Set size" )
3358 sizeResponses = []
3359 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003360 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003361 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003362 name="setTestSize-" + str( i ),
3363 args=[ onosSetName ] )
3364 threads.append( t )
3365 t.start()
3366 for t in threads:
3367 t.join()
3368 sizeResponses.append( t.result )
3369
3370 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003371 for i in range( len( main.activeNodes ) ):
3372 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003373 if size != sizeResponses[ i ]:
3374 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003375 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003376 " expected a size of " + str( size ) +
3377 " for set " + onosSetName +
3378 " but got " + str( sizeResponses[ i ] ) )
3379 utilities.assert_equals( expect=main.TRUE,
3380 actual=sizeResults,
3381 onpass="Set sizes are correct",
3382 onfail="Set sizes are incorrect" )
3383
3384 main.step( "Distributed Set add()" )
3385 onosSet.add( addValue )
3386 addResponses = []
3387 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003388 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003389 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003390 name="setTestAdd-" + str( i ),
3391 args=[ onosSetName, addValue ] )
3392 threads.append( t )
3393 t.start()
3394 for t in threads:
3395 t.join()
3396 addResponses.append( t.result )
3397
3398 # main.TRUE = successfully changed the set
3399 # main.FALSE = action resulted in no change in set
3400 # main.ERROR - Some error in executing the function
3401 addResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003402 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003403 if addResponses[ i ] == main.TRUE:
3404 # All is well
3405 pass
3406 elif addResponses[ i ] == main.FALSE:
3407 # Already in set, probably fine
3408 pass
3409 elif addResponses[ i ] == main.ERROR:
3410 # Error in execution
3411 addResults = main.FALSE
3412 else:
3413 # unexpected result
3414 addResults = main.FALSE
3415 if addResults != main.TRUE:
3416 main.log.error( "Error executing set add" )
3417
3418 # Check if set is still correct
3419 size = len( onosSet )
3420 getResponses = []
3421 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003422 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003423 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003424 name="setTestGet-" + str( i ),
3425 args=[ onosSetName ] )
3426 threads.append( t )
3427 t.start()
3428 for t in threads:
3429 t.join()
3430 getResponses.append( t.result )
3431 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003432 for i in range( len( main.activeNodes ) ):
3433 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003434 if isinstance( getResponses[ i ], list):
3435 current = set( getResponses[ i ] )
3436 if len( current ) == len( getResponses[ i ] ):
3437 # no repeats
3438 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003439 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003440 " of set " + onosSetName + ":\n" +
3441 str( getResponses[ i ] ) )
3442 main.log.debug( "Expected: " + str( onosSet ) )
3443 main.log.debug( "Actual: " + str( current ) )
3444 getResults = main.FALSE
3445 else:
3446 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003447 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003448 " set " + onosSetName + ":\n" +
3449 str( getResponses[ i ] ) )
3450 getResults = main.FALSE
3451 elif getResponses[ i ] == main.ERROR:
3452 getResults = main.FALSE
3453 sizeResponses = []
3454 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003455 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003456 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003457 name="setTestSize-" + str( i ),
3458 args=[ onosSetName ] )
3459 threads.append( t )
3460 t.start()
3461 for t in threads:
3462 t.join()
3463 sizeResponses.append( t.result )
3464 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003465 for i in range( len( main.activeNodes ) ):
3466 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003467 if size != sizeResponses[ i ]:
3468 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003469 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003470 " expected a size of " + str( size ) +
3471 " for set " + onosSetName +
3472 " but got " + str( sizeResponses[ i ] ) )
3473 addResults = addResults and getResults and sizeResults
3474 utilities.assert_equals( expect=main.TRUE,
3475 actual=addResults,
3476 onpass="Set add correct",
3477 onfail="Set add was incorrect" )
3478
3479 main.step( "Distributed Set addAll()" )
3480 onosSet.update( addAllValue.split() )
3481 addResponses = []
3482 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003483 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003484 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003485 name="setTestAddAll-" + str( i ),
3486 args=[ onosSetName, addAllValue ] )
3487 threads.append( t )
3488 t.start()
3489 for t in threads:
3490 t.join()
3491 addResponses.append( t.result )
3492
3493 # main.TRUE = successfully changed the set
3494 # main.FALSE = action resulted in no change in set
3495 # main.ERROR - Some error in executing the function
3496 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003497 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003498 if addResponses[ i ] == main.TRUE:
3499 # All is well
3500 pass
3501 elif addResponses[ i ] == main.FALSE:
3502 # Already in set, probably fine
3503 pass
3504 elif addResponses[ i ] == main.ERROR:
3505 # Error in execution
3506 addAllResults = main.FALSE
3507 else:
3508 # unexpected result
3509 addAllResults = main.FALSE
3510 if addAllResults != main.TRUE:
3511 main.log.error( "Error executing set addAll" )
3512
3513 # Check if set is still correct
3514 size = len( onosSet )
3515 getResponses = []
3516 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003517 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003518 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003519 name="setTestGet-" + str( i ),
3520 args=[ onosSetName ] )
3521 threads.append( t )
3522 t.start()
3523 for t in threads:
3524 t.join()
3525 getResponses.append( t.result )
3526 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003527 for i in range( len( main.activeNodes ) ):
3528 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003529 if isinstance( getResponses[ i ], list):
3530 current = set( getResponses[ i ] )
3531 if len( current ) == len( getResponses[ i ] ):
3532 # no repeats
3533 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003534 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003535 " has incorrect view" +
3536 " of set " + onosSetName + ":\n" +
3537 str( getResponses[ i ] ) )
3538 main.log.debug( "Expected: " + str( onosSet ) )
3539 main.log.debug( "Actual: " + str( current ) )
3540 getResults = main.FALSE
3541 else:
3542 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003543 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003544 " has repeat elements in" +
3545 " set " + onosSetName + ":\n" +
3546 str( getResponses[ i ] ) )
3547 getResults = main.FALSE
3548 elif getResponses[ i ] == main.ERROR:
3549 getResults = main.FALSE
3550 sizeResponses = []
3551 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003552 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003553 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003554 name="setTestSize-" + str( i ),
3555 args=[ onosSetName ] )
3556 threads.append( t )
3557 t.start()
3558 for t in threads:
3559 t.join()
3560 sizeResponses.append( t.result )
3561 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003562 for i in range( len( main.activeNodes ) ):
3563 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003564 if size != sizeResponses[ i ]:
3565 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003566 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003567 " expected a size of " + str( size ) +
3568 " for set " + onosSetName +
3569 " but got " + str( sizeResponses[ i ] ) )
3570 addAllResults = addAllResults and getResults and sizeResults
3571 utilities.assert_equals( expect=main.TRUE,
3572 actual=addAllResults,
3573 onpass="Set addAll correct",
3574 onfail="Set addAll was incorrect" )
3575
3576 main.step( "Distributed Set contains()" )
3577 containsResponses = []
3578 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003579 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003580 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003581 name="setContains-" + str( i ),
3582 args=[ onosSetName ],
3583 kwargs={ "values": addValue } )
3584 threads.append( t )
3585 t.start()
3586 for t in threads:
3587 t.join()
3588 # NOTE: This is the tuple
3589 containsResponses.append( t.result )
3590
3591 containsResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003592 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003593 if containsResponses[ i ] == main.ERROR:
3594 containsResults = main.FALSE
3595 else:
3596 containsResults = containsResults and\
3597 containsResponses[ i ][ 1 ]
3598 utilities.assert_equals( expect=main.TRUE,
3599 actual=containsResults,
3600 onpass="Set contains is functional",
3601 onfail="Set contains failed" )
3602
3603 main.step( "Distributed Set containsAll()" )
3604 containsAllResponses = []
3605 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003606 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003607 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003608 name="setContainsAll-" + str( i ),
3609 args=[ onosSetName ],
3610 kwargs={ "values": addAllValue } )
3611 threads.append( t )
3612 t.start()
3613 for t in threads:
3614 t.join()
3615 # NOTE: This is the tuple
3616 containsAllResponses.append( t.result )
3617
3618 containsAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003619 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003620 if containsResponses[ i ] == main.ERROR:
3621 containsResults = main.FALSE
3622 else:
3623 containsResults = containsResults and\
3624 containsResponses[ i ][ 1 ]
3625 utilities.assert_equals( expect=main.TRUE,
3626 actual=containsAllResults,
3627 onpass="Set containsAll is functional",
3628 onfail="Set containsAll failed" )
3629
3630 main.step( "Distributed Set remove()" )
3631 onosSet.remove( addValue )
3632 removeResponses = []
3633 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003634 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003635 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003636 name="setTestRemove-" + str( i ),
3637 args=[ onosSetName, addValue ] )
3638 threads.append( t )
3639 t.start()
3640 for t in threads:
3641 t.join()
3642 removeResponses.append( t.result )
3643
3644 # main.TRUE = successfully changed the set
3645 # main.FALSE = action resulted in no change in set
3646 # main.ERROR - Some error in executing the function
3647 removeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003648 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003649 if removeResponses[ i ] == main.TRUE:
3650 # All is well
3651 pass
3652 elif removeResponses[ i ] == main.FALSE:
3653 # not in set, probably fine
3654 pass
3655 elif removeResponses[ i ] == main.ERROR:
3656 # Error in execution
3657 removeResults = main.FALSE
3658 else:
3659 # unexpected result
3660 removeResults = main.FALSE
3661 if removeResults != main.TRUE:
3662 main.log.error( "Error executing set remove" )
3663
3664 # Check if set is still correct
3665 size = len( onosSet )
3666 getResponses = []
3667 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003668 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003669 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003670 name="setTestGet-" + str( i ),
3671 args=[ onosSetName ] )
3672 threads.append( t )
3673 t.start()
3674 for t in threads:
3675 t.join()
3676 getResponses.append( t.result )
3677 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003678 for i in range( len( main.activeNodes ) ):
3679 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003680 if isinstance( getResponses[ i ], list):
3681 current = set( getResponses[ i ] )
3682 if len( current ) == len( getResponses[ i ] ):
3683 # no repeats
3684 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003685 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003686 " has incorrect view" +
3687 " of set " + onosSetName + ":\n" +
3688 str( getResponses[ i ] ) )
3689 main.log.debug( "Expected: " + str( onosSet ) )
3690 main.log.debug( "Actual: " + str( current ) )
3691 getResults = main.FALSE
3692 else:
3693 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003694 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003695 " has repeat elements in" +
3696 " set " + onosSetName + ":\n" +
3697 str( getResponses[ i ] ) )
3698 getResults = main.FALSE
3699 elif getResponses[ i ] == main.ERROR:
3700 getResults = main.FALSE
3701 sizeResponses = []
3702 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003703 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003704 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003705 name="setTestSize-" + str( i ),
3706 args=[ onosSetName ] )
3707 threads.append( t )
3708 t.start()
3709 for t in threads:
3710 t.join()
3711 sizeResponses.append( t.result )
3712 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003713 for i in range( len( main.activeNodes ) ):
3714 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003715 if size != sizeResponses[ i ]:
3716 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003717 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003718 " expected a size of " + str( size ) +
3719 " for set " + onosSetName +
3720 " but got " + str( sizeResponses[ i ] ) )
3721 removeResults = removeResults and getResults and sizeResults
3722 utilities.assert_equals( expect=main.TRUE,
3723 actual=removeResults,
3724 onpass="Set remove correct",
3725 onfail="Set remove was incorrect" )
3726
3727 main.step( "Distributed Set removeAll()" )
3728 onosSet.difference_update( addAllValue.split() )
3729 removeAllResponses = []
3730 threads = []
3731 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003732 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003733 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003734 name="setTestRemoveAll-" + str( i ),
3735 args=[ onosSetName, addAllValue ] )
3736 threads.append( t )
3737 t.start()
3738 for t in threads:
3739 t.join()
3740 removeAllResponses.append( t.result )
3741 except Exception, e:
3742 main.log.exception(e)
3743
3744 # main.TRUE = successfully changed the set
3745 # main.FALSE = action resulted in no change in set
3746 # main.ERROR - Some error in executing the function
3747 removeAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003748 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003749 if removeAllResponses[ i ] == main.TRUE:
3750 # All is well
3751 pass
3752 elif removeAllResponses[ i ] == main.FALSE:
3753 # not in set, probably fine
3754 pass
3755 elif removeAllResponses[ i ] == main.ERROR:
3756 # Error in execution
3757 removeAllResults = main.FALSE
3758 else:
3759 # unexpected result
3760 removeAllResults = main.FALSE
3761 if removeAllResults != main.TRUE:
3762 main.log.error( "Error executing set removeAll" )
3763
3764 # Check if set is still correct
3765 size = len( onosSet )
3766 getResponses = []
3767 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003768 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003769 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003770 name="setTestGet-" + str( i ),
3771 args=[ onosSetName ] )
3772 threads.append( t )
3773 t.start()
3774 for t in threads:
3775 t.join()
3776 getResponses.append( t.result )
3777 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003778 for i in range( len( main.activeNodes ) ):
3779 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003780 if isinstance( getResponses[ i ], list):
3781 current = set( getResponses[ i ] )
3782 if len( current ) == len( getResponses[ i ] ):
3783 # no repeats
3784 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003785 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003786 " has incorrect view" +
3787 " of set " + onosSetName + ":\n" +
3788 str( getResponses[ i ] ) )
3789 main.log.debug( "Expected: " + str( onosSet ) )
3790 main.log.debug( "Actual: " + str( current ) )
3791 getResults = main.FALSE
3792 else:
3793 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003794 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003795 " has repeat elements in" +
3796 " set " + onosSetName + ":\n" +
3797 str( getResponses[ i ] ) )
3798 getResults = main.FALSE
3799 elif getResponses[ i ] == main.ERROR:
3800 getResults = main.FALSE
3801 sizeResponses = []
3802 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003803 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003804 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003805 name="setTestSize-" + str( i ),
3806 args=[ onosSetName ] )
3807 threads.append( t )
3808 t.start()
3809 for t in threads:
3810 t.join()
3811 sizeResponses.append( t.result )
3812 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003813 for i in range( len( main.activeNodes ) ):
3814 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003815 if size != sizeResponses[ i ]:
3816 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003817 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003818 " expected a size of " + str( size ) +
3819 " for set " + onosSetName +
3820 " but got " + str( sizeResponses[ i ] ) )
3821 removeAllResults = removeAllResults and getResults and sizeResults
3822 utilities.assert_equals( expect=main.TRUE,
3823 actual=removeAllResults,
3824 onpass="Set removeAll correct",
3825 onfail="Set removeAll was incorrect" )
3826
3827 main.step( "Distributed Set addAll()" )
3828 onosSet.update( addAllValue.split() )
3829 addResponses = []
3830 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003831 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003832 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003833 name="setTestAddAll-" + str( i ),
3834 args=[ onosSetName, addAllValue ] )
3835 threads.append( t )
3836 t.start()
3837 for t in threads:
3838 t.join()
3839 addResponses.append( t.result )
3840
3841 # main.TRUE = successfully changed the set
3842 # main.FALSE = action resulted in no change in set
3843 # main.ERROR - Some error in executing the function
3844 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003845 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003846 if addResponses[ i ] == main.TRUE:
3847 # All is well
3848 pass
3849 elif addResponses[ i ] == main.FALSE:
3850 # Already in set, probably fine
3851 pass
3852 elif addResponses[ i ] == main.ERROR:
3853 # Error in execution
3854 addAllResults = main.FALSE
3855 else:
3856 # unexpected result
3857 addAllResults = main.FALSE
3858 if addAllResults != main.TRUE:
3859 main.log.error( "Error executing set addAll" )
3860
3861 # Check if set is still correct
3862 size = len( onosSet )
3863 getResponses = []
3864 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003865 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003866 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003867 name="setTestGet-" + str( i ),
3868 args=[ onosSetName ] )
3869 threads.append( t )
3870 t.start()
3871 for t in threads:
3872 t.join()
3873 getResponses.append( t.result )
3874 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003875 for i in range( len( main.activeNodes ) ):
3876 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003877 if isinstance( getResponses[ i ], list):
3878 current = set( getResponses[ i ] )
3879 if len( current ) == len( getResponses[ i ] ):
3880 # no repeats
3881 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003882 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003883 " has incorrect view" +
3884 " of set " + onosSetName + ":\n" +
3885 str( getResponses[ i ] ) )
3886 main.log.debug( "Expected: " + str( onosSet ) )
3887 main.log.debug( "Actual: " + str( current ) )
3888 getResults = main.FALSE
3889 else:
3890 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003891 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003892 " has repeat elements in" +
3893 " set " + onosSetName + ":\n" +
3894 str( getResponses[ i ] ) )
3895 getResults = main.FALSE
3896 elif getResponses[ i ] == main.ERROR:
3897 getResults = main.FALSE
3898 sizeResponses = []
3899 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003900 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003901 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003902 name="setTestSize-" + str( i ),
3903 args=[ onosSetName ] )
3904 threads.append( t )
3905 t.start()
3906 for t in threads:
3907 t.join()
3908 sizeResponses.append( t.result )
3909 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003910 for i in range( len( main.activeNodes ) ):
3911 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003912 if size != sizeResponses[ i ]:
3913 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003914 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003915 " expected a size of " + str( size ) +
3916 " for set " + onosSetName +
3917 " but got " + str( sizeResponses[ i ] ) )
3918 addAllResults = addAllResults and getResults and sizeResults
3919 utilities.assert_equals( expect=main.TRUE,
3920 actual=addAllResults,
3921 onpass="Set addAll correct",
3922 onfail="Set addAll was incorrect" )
3923
3924 main.step( "Distributed Set clear()" )
3925 onosSet.clear()
3926 clearResponses = []
3927 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003928 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003929 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003930 name="setTestClear-" + str( i ),
3931 args=[ onosSetName, " "], # Values doesn't matter
3932 kwargs={ "clear": True } )
3933 threads.append( t )
3934 t.start()
3935 for t in threads:
3936 t.join()
3937 clearResponses.append( t.result )
3938
3939 # main.TRUE = successfully changed the set
3940 # main.FALSE = action resulted in no change in set
3941 # main.ERROR - Some error in executing the function
3942 clearResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003943 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003944 if clearResponses[ i ] == main.TRUE:
3945 # All is well
3946 pass
3947 elif clearResponses[ i ] == main.FALSE:
3948 # Nothing set, probably fine
3949 pass
3950 elif clearResponses[ i ] == main.ERROR:
3951 # Error in execution
3952 clearResults = main.FALSE
3953 else:
3954 # unexpected result
3955 clearResults = main.FALSE
3956 if clearResults != main.TRUE:
3957 main.log.error( "Error executing set clear" )
3958
3959 # Check if set is still correct
3960 size = len( onosSet )
3961 getResponses = []
3962 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003963 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003964 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003965 name="setTestGet-" + str( i ),
3966 args=[ onosSetName ] )
3967 threads.append( t )
3968 t.start()
3969 for t in threads:
3970 t.join()
3971 getResponses.append( t.result )
3972 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003973 for i in range( len( main.activeNodes ) ):
3974 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003975 if isinstance( getResponses[ i ], list):
3976 current = set( getResponses[ i ] )
3977 if len( current ) == len( getResponses[ i ] ):
3978 # no repeats
3979 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003980 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003981 " has incorrect view" +
3982 " of set " + onosSetName + ":\n" +
3983 str( getResponses[ i ] ) )
3984 main.log.debug( "Expected: " + str( onosSet ) )
3985 main.log.debug( "Actual: " + str( current ) )
3986 getResults = main.FALSE
3987 else:
3988 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003989 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003990 " has repeat elements in" +
3991 " set " + onosSetName + ":\n" +
3992 str( getResponses[ i ] ) )
3993 getResults = main.FALSE
3994 elif getResponses[ i ] == main.ERROR:
3995 getResults = main.FALSE
3996 sizeResponses = []
3997 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003998 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003999 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004000 name="setTestSize-" + str( i ),
4001 args=[ onosSetName ] )
4002 threads.append( t )
4003 t.start()
4004 for t in threads:
4005 t.join()
4006 sizeResponses.append( t.result )
4007 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004008 for i in range( len( main.activeNodes ) ):
4009 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004010 if size != sizeResponses[ i ]:
4011 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004012 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004013 " expected a size of " + str( size ) +
4014 " for set " + onosSetName +
4015 " but got " + str( sizeResponses[ i ] ) )
4016 clearResults = clearResults and getResults and sizeResults
4017 utilities.assert_equals( expect=main.TRUE,
4018 actual=clearResults,
4019 onpass="Set clear correct",
4020 onfail="Set clear was incorrect" )
4021
4022 main.step( "Distributed Set addAll()" )
4023 onosSet.update( addAllValue.split() )
4024 addResponses = []
4025 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004026 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004027 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004028 name="setTestAddAll-" + str( i ),
4029 args=[ onosSetName, addAllValue ] )
4030 threads.append( t )
4031 t.start()
4032 for t in threads:
4033 t.join()
4034 addResponses.append( t.result )
4035
4036 # main.TRUE = successfully changed the set
4037 # main.FALSE = action resulted in no change in set
4038 # main.ERROR - Some error in executing the function
4039 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004040 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004041 if addResponses[ i ] == main.TRUE:
4042 # All is well
4043 pass
4044 elif addResponses[ i ] == main.FALSE:
4045 # Already in set, probably fine
4046 pass
4047 elif addResponses[ i ] == main.ERROR:
4048 # Error in execution
4049 addAllResults = main.FALSE
4050 else:
4051 # unexpected result
4052 addAllResults = main.FALSE
4053 if addAllResults != main.TRUE:
4054 main.log.error( "Error executing set addAll" )
4055
4056 # Check if set is still correct
4057 size = len( onosSet )
4058 getResponses = []
4059 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004060 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004061 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004062 name="setTestGet-" + str( i ),
4063 args=[ onosSetName ] )
4064 threads.append( t )
4065 t.start()
4066 for t in threads:
4067 t.join()
4068 getResponses.append( t.result )
4069 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004070 for i in range( len( main.activeNodes ) ):
4071 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004072 if isinstance( getResponses[ i ], list):
4073 current = set( getResponses[ i ] )
4074 if len( current ) == len( getResponses[ i ] ):
4075 # no repeats
4076 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004077 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004078 " has incorrect view" +
4079 " of set " + onosSetName + ":\n" +
4080 str( getResponses[ i ] ) )
4081 main.log.debug( "Expected: " + str( onosSet ) )
4082 main.log.debug( "Actual: " + str( current ) )
4083 getResults = main.FALSE
4084 else:
4085 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004086 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004087 " has repeat elements in" +
4088 " set " + onosSetName + ":\n" +
4089 str( getResponses[ i ] ) )
4090 getResults = main.FALSE
4091 elif getResponses[ i ] == main.ERROR:
4092 getResults = main.FALSE
4093 sizeResponses = []
4094 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004095 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004096 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004097 name="setTestSize-" + str( i ),
4098 args=[ onosSetName ] )
4099 threads.append( t )
4100 t.start()
4101 for t in threads:
4102 t.join()
4103 sizeResponses.append( t.result )
4104 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004105 for i in range( len( main.activeNodes ) ):
4106 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004107 if size != sizeResponses[ i ]:
4108 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004109 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004110 " expected a size of " + str( size ) +
4111 " for set " + onosSetName +
4112 " but got " + str( sizeResponses[ i ] ) )
4113 addAllResults = addAllResults and getResults and sizeResults
4114 utilities.assert_equals( expect=main.TRUE,
4115 actual=addAllResults,
4116 onpass="Set addAll correct",
4117 onfail="Set addAll was incorrect" )
4118
4119 main.step( "Distributed Set retain()" )
4120 onosSet.intersection_update( retainValue.split() )
4121 retainResponses = []
4122 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004123 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004124 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004125 name="setTestRetain-" + str( i ),
4126 args=[ onosSetName, retainValue ],
4127 kwargs={ "retain": True } )
4128 threads.append( t )
4129 t.start()
4130 for t in threads:
4131 t.join()
4132 retainResponses.append( t.result )
4133
4134 # main.TRUE = successfully changed the set
4135 # main.FALSE = action resulted in no change in set
4136 # main.ERROR - Some error in executing the function
4137 retainResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004138 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004139 if retainResponses[ i ] == main.TRUE:
4140 # All is well
4141 pass
4142 elif retainResponses[ i ] == main.FALSE:
4143 # Already in set, probably fine
4144 pass
4145 elif retainResponses[ i ] == main.ERROR:
4146 # Error in execution
4147 retainResults = main.FALSE
4148 else:
4149 # unexpected result
4150 retainResults = main.FALSE
4151 if retainResults != main.TRUE:
4152 main.log.error( "Error executing set retain" )
4153
4154 # Check if set is still correct
4155 size = len( onosSet )
4156 getResponses = []
4157 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004158 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004159 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004160 name="setTestGet-" + str( i ),
4161 args=[ onosSetName ] )
4162 threads.append( t )
4163 t.start()
4164 for t in threads:
4165 t.join()
4166 getResponses.append( t.result )
4167 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004168 for i in range( len( main.activeNodes ) ):
4169 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004170 if isinstance( getResponses[ i ], list):
4171 current = set( getResponses[ i ] )
4172 if len( current ) == len( getResponses[ i ] ):
4173 # no repeats
4174 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004175 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004176 " has incorrect view" +
4177 " of set " + onosSetName + ":\n" +
4178 str( getResponses[ i ] ) )
4179 main.log.debug( "Expected: " + str( onosSet ) )
4180 main.log.debug( "Actual: " + str( current ) )
4181 getResults = main.FALSE
4182 else:
4183 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004184 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004185 " has repeat elements in" +
4186 " set " + onosSetName + ":\n" +
4187 str( getResponses[ i ] ) )
4188 getResults = main.FALSE
4189 elif getResponses[ i ] == main.ERROR:
4190 getResults = main.FALSE
4191 sizeResponses = []
4192 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004193 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004194 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004195 name="setTestSize-" + str( i ),
4196 args=[ onosSetName ] )
4197 threads.append( t )
4198 t.start()
4199 for t in threads:
4200 t.join()
4201 sizeResponses.append( t.result )
4202 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004203 for i in range( len( main.activeNodes ) ):
4204 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004205 if size != sizeResponses[ i ]:
4206 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004207 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004208 str( size ) + " for set " + onosSetName +
4209 " but got " + str( sizeResponses[ i ] ) )
4210 retainResults = retainResults and getResults and sizeResults
4211 utilities.assert_equals( expect=main.TRUE,
4212 actual=retainResults,
4213 onpass="Set retain correct",
4214 onfail="Set retain was incorrect" )
4215
Jon Hall2a5002c2015-08-21 16:49:11 -07004216 # Transactional maps
4217 main.step( "Partitioned Transactional maps put" )
4218 tMapValue = "Testing"
4219 numKeys = 100
4220 putResult = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004221 node = main.activeNodes[0]
4222 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004223 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004224 for i in putResponses:
4225 if putResponses[ i ][ 'value' ] != tMapValue:
4226 putResult = False
4227 else:
4228 putResult = False
4229 if not putResult:
4230 main.log.debug( "Put response values: " + str( putResponses ) )
4231 utilities.assert_equals( expect=True,
4232 actual=putResult,
4233 onpass="Partitioned Transactional Map put successful",
4234 onfail="Partitioned Transactional Map put values are incorrect" )
4235
4236 main.step( "Partitioned Transactional maps get" )
4237 getCheck = True
4238 for n in range( 1, numKeys + 1 ):
4239 getResponses = []
4240 threads = []
4241 valueCheck = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004242 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004243 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4244 name="TMap-get-" + str( i ),
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004245 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004246 threads.append( t )
4247 t.start()
4248 for t in threads:
4249 t.join()
4250 getResponses.append( t.result )
4251 for node in getResponses:
4252 if node != tMapValue:
4253 valueCheck = False
4254 if not valueCheck:
4255 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4256 main.log.warn( getResponses )
4257 getCheck = getCheck and valueCheck
4258 utilities.assert_equals( expect=True,
4259 actual=getCheck,
4260 onpass="Partitioned Transactional Map get values were correct",
4261 onfail="Partitioned Transactional Map values incorrect" )