blob: 147f183eba05f887e817d06c11d8f320803107bf [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
Jon Hallb3ed8ed2015-10-28 16:43:55 -070012CASE61: The Failure inducing case.
13CASE62: The Failure recovery case.
Jon Hall5cf14d52015-07-16 12:15:19 -070014CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
Jon Hallb3ed8ed2015-10-28 16:43:55 -070028class HAkillNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -070029
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall3b489db2015-10-05 14:38:37 -070052 import pexpect
Jon Hall6e709752016-02-01 13:38:46 -080053 import time
Jon Halla440e872016-03-31 15:15:50 -070054 import json
Jon Hall6e709752016-02-01 13:38:46 -080055 main.log.info( "ONOS HA test: Restart a minority of ONOS nodes - " +
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "initialization" )
57 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070058 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070059 "installing ONOS, starting Mininet and ONOS" +\
60 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070061
62 # load some variables from the params file
63 PULLCODE = False
64 if main.params[ 'Git' ] == 'True':
65 PULLCODE = True
66 gitBranch = main.params[ 'branch' ]
67 cellName = main.params[ 'ENV' ][ 'cellName' ]
68
Jon Halle1a3b752015-07-22 13:02:46 -070069 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070071 if main.ONOSbench.maxNodes < main.numCtrls:
72 main.numCtrls = int( main.ONOSbench.maxNodes )
73 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070074 global ONOS1Port
75 global ONOS2Port
76 global ONOS3Port
77 global ONOS4Port
78 global ONOS5Port
79 global ONOS6Port
80 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070081 # These are for csv plotting in jenkins
82 global labels
83 global data
84 labels = []
85 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070086
87 # FIXME: just get controller port from params?
88 # TODO: do we really need all these?
89 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
90 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
91 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
92 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
93 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
94 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
95 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
96
Jon Halle1a3b752015-07-22 13:02:46 -070097 try:
Jon Hall53c5e662016-04-13 16:06:56 -070098 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070099 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -0700100 except Exception as e:
101 main.log.exception( e )
102 main.cleanup()
103 main.exit()
104
105 main.CLIs = []
106 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700107 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700108 for i in range( 1, main.numCtrls + 1 ):
109 try:
110 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
111 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
112 ipList.append( main.nodes[ -1 ].ip_address )
113 except AttributeError:
114 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700115
116 main.step( "Create cell file" )
117 cellAppString = main.params[ 'ENV' ][ 'appString' ]
118 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
119 main.Mininet1.ip_address,
120 cellAppString, ipList )
121 main.step( "Applying cell variable to environment" )
122 cellResult = main.ONOSbench.setCell( cellName )
123 verifyResult = main.ONOSbench.verifyCell()
124
125 # FIXME:this is short term fix
126 main.log.info( "Removing raft logs" )
127 main.ONOSbench.onosRemoveRaftLogs()
128
129 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700130 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700131 main.ONOSbench.onosUninstall( node.ip_address )
132
133 # Make sure ONOS is DEAD
134 main.log.info( "Killing any ONOS processes" )
135 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700136 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700137 killed = main.ONOSbench.onosKill( node.ip_address )
138 killResults = killResults and killed
139
140 cleanInstallResult = main.TRUE
141 gitPullResult = main.TRUE
142
143 main.step( "Starting Mininet" )
144 # scp topo file to mininet
145 # TODO: move to params?
146 topoName = "obelisk.py"
147 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700148 main.ONOSbench.scp( main.Mininet1,
149 filePath + topoName,
150 main.Mininet1.home,
151 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700152 mnResult = main.Mininet1.startNet( )
153 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
154 onpass="Mininet Started",
155 onfail="Error starting Mininet" )
156
157 main.step( "Git checkout and pull " + gitBranch )
158 if PULLCODE:
159 main.ONOSbench.gitCheckout( gitBranch )
160 gitPullResult = main.ONOSbench.gitPull()
161 # values of 1 or 3 are good
162 utilities.assert_lesser( expect=0, actual=gitPullResult,
163 onpass="Git pull successful",
164 onfail="Git pull failed" )
165 main.ONOSbench.getVersion( report=True )
166
167 main.step( "Using mvn clean install" )
168 cleanInstallResult = main.TRUE
169 if PULLCODE and gitPullResult == main.TRUE:
170 cleanInstallResult = main.ONOSbench.cleanInstall()
171 else:
172 main.log.warn( "Did not pull new code so skipping mvn " +
173 "clean install" )
174 utilities.assert_equals( expect=main.TRUE,
175 actual=cleanInstallResult,
176 onpass="MCI successful",
177 onfail="MCI failed" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700178
179 main.step( "Make sure ONOS service doesn't automatically respawn" )
180 handle = main.ONOSbench.handle
181 handle.sendline( "sed -i -e 's/^respawn$/#respawn/g' tools/package/init/onos.conf" )
182 handle.expect( "\$" ) # $ from the command
183 handle.expect( "\$" ) # $ from the prompt
184
Jon Hall5cf14d52015-07-16 12:15:19 -0700185 # GRAPHS
186 # NOTE: important params here:
187 # job = name of Jenkins job
188 # Plot Name = Plot-HA, only can be used if multiple plots
189 # index = The number of the graph under plot name
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700190 job = "HAkillNodes"
Jon Hall5cf14d52015-07-16 12:15:19 -0700191 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700192 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700193 graphs = '<ac:structured-macro ac:name="html">\n'
194 graphs += '<ac:plain-text-body><![CDATA[\n'
195 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800196 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700197 '&width=500&height=300"' +\
198 'noborder="0" width="500" height="300" scrolling="yes" ' +\
199 'seamless="seamless"></iframe>\n'
200 graphs += ']]></ac:plain-text-body>\n'
201 graphs += '</ac:structured-macro>\n'
202 main.log.wiki(graphs)
203
204 main.step( "Creating ONOS package" )
Jon Hall3b489db2015-10-05 14:38:37 -0700205 # copy gen-partions file to ONOS
206 # NOTE: this assumes TestON and ONOS are on the same machine
Jon Hall53c5e662016-04-13 16:06:56 -0700207 srcFile = main.testDir + "/HA/dependencies/onos-gen-partitions"
Jon Hall3b489db2015-10-05 14:38:37 -0700208 dstDir = main.ONOSbench.home + "/tools/test/bin/onos-gen-partitions"
209 cpResult = main.ONOSbench.secureCopy( main.ONOSbench.user_name,
210 main.ONOSbench.ip_address,
211 srcFile,
212 dstDir,
213 pwd=main.ONOSbench.pwd,
214 direction="from" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700215 packageResult = main.ONOSbench.buckBuild()
Jon Hall5cf14d52015-07-16 12:15:19 -0700216 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
217 onpass="ONOS package successful",
218 onfail="ONOS package failed" )
219
220 main.step( "Installing ONOS package" )
221 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700222 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700223 tmpResult = main.ONOSbench.onosInstall( options="-f",
224 node=node.ip_address )
225 onosInstallResult = onosInstallResult and tmpResult
226 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
227 onpass="ONOS install successful",
228 onfail="ONOS install failed" )
Jon Hall3b489db2015-10-05 14:38:37 -0700229 # clean up gen-partitions file
230 try:
231 main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
232 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
233 main.ONOSbench.handle.sendline( "git checkout -- tools/test/bin/onos-gen-partitions" )
234 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
235 main.log.info( " Cleaning custom gen partitions file, response was: \n" +
236 str( main.ONOSbench.handle.before ) )
237 except ( pexpect.TIMEOUT, pexpect.EOF ):
238 main.log.exception( "ONOSbench: pexpect exception found:" +
239 main.ONOSbench.handle.before )
240 main.cleanup()
241 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700242
243 main.step( "Checking if ONOS is up yet" )
244 for i in range( 2 ):
245 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700246 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700247 started = main.ONOSbench.isup( node.ip_address )
248 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800249 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700250 onosIsupResult = onosIsupResult and started
251 if onosIsupResult == main.TRUE:
252 break
253 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
254 onpass="ONOS startup successful",
255 onfail="ONOS startup failed" )
256
Jon Hall6509dbf2016-06-21 17:01:17 -0700257 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700258 cliResults = main.TRUE
259 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700260 for i in range( main.numCtrls ):
261 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700262 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700263 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700264 threads.append( t )
265 t.start()
266
267 for t in threads:
268 t.join()
269 cliResults = cliResults and t.result
270 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
271 onpass="ONOS cli startup successful",
272 onfail="ONOS cli startup failed" )
273
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700274 # Create a list of active nodes for use when some nodes are stopped
275 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
276
Jon Hall5cf14d52015-07-16 12:15:19 -0700277 if main.params[ 'tcpdump' ].lower() == "true":
278 main.step( "Start Packet Capture MN" )
279 main.Mininet2.startTcpdump(
280 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
281 + "-MN.pcap",
282 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
283 port=main.params[ 'MNtcpdump' ][ 'port' ] )
284
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700285 main.step( "Clean up ONOS service changes" )
286 handle.sendline( "git checkout -- tools/package/init/onos.conf" )
287 handle.expect( "\$" )
288
Jon Halla440e872016-03-31 15:15:50 -0700289 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700290 nodeResults = utilities.retry( main.HA.nodesCheck,
291 False,
292 args=[main.activeNodes],
293 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700294
Jon Hall41d39f12016-04-11 22:54:35 -0700295 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700296 onpass="Nodes check successful",
297 onfail="Nodes check NOT successful" )
298
299 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700300 for i in main.activeNodes:
301 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700302 main.log.debug( "{} components not ACTIVE: \n{}".format(
303 cli.name,
304 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700305 main.log.error( "Failed to start ONOS, stopping test" )
306 main.cleanup()
307 main.exit()
308
Jon Hall172b7ba2016-04-07 18:12:20 -0700309 main.step( "Activate apps defined in the params file" )
310 # get data from the params
311 apps = main.params.get( 'apps' )
312 if apps:
313 apps = apps.split(',')
314 main.log.warn( apps )
315 activateResult = True
316 for app in apps:
317 main.CLIs[ 0 ].app( app, "Activate" )
318 # TODO: check this worked
319 time.sleep( 10 ) # wait for apps to activate
320 for app in apps:
321 state = main.CLIs[ 0 ].appStatus( app )
322 if state == "ACTIVE":
323 activateResult = activeResult and True
324 else:
325 main.log.error( "{} is in {} state".format( app, state ) )
326 activeResult = False
327 utilities.assert_equals( expect=True,
328 actual=activateResult,
329 onpass="Successfully activated apps",
330 onfail="Failed to activate apps" )
331 else:
332 main.log.warn( "No apps were specified to be loaded after startup" )
333
334 main.step( "Set ONOS configurations" )
335 config = main.params.get( 'ONOS_Configuration' )
336 if config:
337 main.log.debug( config )
338 checkResult = main.TRUE
339 for component in config:
340 for setting in config[component]:
341 value = config[component][setting]
342 check = main.CLIs[ 0 ].setCfg( component, setting, value )
343 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
344 checkResult = check and checkResult
345 utilities.assert_equals( expect=main.TRUE,
346 actual=checkResult,
347 onpass="Successfully set config",
348 onfail="Failed to set config" )
349 else:
350 main.log.warn( "No configurations were specified to be changed after startup" )
351
Jon Hall9d2dcad2016-04-08 10:15:20 -0700352 main.step( "App Ids check" )
353 appCheck = main.TRUE
354 threads = []
355 for i in main.activeNodes:
356 t = main.Thread( target=main.CLIs[i].appToIDCheck,
357 name="appToIDCheck-" + str( i ),
358 args=[] )
359 threads.append( t )
360 t.start()
361
362 for t in threads:
363 t.join()
364 appCheck = appCheck and t.result
365 if appCheck != main.TRUE:
366 node = main.activeNodes[0]
367 main.log.warn( main.CLIs[node].apps() )
368 main.log.warn( main.CLIs[node].appIDs() )
369 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
370 onpass="App Ids seem to be correct",
371 onfail="Something is wrong with app Ids" )
372
Jon Hall5cf14d52015-07-16 12:15:19 -0700373 def CASE2( self, main ):
374 """
375 Assign devices to controllers
376 """
377 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700378 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700379 assert main, "main not defined"
380 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700381 assert main.CLIs, "main.CLIs not defined"
382 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700383 assert ONOS1Port, "ONOS1Port not defined"
384 assert ONOS2Port, "ONOS2Port not defined"
385 assert ONOS3Port, "ONOS3Port not defined"
386 assert ONOS4Port, "ONOS4Port not defined"
387 assert ONOS5Port, "ONOS5Port not defined"
388 assert ONOS6Port, "ONOS6Port not defined"
389 assert ONOS7Port, "ONOS7Port not defined"
390
391 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700392 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700393 "and check that an ONOS node becomes the " +\
394 "master of the device."
395 main.step( "Assign switches to controllers" )
396
397 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700398 for i in range( main.numCtrls ):
399 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700400 swList = []
401 for i in range( 1, 29 ):
402 swList.append( "s" + str( i ) )
403 main.Mininet1.assignSwController( sw=swList, ip=ipList )
404
405 mastershipCheck = main.TRUE
406 for i in range( 1, 29 ):
407 response = main.Mininet1.getSwController( "s" + str( i ) )
408 try:
409 main.log.info( str( response ) )
410 except Exception:
411 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700412 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700413 if re.search( "tcp:" + node.ip_address, response ):
414 mastershipCheck = mastershipCheck and main.TRUE
415 else:
416 main.log.error( "Error, node " + node.ip_address + " is " +
417 "not in the list of controllers s" +
418 str( i ) + " is connecting to." )
419 mastershipCheck = main.FALSE
420 utilities.assert_equals(
421 expect=main.TRUE,
422 actual=mastershipCheck,
423 onpass="Switch mastership assigned correctly",
424 onfail="Switches not assigned correctly to controllers" )
425
426 def CASE21( self, main ):
427 """
428 Assign mastership to controllers
429 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700430 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700431 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700432 assert main, "main not defined"
433 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700434 assert main.CLIs, "main.CLIs not defined"
435 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700436 assert ONOS1Port, "ONOS1Port not defined"
437 assert ONOS2Port, "ONOS2Port not defined"
438 assert ONOS3Port, "ONOS3Port not defined"
439 assert ONOS4Port, "ONOS4Port not defined"
440 assert ONOS5Port, "ONOS5Port not defined"
441 assert ONOS6Port, "ONOS6Port not defined"
442 assert ONOS7Port, "ONOS7Port not defined"
443
444 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700445 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700446 "device. Then manually assign" +\
447 " mastership to specific ONOS nodes using" +\
448 " 'device-role'"
449 main.step( "Assign mastership of switches to specific controllers" )
450 # Manually assign mastership to the controller we want
451 roleCall = main.TRUE
452
453 ipList = [ ]
454 deviceList = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700455 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700456 try:
457 # Assign mastership to specific controllers. This assignment was
458 # determined for a 7 node cluser, but will work with any sized
459 # cluster
460 for i in range( 1, 29 ): # switches 1 through 28
461 # set up correct variables:
462 if i == 1:
463 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700464 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700465 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700466 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700467 c = 1 % main.numCtrls
468 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700469 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700470 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700471 c = 1 % main.numCtrls
472 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700473 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700474 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700475 c = 3 % main.numCtrls
476 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700477 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700478 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700479 c = 2 % main.numCtrls
480 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700481 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700482 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700483 c = 2 % main.numCtrls
484 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700485 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700486 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700487 c = 5 % main.numCtrls
488 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700489 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700490 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700491 c = 4 % main.numCtrls
492 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700493 dpid = '3' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700494 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700495 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700496 c = 6 % main.numCtrls
497 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700498 dpid = '6' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700499 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700500 elif i == 28:
501 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700502 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700503 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700504 else:
505 main.log.error( "You didn't write an else statement for " +
506 "switch s" + str( i ) )
507 roleCall = main.FALSE
508 # Assign switch
509 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
510 # TODO: make this controller dynamic
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700511 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700512 ipList.append( ip )
513 deviceList.append( deviceId )
514 except ( AttributeError, AssertionError ):
515 main.log.exception( "Something is wrong with ONOS device view" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700516 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700517 utilities.assert_equals(
518 expect=main.TRUE,
519 actual=roleCall,
520 onpass="Re-assigned switch mastership to designated controller",
521 onfail="Something wrong with deviceRole calls" )
522
523 main.step( "Check mastership was correctly assigned" )
524 roleCheck = main.TRUE
525 # NOTE: This is due to the fact that device mastership change is not
526 # atomic and is actually a multi step process
527 time.sleep( 5 )
528 for i in range( len( ipList ) ):
529 ip = ipList[i]
530 deviceId = deviceList[i]
531 # Check assignment
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700532 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700533 if ip in master:
534 roleCheck = roleCheck and main.TRUE
535 else:
536 roleCheck = roleCheck and main.FALSE
537 main.log.error( "Error, controller " + ip + " is not" +
538 " master " + "of device " +
539 str( deviceId ) + ". Master is " +
540 repr( master ) + "." )
541 utilities.assert_equals(
542 expect=main.TRUE,
543 actual=roleCheck,
544 onpass="Switches were successfully reassigned to designated " +
545 "controller",
546 onfail="Switches were not successfully reassigned" )
547
548 def CASE3( self, main ):
549 """
550 Assign intents
551 """
552 import time
553 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700554 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700555 assert main, "main not defined"
556 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700557 assert main.CLIs, "main.CLIs not defined"
558 assert main.nodes, "main.nodes not defined"
Jon Halla440e872016-03-31 15:15:50 -0700559 try:
560 labels
561 except NameError:
562 main.log.error( "labels not defined, setting to []" )
563 labels = []
564 try:
565 data
566 except NameError:
567 main.log.error( "data not defined, setting to []" )
568 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700569 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700570 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700571 "assign predetermined host-to-host intents." +\
572 " After installation, check that the intent" +\
573 " is distributed to all nodes and the state" +\
574 " is INSTALLED"
575
576 # install onos-app-fwd
577 main.step( "Install reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700578 onosCli = main.CLIs[ main.activeNodes[0] ]
579 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700580 utilities.assert_equals( expect=main.TRUE, actual=installResults,
581 onpass="Install fwd successful",
582 onfail="Install fwd failed" )
583
584 main.step( "Check app ids" )
585 appCheck = main.TRUE
586 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700587 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700588 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700589 name="appToIDCheck-" + str( i ),
590 args=[] )
591 threads.append( t )
592 t.start()
593
594 for t in threads:
595 t.join()
596 appCheck = appCheck and t.result
597 if appCheck != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700598 main.log.warn( onosCli.apps() )
599 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700600 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
601 onpass="App Ids seem to be correct",
602 onfail="Something is wrong with app Ids" )
603
604 main.step( "Discovering Hosts( Via pingall for now )" )
605 # FIXME: Once we have a host discovery mechanism, use that instead
606 # REACTIVE FWD test
607 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700608 passMsg = "Reactive Pingall test passed"
609 time1 = time.time()
610 pingResult = main.Mininet1.pingall()
611 time2 = time.time()
612 if not pingResult:
613 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700614 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700615 passMsg += " on the second try"
616 utilities.assert_equals(
617 expect=main.TRUE,
618 actual=pingResult,
619 onpass= passMsg,
620 onfail="Reactive Pingall failed, " +
621 "one or more ping pairs failed" )
622 main.log.info( "Time for pingall: %2f seconds" %
623 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700624 # timeout for fwd flows
625 time.sleep( 11 )
626 # uninstall onos-app-fwd
627 main.step( "Uninstall reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700628 node = main.activeNodes[0]
629 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700630 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
631 onpass="Uninstall fwd successful",
632 onfail="Uninstall fwd failed" )
633
634 main.step( "Check app ids" )
635 threads = []
636 appCheck2 = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700637 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700638 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700639 name="appToIDCheck-" + str( i ),
640 args=[] )
641 threads.append( t )
642 t.start()
643
644 for t in threads:
645 t.join()
646 appCheck2 = appCheck2 and t.result
647 if appCheck2 != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700648 node = main.activeNodes[0]
649 main.log.warn( main.CLIs[node].apps() )
650 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700651 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
652 onpass="App Ids seem to be correct",
653 onfail="Something is wrong with app Ids" )
654
655 main.step( "Add host intents via cli" )
656 intentIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700657 # TODO: move the host numbers to params
658 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700659 intentAddResult = True
660 hostResult = main.TRUE
661 for i in range( 8, 18 ):
662 main.log.info( "Adding host intent between h" + str( i ) +
663 " and h" + str( i + 10 ) )
664 host1 = "00:00:00:00:00:" + \
665 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
666 host2 = "00:00:00:00:00:" + \
667 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
668 # NOTE: getHost can return None
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700669 host1Dict = onosCli.getHost( host1 )
670 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700671 host1Id = None
672 host2Id = None
673 if host1Dict and host2Dict:
674 host1Id = host1Dict.get( 'id', None )
675 host2Id = host2Dict.get( 'id', None )
676 if host1Id and host2Id:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700677 nodeNum = ( i % len( main.activeNodes ) )
678 node = main.activeNodes[nodeNum]
679 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700680 if tmpId:
681 main.log.info( "Added intent with id: " + tmpId )
682 intentIds.append( tmpId )
683 else:
684 main.log.error( "addHostIntent returned: " +
685 repr( tmpId ) )
686 else:
687 main.log.error( "Error, getHost() failed for h" + str( i ) +
688 " and/or h" + str( i + 10 ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700689 node = main.activeNodes[0]
690 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700691 main.log.warn( "Hosts output: " )
692 try:
693 main.log.warn( json.dumps( json.loads( hosts ),
694 sort_keys=True,
695 indent=4,
696 separators=( ',', ': ' ) ) )
697 except ( ValueError, TypeError ):
698 main.log.warn( repr( hosts ) )
699 hostResult = main.FALSE
700 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
701 onpass="Found a host id for each host",
702 onfail="Error looking up host ids" )
703
704 intentStart = time.time()
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700705 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700706 main.log.info( "Submitted intents: " + str( intentIds ) )
707 main.log.info( "Intents in ONOS: " + str( onosIds ) )
708 for intent in intentIds:
709 if intent in onosIds:
710 pass # intent submitted is in onos
711 else:
712 intentAddResult = False
713 if intentAddResult:
714 intentStop = time.time()
715 else:
716 intentStop = None
717 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700718 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700719 intentStates = []
720 installedCheck = True
721 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
722 count = 0
723 try:
724 for intent in json.loads( intents ):
725 state = intent.get( 'state', None )
726 if "INSTALLED" not in state:
727 installedCheck = False
728 intentId = intent.get( 'id', None )
729 intentStates.append( ( intentId, state ) )
730 except ( ValueError, TypeError ):
731 main.log.exception( "Error parsing intents" )
732 # add submitted intents not in the store
733 tmplist = [ i for i, s in intentStates ]
734 missingIntents = False
735 for i in intentIds:
736 if i not in tmplist:
737 intentStates.append( ( i, " - " ) )
738 missingIntents = True
739 intentStates.sort()
740 for i, s in intentStates:
741 count += 1
742 main.log.info( "%-6s%-15s%-15s" %
743 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700744 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700745 try:
746 missing = False
747 if leaders:
748 parsedLeaders = json.loads( leaders )
749 main.log.warn( json.dumps( parsedLeaders,
750 sort_keys=True,
751 indent=4,
752 separators=( ',', ': ' ) ) )
753 # check for all intent partitions
754 topics = []
755 for i in range( 14 ):
756 topics.append( "intent-partition-" + str( i ) )
757 main.log.debug( topics )
758 ONOStopics = [ j['topic'] for j in parsedLeaders ]
759 for topic in topics:
760 if topic not in ONOStopics:
761 main.log.error( "Error: " + topic +
762 " not in leaders" )
763 missing = True
764 else:
765 main.log.error( "leaders() returned None" )
766 except ( ValueError, TypeError ):
767 main.log.exception( "Error parsing leaders" )
768 main.log.error( repr( leaders ) )
769 # Check all nodes
770 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700771 for i in main.activeNodes:
772 response = main.CLIs[i].leaders( jsonFormat=False)
773 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700774 str( response ) )
775
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700776 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700777 try:
778 if partitions :
779 parsedPartitions = json.loads( partitions )
780 main.log.warn( json.dumps( parsedPartitions,
781 sort_keys=True,
782 indent=4,
783 separators=( ',', ': ' ) ) )
784 # TODO check for a leader in all paritions
785 # TODO check for consistency among nodes
786 else:
787 main.log.error( "partitions() returned None" )
788 except ( ValueError, TypeError ):
789 main.log.exception( "Error parsing partitions" )
790 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700791 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700792 try:
793 if pendingMap :
794 parsedPending = json.loads( pendingMap )
795 main.log.warn( json.dumps( parsedPending,
796 sort_keys=True,
797 indent=4,
798 separators=( ',', ': ' ) ) )
799 # TODO check something here?
800 else:
801 main.log.error( "pendingMap() returned None" )
802 except ( ValueError, TypeError ):
803 main.log.exception( "Error parsing pending map" )
804 main.log.error( repr( pendingMap ) )
805
806 intentAddResult = bool( intentAddResult and not missingIntents and
807 installedCheck )
808 if not intentAddResult:
809 main.log.error( "Error in pushing host intents to ONOS" )
810
811 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700812 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700813 correct = True
814 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700815 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700816 onosIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700817 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700818 onosIds.append( ids )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700819 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700820 str( sorted( onosIds ) ) )
821 if sorted( ids ) != sorted( intentIds ):
822 main.log.warn( "Set of intent IDs doesn't match" )
823 correct = False
824 break
825 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700826 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700827 for intent in intents:
828 if intent[ 'state' ] != "INSTALLED":
829 main.log.warn( "Intent " + intent[ 'id' ] +
830 " is " + intent[ 'state' ] )
831 correct = False
832 break
833 if correct:
834 break
835 else:
836 time.sleep(1)
837 if not intentStop:
838 intentStop = time.time()
839 global gossipTime
840 gossipTime = intentStop - intentStart
841 main.log.info( "It took about " + str( gossipTime ) +
842 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700843 gossipPeriod = int( main.params['timers']['gossip'] )
844 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700845 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700846 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700847 onpass="ECM anti-entropy for intents worked within " +
848 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700849 onfail="Intent ECM anti-entropy took too long. " +
850 "Expected time:{}, Actual time:{}".format( maxGossipTime,
851 gossipTime ) )
852 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700853 intentAddResult = True
854
855 if not intentAddResult or "key" in pendingMap:
856 import time
857 installedCheck = True
858 main.log.info( "Sleeping 60 seconds to see if intents are found" )
859 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700860 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700861 main.log.info( "Submitted intents: " + str( intentIds ) )
862 main.log.info( "Intents in ONOS: " + str( onosIds ) )
863 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700864 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700865 intentStates = []
866 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
867 count = 0
868 try:
869 for intent in json.loads( intents ):
870 # Iter through intents of a node
871 state = intent.get( 'state', None )
872 if "INSTALLED" not in state:
873 installedCheck = False
874 intentId = intent.get( 'id', None )
875 intentStates.append( ( intentId, state ) )
876 except ( ValueError, TypeError ):
877 main.log.exception( "Error parsing intents" )
878 # add submitted intents not in the store
879 tmplist = [ i for i, s in intentStates ]
880 for i in intentIds:
881 if i not in tmplist:
882 intentStates.append( ( i, " - " ) )
883 intentStates.sort()
884 for i, s in intentStates:
885 count += 1
886 main.log.info( "%-6s%-15s%-15s" %
887 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700888 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700889 try:
890 missing = False
891 if leaders:
892 parsedLeaders = json.loads( leaders )
893 main.log.warn( json.dumps( parsedLeaders,
894 sort_keys=True,
895 indent=4,
896 separators=( ',', ': ' ) ) )
897 # check for all intent partitions
898 # check for election
899 topics = []
900 for i in range( 14 ):
901 topics.append( "intent-partition-" + str( i ) )
902 # FIXME: this should only be after we start the app
903 topics.append( "org.onosproject.election" )
904 main.log.debug( topics )
905 ONOStopics = [ j['topic'] for j in parsedLeaders ]
906 for topic in topics:
907 if topic not in ONOStopics:
908 main.log.error( "Error: " + topic +
909 " not in leaders" )
910 missing = True
911 else:
912 main.log.error( "leaders() returned None" )
913 except ( ValueError, TypeError ):
914 main.log.exception( "Error parsing leaders" )
915 main.log.error( repr( leaders ) )
916 # Check all nodes
917 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700918 for i in main.activeNodes:
919 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700920 response = node.leaders( jsonFormat=False)
921 main.log.warn( str( node.name ) + " leaders output: \n" +
922 str( response ) )
923
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700924 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700925 try:
926 if partitions :
927 parsedPartitions = json.loads( partitions )
928 main.log.warn( json.dumps( parsedPartitions,
929 sort_keys=True,
930 indent=4,
931 separators=( ',', ': ' ) ) )
932 # TODO check for a leader in all paritions
933 # TODO check for consistency among nodes
934 else:
935 main.log.error( "partitions() returned None" )
936 except ( ValueError, TypeError ):
937 main.log.exception( "Error parsing partitions" )
938 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700939 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700940 try:
941 if pendingMap :
942 parsedPending = json.loads( pendingMap )
943 main.log.warn( json.dumps( parsedPending,
944 sort_keys=True,
945 indent=4,
946 separators=( ',', ': ' ) ) )
947 # TODO check something here?
948 else:
949 main.log.error( "pendingMap() returned None" )
950 except ( ValueError, TypeError ):
951 main.log.exception( "Error parsing pending map" )
952 main.log.error( repr( pendingMap ) )
953
954 def CASE4( self, main ):
955 """
956 Ping across added host intents
957 """
958 import json
959 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700960 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700961 assert main, "main not defined"
962 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700963 assert main.CLIs, "main.CLIs not defined"
964 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700965 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700966 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700967 "functionality and check the state of " +\
968 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700969
Jon Hall41d39f12016-04-11 22:54:35 -0700970 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700971 main.step( "Check Intent state" )
972 installedCheck = False
973 loopCount = 0
974 while not installedCheck and loopCount < 40:
975 installedCheck = True
976 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700977 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700978 intentStates = []
979 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
980 count = 0
981 # Iter through intents of a node
982 try:
983 for intent in json.loads( intents ):
984 state = intent.get( 'state', None )
985 if "INSTALLED" not in state:
986 installedCheck = False
987 intentId = intent.get( 'id', None )
988 intentStates.append( ( intentId, state ) )
989 except ( ValueError, TypeError ):
990 main.log.exception( "Error parsing intents." )
991 # Print states
992 intentStates.sort()
993 for i, s in intentStates:
994 count += 1
995 main.log.info( "%-6s%-15s%-15s" %
996 ( str( count ), str( i ), str( s ) ) )
997 if not installedCheck:
998 time.sleep( 1 )
999 loopCount += 1
1000 utilities.assert_equals( expect=True, actual=installedCheck,
1001 onpass="Intents are all INSTALLED",
1002 onfail="Intents are not all in " +
1003 "INSTALLED state" )
1004
Jon Hall9d2dcad2016-04-08 10:15:20 -07001005 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -07001006 PingResult = main.TRUE
1007 for i in range( 8, 18 ):
1008 ping = main.Mininet1.pingHost( src="h" + str( i ),
1009 target="h" + str( i + 10 ) )
1010 PingResult = PingResult and ping
1011 if ping == main.FALSE:
1012 main.log.warn( "Ping failed between h" + str( i ) +
1013 " and h" + str( i + 10 ) )
1014 elif ping == main.TRUE:
1015 main.log.info( "Ping test passed!" )
1016 # Don't set PingResult or you'd override failures
1017 if PingResult == main.FALSE:
1018 main.log.error(
1019 "Intents have not been installed correctly, pings failed." )
1020 # TODO: pretty print
1021 main.log.warn( "ONOS1 intents: " )
1022 try:
1023 tmpIntents = onosCli.intents()
1024 main.log.warn( json.dumps( json.loads( tmpIntents ),
1025 sort_keys=True,
1026 indent=4,
1027 separators=( ',', ': ' ) ) )
1028 except ( ValueError, TypeError ):
1029 main.log.warn( repr( tmpIntents ) )
1030 utilities.assert_equals(
1031 expect=main.TRUE,
1032 actual=PingResult,
1033 onpass="Intents have been installed correctly and pings work",
1034 onfail="Intents have not been installed correctly, pings failed." )
1035
Jon Hall5cf14d52015-07-16 12:15:19 -07001036 main.step( "Check leadership of topics" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001037 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001038 topicCheck = main.TRUE
1039 try:
1040 if leaders:
1041 parsedLeaders = json.loads( leaders )
1042 main.log.warn( json.dumps( parsedLeaders,
1043 sort_keys=True,
1044 indent=4,
1045 separators=( ',', ': ' ) ) )
1046 # check for all intent partitions
1047 # check for election
1048 # TODO: Look at Devices as topics now that it uses this system
1049 topics = []
1050 for i in range( 14 ):
1051 topics.append( "intent-partition-" + str( i ) )
1052 # FIXME: this should only be after we start the app
1053 # FIXME: topics.append( "org.onosproject.election" )
1054 # Print leaders output
1055 main.log.debug( topics )
1056 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1057 for topic in topics:
1058 if topic not in ONOStopics:
1059 main.log.error( "Error: " + topic +
1060 " not in leaders" )
1061 topicCheck = main.FALSE
1062 else:
1063 main.log.error( "leaders() returned None" )
1064 topicCheck = main.FALSE
1065 except ( ValueError, TypeError ):
1066 topicCheck = main.FALSE
1067 main.log.exception( "Error parsing leaders" )
1068 main.log.error( repr( leaders ) )
1069 # TODO: Check for a leader of these topics
1070 # Check all nodes
1071 if topicCheck:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001072 for i in main.activeNodes:
1073 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001074 response = node.leaders( jsonFormat=False)
1075 main.log.warn( str( node.name ) + " leaders output: \n" +
1076 str( response ) )
1077
1078 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1079 onpass="intent Partitions is in leaders",
1080 onfail="Some topics were lost " )
1081 # Print partitions
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001082 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001083 try:
1084 if partitions :
1085 parsedPartitions = json.loads( partitions )
1086 main.log.warn( json.dumps( parsedPartitions,
1087 sort_keys=True,
1088 indent=4,
1089 separators=( ',', ': ' ) ) )
1090 # TODO check for a leader in all paritions
1091 # TODO check for consistency among nodes
1092 else:
1093 main.log.error( "partitions() returned None" )
1094 except ( ValueError, TypeError ):
1095 main.log.exception( "Error parsing partitions" )
1096 main.log.error( repr( partitions ) )
1097 # Print Pending Map
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001098 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001099 try:
1100 if pendingMap :
1101 parsedPending = json.loads( pendingMap )
1102 main.log.warn( json.dumps( parsedPending,
1103 sort_keys=True,
1104 indent=4,
1105 separators=( ',', ': ' ) ) )
1106 # TODO check something here?
1107 else:
1108 main.log.error( "pendingMap() returned None" )
1109 except ( ValueError, TypeError ):
1110 main.log.exception( "Error parsing pending map" )
1111 main.log.error( repr( pendingMap ) )
1112
1113 if not installedCheck:
1114 main.log.info( "Waiting 60 seconds to see if the state of " +
1115 "intents change" )
1116 time.sleep( 60 )
1117 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001118 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001119 intentStates = []
1120 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1121 count = 0
1122 # Iter through intents of a node
1123 try:
1124 for intent in json.loads( intents ):
1125 state = intent.get( 'state', None )
1126 if "INSTALLED" not in state:
1127 installedCheck = False
1128 intentId = intent.get( 'id', None )
1129 intentStates.append( ( intentId, state ) )
1130 except ( ValueError, TypeError ):
1131 main.log.exception( "Error parsing intents." )
1132 intentStates.sort()
1133 for i, s in intentStates:
1134 count += 1
1135 main.log.info( "%-6s%-15s%-15s" %
1136 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001137 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001138 try:
1139 missing = False
1140 if leaders:
1141 parsedLeaders = json.loads( leaders )
1142 main.log.warn( json.dumps( parsedLeaders,
1143 sort_keys=True,
1144 indent=4,
1145 separators=( ',', ': ' ) ) )
1146 # check for all intent partitions
1147 # check for election
1148 topics = []
1149 for i in range( 14 ):
1150 topics.append( "intent-partition-" + str( i ) )
1151 # FIXME: this should only be after we start the app
1152 topics.append( "org.onosproject.election" )
1153 main.log.debug( topics )
1154 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1155 for topic in topics:
1156 if topic not in ONOStopics:
1157 main.log.error( "Error: " + topic +
1158 " not in leaders" )
1159 missing = True
1160 else:
1161 main.log.error( "leaders() returned None" )
1162 except ( ValueError, TypeError ):
1163 main.log.exception( "Error parsing leaders" )
1164 main.log.error( repr( leaders ) )
1165 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001166 for i in main.activeNodes:
1167 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001168 response = node.leaders( jsonFormat=False)
1169 main.log.warn( str( node.name ) + " leaders output: \n" +
1170 str( response ) )
1171
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001172 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001173 try:
1174 if partitions :
1175 parsedPartitions = json.loads( partitions )
1176 main.log.warn( json.dumps( parsedPartitions,
1177 sort_keys=True,
1178 indent=4,
1179 separators=( ',', ': ' ) ) )
1180 # TODO check for a leader in all paritions
1181 # TODO check for consistency among nodes
1182 else:
1183 main.log.error( "partitions() returned None" )
1184 except ( ValueError, TypeError ):
1185 main.log.exception( "Error parsing partitions" )
1186 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001187 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001188 try:
1189 if pendingMap :
1190 parsedPending = json.loads( pendingMap )
1191 main.log.warn( json.dumps( parsedPending,
1192 sort_keys=True,
1193 indent=4,
1194 separators=( ',', ': ' ) ) )
1195 # TODO check something here?
1196 else:
1197 main.log.error( "pendingMap() returned None" )
1198 except ( ValueError, TypeError ):
1199 main.log.exception( "Error parsing pending map" )
1200 main.log.error( repr( pendingMap ) )
1201 # Print flowrules
Jon Hall41d39f12016-04-11 22:54:35 -07001202 main.log.debug( onosCli.flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001203 main.step( "Wait a minute then ping again" )
1204 # the wait is above
1205 PingResult = main.TRUE
1206 for i in range( 8, 18 ):
1207 ping = main.Mininet1.pingHost( src="h" + str( i ),
1208 target="h" + str( i + 10 ) )
1209 PingResult = PingResult and ping
1210 if ping == main.FALSE:
1211 main.log.warn( "Ping failed between h" + str( i ) +
1212 " and h" + str( i + 10 ) )
1213 elif ping == main.TRUE:
1214 main.log.info( "Ping test passed!" )
1215 # Don't set PingResult or you'd override failures
1216 if PingResult == main.FALSE:
1217 main.log.error(
1218 "Intents have not been installed correctly, pings failed." )
1219 # TODO: pretty print
1220 main.log.warn( "ONOS1 intents: " )
1221 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001222 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001223 main.log.warn( json.dumps( json.loads( tmpIntents ),
1224 sort_keys=True,
1225 indent=4,
1226 separators=( ',', ': ' ) ) )
1227 except ( ValueError, TypeError ):
1228 main.log.warn( repr( tmpIntents ) )
1229 utilities.assert_equals(
1230 expect=main.TRUE,
1231 actual=PingResult,
1232 onpass="Intents have been installed correctly and pings work",
1233 onfail="Intents have not been installed correctly, pings failed." )
1234
1235 def CASE5( self, main ):
1236 """
1237 Reading state of ONOS
1238 """
1239 import json
1240 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001241 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001242 assert main, "main not defined"
1243 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001244 assert main.CLIs, "main.CLIs not defined"
1245 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001246
1247 main.case( "Setting up and gathering data for current state" )
1248 # The general idea for this test case is to pull the state of
1249 # ( intents,flows, topology,... ) from each ONOS node
1250 # We can then compare them with each other and also with past states
1251
1252 main.step( "Check that each switch has a master" )
1253 global mastershipState
1254 mastershipState = '[]'
1255
1256 # Assert that each device has a master
1257 rolesNotNull = main.TRUE
1258 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001259 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001260 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001261 name="rolesNotNull-" + str( i ),
1262 args=[] )
1263 threads.append( t )
1264 t.start()
1265
1266 for t in threads:
1267 t.join()
1268 rolesNotNull = rolesNotNull and t.result
1269 utilities.assert_equals(
1270 expect=main.TRUE,
1271 actual=rolesNotNull,
1272 onpass="Each device has a master",
1273 onfail="Some devices don't have a master assigned" )
1274
1275 main.step( "Get the Mastership of each switch from each controller" )
1276 ONOSMastership = []
1277 mastershipCheck = main.FALSE
1278 consistentMastership = True
1279 rolesResults = True
1280 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001281 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001282 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001283 name="roles-" + str( i ),
1284 args=[] )
1285 threads.append( t )
1286 t.start()
1287
1288 for t in threads:
1289 t.join()
1290 ONOSMastership.append( t.result )
1291
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001292 for i in range( len( ONOSMastership ) ):
1293 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001294 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001295 main.log.error( "Error in getting ONOS" + node + " roles" )
1296 main.log.warn( "ONOS" + node + " mastership response: " +
1297 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001298 rolesResults = False
1299 utilities.assert_equals(
1300 expect=True,
1301 actual=rolesResults,
1302 onpass="No error in reading roles output",
1303 onfail="Error in reading roles from ONOS" )
1304
1305 main.step( "Check for consistency in roles from each controller" )
1306 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1307 main.log.info(
1308 "Switch roles are consistent across all ONOS nodes" )
1309 else:
1310 consistentMastership = False
1311 utilities.assert_equals(
1312 expect=True,
1313 actual=consistentMastership,
1314 onpass="Switch roles are consistent across all ONOS nodes",
1315 onfail="ONOS nodes have different views of switch roles" )
1316
1317 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001318 for i in range( len( main.activeNodes ) ):
1319 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001320 try:
1321 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001322 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001323 json.dumps(
1324 json.loads( ONOSMastership[ i ] ),
1325 sort_keys=True,
1326 indent=4,
1327 separators=( ',', ': ' ) ) )
1328 except ( ValueError, TypeError ):
1329 main.log.warn( repr( ONOSMastership[ i ] ) )
1330 elif rolesResults and consistentMastership:
1331 mastershipCheck = main.TRUE
1332 mastershipState = ONOSMastership[ 0 ]
1333
1334 main.step( "Get the intents from each controller" )
1335 global intentState
1336 intentState = []
1337 ONOSIntents = []
1338 intentCheck = main.FALSE
1339 consistentIntents = True
1340 intentsResults = True
1341 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001342 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001343 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001344 name="intents-" + str( i ),
1345 args=[],
1346 kwargs={ 'jsonFormat': True } )
1347 threads.append( t )
1348 t.start()
1349
1350 for t in threads:
1351 t.join()
1352 ONOSIntents.append( t.result )
1353
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001354 for i in range( len( ONOSIntents ) ):
1355 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001356 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001357 main.log.error( "Error in getting ONOS" + node + " intents" )
1358 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001359 repr( ONOSIntents[ i ] ) )
1360 intentsResults = False
1361 utilities.assert_equals(
1362 expect=True,
1363 actual=intentsResults,
1364 onpass="No error in reading intents output",
1365 onfail="Error in reading intents from ONOS" )
1366
1367 main.step( "Check for consistency in Intents from each controller" )
1368 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1369 main.log.info( "Intents are consistent across all ONOS " +
1370 "nodes" )
1371 else:
1372 consistentIntents = False
1373 main.log.error( "Intents not consistent" )
1374 utilities.assert_equals(
1375 expect=True,
1376 actual=consistentIntents,
1377 onpass="Intents are consistent across all ONOS nodes",
1378 onfail="ONOS nodes have different views of intents" )
1379
1380 if intentsResults:
1381 # Try to make it easy to figure out what is happening
1382 #
1383 # Intent ONOS1 ONOS2 ...
1384 # 0x01 INSTALLED INSTALLING
1385 # ... ... ...
1386 # ... ... ...
1387 title = " Id"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001388 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001389 title += " " * 10 + "ONOS" + str( n + 1 )
1390 main.log.warn( title )
1391 # get all intent keys in the cluster
1392 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001393 try:
1394 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001395 for nodeStr in ONOSIntents:
1396 node = json.loads( nodeStr )
1397 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001398 keys.append( intent.get( 'id' ) )
1399 keys = set( keys )
1400 # For each intent key, print the state on each node
1401 for key in keys:
1402 row = "%-13s" % key
1403 for nodeStr in ONOSIntents:
1404 node = json.loads( nodeStr )
1405 for intent in node:
1406 if intent.get( 'id', "Error" ) == key:
1407 row += "%-15s" % intent.get( 'state' )
1408 main.log.warn( row )
1409 # End of intent state table
1410 except ValueError as e:
1411 main.log.exception( e )
1412 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001413
1414 if intentsResults and not consistentIntents:
1415 # print the json objects
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001416 n = str( main.activeNodes[-1] + 1 )
1417 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001418 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1419 sort_keys=True,
1420 indent=4,
1421 separators=( ',', ': ' ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001422 for i in range( len( ONOSIntents ) ):
1423 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001424 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001425 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001426 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1427 sort_keys=True,
1428 indent=4,
1429 separators=( ',', ': ' ) ) )
1430 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001431 main.log.debug( "ONOS" + node + " intents match ONOS" +
1432 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001433 elif intentsResults and consistentIntents:
1434 intentCheck = main.TRUE
1435 intentState = ONOSIntents[ 0 ]
1436
1437 main.step( "Get the flows from each controller" )
1438 global flowState
1439 flowState = []
1440 ONOSFlows = []
1441 ONOSFlowsJson = []
1442 flowCheck = main.FALSE
1443 consistentFlows = True
1444 flowsResults = True
1445 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001446 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001447 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001448 name="flows-" + str( i ),
1449 args=[],
1450 kwargs={ 'jsonFormat': True } )
1451 threads.append( t )
1452 t.start()
1453
1454 # NOTE: Flows command can take some time to run
1455 time.sleep(30)
1456 for t in threads:
1457 t.join()
1458 result = t.result
1459 ONOSFlows.append( result )
1460
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001461 for i in range( len( ONOSFlows ) ):
1462 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001463 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1464 main.log.error( "Error in getting ONOS" + num + " flows" )
1465 main.log.warn( "ONOS" + num + " flows response: " +
1466 repr( ONOSFlows[ i ] ) )
1467 flowsResults = False
1468 ONOSFlowsJson.append( None )
1469 else:
1470 try:
1471 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1472 except ( ValueError, TypeError ):
1473 # FIXME: change this to log.error?
1474 main.log.exception( "Error in parsing ONOS" + num +
1475 " response as json." )
1476 main.log.error( repr( ONOSFlows[ i ] ) )
1477 ONOSFlowsJson.append( None )
1478 flowsResults = False
1479 utilities.assert_equals(
1480 expect=True,
1481 actual=flowsResults,
1482 onpass="No error in reading flows output",
1483 onfail="Error in reading flows from ONOS" )
1484
1485 main.step( "Check for consistency in Flows from each controller" )
1486 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1487 if all( tmp ):
1488 main.log.info( "Flow count is consistent across all ONOS nodes" )
1489 else:
1490 consistentFlows = False
1491 utilities.assert_equals(
1492 expect=True,
1493 actual=consistentFlows,
1494 onpass="The flow count is consistent across all ONOS nodes",
1495 onfail="ONOS nodes have different flow counts" )
1496
1497 if flowsResults and not consistentFlows:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001498 for i in range( len( ONOSFlows ) ):
1499 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001500 try:
1501 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001502 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001503 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1504 indent=4, separators=( ',', ': ' ) ) )
1505 except ( ValueError, TypeError ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001506 main.log.warn( "ONOS" + node + " flows: " +
1507 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001508 elif flowsResults and consistentFlows:
1509 flowCheck = main.TRUE
1510 flowState = ONOSFlows[ 0 ]
1511
1512 main.step( "Get the OF Table entries" )
1513 global flows
1514 flows = []
1515 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001516 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001517 if flowCheck == main.FALSE:
1518 for table in flows:
1519 main.log.warn( table )
1520 # TODO: Compare switch flow tables with ONOS flow tables
1521
1522 main.step( "Start continuous pings" )
1523 main.Mininet2.pingLong(
1524 src=main.params[ 'PING' ][ 'source1' ],
1525 target=main.params[ 'PING' ][ 'target1' ],
1526 pingTime=500 )
1527 main.Mininet2.pingLong(
1528 src=main.params[ 'PING' ][ 'source2' ],
1529 target=main.params[ 'PING' ][ 'target2' ],
1530 pingTime=500 )
1531 main.Mininet2.pingLong(
1532 src=main.params[ 'PING' ][ 'source3' ],
1533 target=main.params[ 'PING' ][ 'target3' ],
1534 pingTime=500 )
1535 main.Mininet2.pingLong(
1536 src=main.params[ 'PING' ][ 'source4' ],
1537 target=main.params[ 'PING' ][ 'target4' ],
1538 pingTime=500 )
1539 main.Mininet2.pingLong(
1540 src=main.params[ 'PING' ][ 'source5' ],
1541 target=main.params[ 'PING' ][ 'target5' ],
1542 pingTime=500 )
1543 main.Mininet2.pingLong(
1544 src=main.params[ 'PING' ][ 'source6' ],
1545 target=main.params[ 'PING' ][ 'target6' ],
1546 pingTime=500 )
1547 main.Mininet2.pingLong(
1548 src=main.params[ 'PING' ][ 'source7' ],
1549 target=main.params[ 'PING' ][ 'target7' ],
1550 pingTime=500 )
1551 main.Mininet2.pingLong(
1552 src=main.params[ 'PING' ][ 'source8' ],
1553 target=main.params[ 'PING' ][ 'target8' ],
1554 pingTime=500 )
1555 main.Mininet2.pingLong(
1556 src=main.params[ 'PING' ][ 'source9' ],
1557 target=main.params[ 'PING' ][ 'target9' ],
1558 pingTime=500 )
1559 main.Mininet2.pingLong(
1560 src=main.params[ 'PING' ][ 'source10' ],
1561 target=main.params[ 'PING' ][ 'target10' ],
1562 pingTime=500 )
1563
1564 main.step( "Collecting topology information from ONOS" )
1565 devices = []
1566 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001567 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001568 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001569 name="devices-" + str( i ),
1570 args=[ ] )
1571 threads.append( t )
1572 t.start()
1573
1574 for t in threads:
1575 t.join()
1576 devices.append( t.result )
1577 hosts = []
1578 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001579 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001580 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001581 name="hosts-" + str( i ),
1582 args=[ ] )
1583 threads.append( t )
1584 t.start()
1585
1586 for t in threads:
1587 t.join()
1588 try:
1589 hosts.append( json.loads( t.result ) )
1590 except ( ValueError, TypeError ):
1591 # FIXME: better handling of this, print which node
1592 # Maybe use thread name?
1593 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001594 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001595 hosts.append( None )
1596
1597 ports = []
1598 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001599 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001600 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001601 name="ports-" + str( i ),
1602 args=[ ] )
1603 threads.append( t )
1604 t.start()
1605
1606 for t in threads:
1607 t.join()
1608 ports.append( t.result )
1609 links = []
1610 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001611 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001612 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001613 name="links-" + str( i ),
1614 args=[ ] )
1615 threads.append( t )
1616 t.start()
1617
1618 for t in threads:
1619 t.join()
1620 links.append( t.result )
1621 clusters = []
1622 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001623 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001624 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001625 name="clusters-" + str( i ),
1626 args=[ ] )
1627 threads.append( t )
1628 t.start()
1629
1630 for t in threads:
1631 t.join()
1632 clusters.append( t.result )
1633 # Compare json objects for hosts and dataplane clusters
1634
1635 # hosts
1636 main.step( "Host view is consistent across ONOS nodes" )
1637 consistentHostsResult = main.TRUE
1638 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001639 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001640 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001641 if hosts[ controller ] == hosts[ 0 ]:
1642 continue
1643 else: # hosts not consistent
1644 main.log.error( "hosts from ONOS" +
1645 controllerStr +
1646 " is inconsistent with ONOS1" )
1647 main.log.warn( repr( hosts[ controller ] ) )
1648 consistentHostsResult = main.FALSE
1649
1650 else:
1651 main.log.error( "Error in getting ONOS hosts from ONOS" +
1652 controllerStr )
1653 consistentHostsResult = main.FALSE
1654 main.log.warn( "ONOS" + controllerStr +
1655 " hosts response: " +
1656 repr( hosts[ controller ] ) )
1657 utilities.assert_equals(
1658 expect=main.TRUE,
1659 actual=consistentHostsResult,
1660 onpass="Hosts view is consistent across all ONOS nodes",
1661 onfail="ONOS nodes have different views of hosts" )
1662
1663 main.step( "Each host has an IP address" )
1664 ipResult = main.TRUE
1665 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001666 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001667 if hosts[ controller ]:
1668 for host in hosts[ controller ]:
1669 if not host.get( 'ipAddresses', [ ] ):
1670 main.log.error( "Error with host ips on controller" +
1671 controllerStr + ": " + str( host ) )
1672 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001673 utilities.assert_equals(
1674 expect=main.TRUE,
1675 actual=ipResult,
1676 onpass="The ips of the hosts aren't empty",
1677 onfail="The ip of at least one host is missing" )
1678
1679 # Strongly connected clusters of devices
1680 main.step( "Cluster view is consistent across ONOS nodes" )
1681 consistentClustersResult = main.TRUE
1682 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001683 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001684 if "Error" not in clusters[ controller ]:
1685 if clusters[ controller ] == clusters[ 0 ]:
1686 continue
1687 else: # clusters not consistent
1688 main.log.error( "clusters from ONOS" + controllerStr +
1689 " is inconsistent with ONOS1" )
1690 consistentClustersResult = main.FALSE
1691
1692 else:
1693 main.log.error( "Error in getting dataplane clusters " +
1694 "from ONOS" + controllerStr )
1695 consistentClustersResult = main.FALSE
1696 main.log.warn( "ONOS" + controllerStr +
1697 " clusters response: " +
1698 repr( clusters[ controller ] ) )
1699 utilities.assert_equals(
1700 expect=main.TRUE,
1701 actual=consistentClustersResult,
1702 onpass="Clusters view is consistent across all ONOS nodes",
1703 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001704 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001705 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001706
Jon Hall5cf14d52015-07-16 12:15:19 -07001707 # there should always only be one cluster
1708 main.step( "Cluster view correct across ONOS nodes" )
1709 try:
1710 numClusters = len( json.loads( clusters[ 0 ] ) )
1711 except ( ValueError, TypeError ):
1712 main.log.exception( "Error parsing clusters[0]: " +
1713 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001714 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001715 clusterResults = main.FALSE
1716 if numClusters == 1:
1717 clusterResults = main.TRUE
1718 utilities.assert_equals(
1719 expect=1,
1720 actual=numClusters,
1721 onpass="ONOS shows 1 SCC",
1722 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1723
1724 main.step( "Comparing ONOS topology to MN" )
1725 devicesResults = main.TRUE
1726 linksResults = main.TRUE
1727 hostsResults = main.TRUE
1728 mnSwitches = main.Mininet1.getSwitches()
1729 mnLinks = main.Mininet1.getLinks()
1730 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001731 for controller in main.activeNodes:
1732 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001733 if devices[ controller ] and ports[ controller ] and\
1734 "Error" not in devices[ controller ] and\
1735 "Error" not in ports[ controller ]:
Jon Hall6e709752016-02-01 13:38:46 -08001736 currentDevicesResult = main.Mininet1.compareSwitches(
1737 mnSwitches,
1738 json.loads( devices[ controller ] ),
1739 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001740 else:
1741 currentDevicesResult = main.FALSE
1742 utilities.assert_equals( expect=main.TRUE,
1743 actual=currentDevicesResult,
1744 onpass="ONOS" + controllerStr +
1745 " Switches view is correct",
1746 onfail="ONOS" + controllerStr +
1747 " Switches view is incorrect" )
1748 if links[ controller ] and "Error" not in links[ controller ]:
1749 currentLinksResult = main.Mininet1.compareLinks(
1750 mnSwitches, mnLinks,
1751 json.loads( links[ controller ] ) )
1752 else:
1753 currentLinksResult = main.FALSE
1754 utilities.assert_equals( expect=main.TRUE,
1755 actual=currentLinksResult,
1756 onpass="ONOS" + controllerStr +
1757 " links view is correct",
1758 onfail="ONOS" + controllerStr +
1759 " links view is incorrect" )
1760
Jon Hall657cdf62015-12-17 14:40:51 -08001761 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001762 currentHostsResult = main.Mininet1.compareHosts(
1763 mnHosts,
1764 hosts[ controller ] )
1765 else:
1766 currentHostsResult = main.FALSE
1767 utilities.assert_equals( expect=main.TRUE,
1768 actual=currentHostsResult,
1769 onpass="ONOS" + controllerStr +
1770 " hosts exist in Mininet",
1771 onfail="ONOS" + controllerStr +
1772 " hosts don't match Mininet" )
1773
1774 devicesResults = devicesResults and currentDevicesResult
1775 linksResults = linksResults and currentLinksResult
1776 hostsResults = hostsResults and currentHostsResult
1777
1778 main.step( "Device information is correct" )
1779 utilities.assert_equals(
1780 expect=main.TRUE,
1781 actual=devicesResults,
1782 onpass="Device information is correct",
1783 onfail="Device information is incorrect" )
1784
1785 main.step( "Links are correct" )
1786 utilities.assert_equals(
1787 expect=main.TRUE,
1788 actual=linksResults,
1789 onpass="Link are correct",
1790 onfail="Links are incorrect" )
1791
1792 main.step( "Hosts are correct" )
1793 utilities.assert_equals(
1794 expect=main.TRUE,
1795 actual=hostsResults,
1796 onpass="Hosts are correct",
1797 onfail="Hosts are incorrect" )
1798
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001799 def CASE61( self, main ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001800 """
1801 The Failure case.
1802 """
Jon Halle1a3b752015-07-22 13:02:46 -07001803 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001804 assert main, "main not defined"
1805 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001806 assert main.CLIs, "main.CLIs not defined"
1807 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001808 main.case( "Kill minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -07001809
1810 main.step( "Checking ONOS Logs for errors" )
1811 for node in main.nodes:
1812 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1813 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1814
Jon Hall3b489db2015-10-05 14:38:37 -07001815 n = len( main.nodes ) # Number of nodes
1816 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
1817 main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
1818 if n > 3:
1819 main.kill.append( p - 1 )
1820 # NOTE: This only works for cluster sizes of 3,5, or 7.
1821
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001822 main.step( "Kill " + str( len( main.kill ) ) + " ONOS nodes" )
Jon Hall3b489db2015-10-05 14:38:37 -07001823 killResults = main.TRUE
1824 for i in main.kill:
1825 killResults = killResults and\
1826 main.ONOSbench.onosKill( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001827 main.activeNodes.remove( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001828 utilities.assert_equals( expect=main.TRUE, actual=killResults,
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001829 onpass="ONOS nodes killed successfully",
1830 onfail="ONOS nodes NOT successfully killed" )
1831
Jon Halld2871c22016-07-26 11:01:14 -07001832 main.step( "Checking ONOS nodes" )
1833 nodeResults = utilities.retry( main.HA.nodesCheck,
1834 False,
1835 args=[main.activeNodes],
1836 sleep=15,
1837 attempts=5 )
1838
1839 utilities.assert_equals( expect=True, actual=nodeResults,
1840 onpass="Nodes check successful",
1841 onfail="Nodes check NOT successful" )
1842
1843 if not nodeResults:
1844 for i in main.activeNodes:
1845 cli = main.CLIs[i]
1846 main.log.debug( "{} components not ACTIVE: \n{}".format(
1847 cli.name,
1848 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
1849 main.log.error( "Failed to start ONOS, stopping test" )
1850 main.cleanup()
1851 main.exit()
1852
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001853 def CASE62( self, main ):
1854 """
1855 The bring up stopped nodes
1856 """
1857 import time
1858 assert main.numCtrls, "main.numCtrls not defined"
1859 assert main, "main not defined"
1860 assert utilities.assert_equals, "utilities.assert_equals not defined"
1861 assert main.CLIs, "main.CLIs not defined"
1862 assert main.nodes, "main.nodes not defined"
1863 assert main.kill, "main.kill not defined"
1864 main.case( "Restart minority of ONOS nodes" )
1865
1866 main.step( "Restarting " + str( len( main.kill ) ) + " ONOS nodes" )
1867 startResults = main.TRUE
1868 restartTime = time.time()
1869 for i in main.kill:
1870 startResults = startResults and\
1871 main.ONOSbench.onosStart( main.nodes[i].ip_address )
1872 utilities.assert_equals( expect=main.TRUE, actual=startResults,
1873 onpass="ONOS nodes started successfully",
1874 onfail="ONOS nodes NOT successfully started" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001875
1876 main.step( "Checking if ONOS is up yet" )
1877 count = 0
1878 onosIsupResult = main.FALSE
1879 while onosIsupResult == main.FALSE and count < 10:
Jon Hall3b489db2015-10-05 14:38:37 -07001880 onosIsupResult = main.TRUE
1881 for i in main.kill:
1882 onosIsupResult = onosIsupResult and\
1883 main.ONOSbench.isup( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001884 count = count + 1
Jon Hall5cf14d52015-07-16 12:15:19 -07001885 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1886 onpass="ONOS restarted successfully",
1887 onfail="ONOS restart NOT successful" )
1888
Jon Halle1a3b752015-07-22 13:02:46 -07001889 main.step( "Restarting ONOS main.CLIs" )
Jon Hall3b489db2015-10-05 14:38:37 -07001890 cliResults = main.TRUE
1891 for i in main.kill:
1892 cliResults = cliResults and\
1893 main.CLIs[i].startOnosCli( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001894 main.activeNodes.append( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001895 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1896 onpass="ONOS cli restarted",
1897 onfail="ONOS cli did not restart" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001898 main.activeNodes.sort()
1899 try:
1900 assert list( set( main.activeNodes ) ) == main.activeNodes,\
1901 "List of active nodes has duplicates, this likely indicates something was run out of order"
1902 except AssertionError:
1903 main.log.exception( "" )
1904 main.cleanup()
1905 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -07001906
1907 # Grab the time of restart so we chan check how long the gossip
1908 # protocol has had time to work
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001909 main.restartTime = time.time() - restartTime
Jon Hall5cf14d52015-07-16 12:15:19 -07001910 main.log.debug( "Restart time: " + str( main.restartTime ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001911 # TODO: MAke this configurable. Also, we are breaking the above timer
Jon Halld2871c22016-07-26 11:01:14 -07001912 main.step( "Checking ONOS nodes" )
1913 nodeResults = utilities.retry( main.HA.nodesCheck,
1914 False,
1915 args=[main.activeNodes],
1916 sleep=15,
1917 attempts=5 )
1918
1919 utilities.assert_equals( expect=True, actual=nodeResults,
1920 onpass="Nodes check successful",
1921 onfail="Nodes check NOT successful" )
1922
1923 if not nodeResults:
1924 for i in main.activeNodes:
1925 cli = main.CLIs[i]
1926 main.log.debug( "{} components not ACTIVE: \n{}".format(
1927 cli.name,
1928 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
1929 main.log.error( "Failed to start ONOS, stopping test" )
1930 main.cleanup()
1931 main.exit()
1932
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001933 node = main.activeNodes[0]
1934 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1935 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1936 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001937
Jon Halla440e872016-03-31 15:15:50 -07001938 main.step( "Rerun for election on the node(s) that were killed" )
1939 runResults = main.TRUE
1940 for i in main.kill:
1941 runResults = runResults and\
1942 main.CLIs[i].electionTestRun()
1943 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1944 onpass="ONOS nodes reran for election topic",
1945 onfail="Errror rerunning for election" )
1946
Jon Hall5cf14d52015-07-16 12:15:19 -07001947 def CASE7( self, main ):
1948 """
1949 Check state after ONOS failure
1950 """
1951 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001952 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001953 assert main, "main not defined"
1954 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001955 assert main.CLIs, "main.CLIs not defined"
1956 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001957 try:
1958 main.kill
1959 except AttributeError:
1960 main.kill = []
1961
Jon Hall5cf14d52015-07-16 12:15:19 -07001962 main.case( "Running ONOS Constant State Tests" )
1963
1964 main.step( "Check that each switch has a master" )
1965 # Assert that each device has a master
1966 rolesNotNull = main.TRUE
1967 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001968 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001969 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001970 name="rolesNotNull-" + str( i ),
1971 args=[ ] )
1972 threads.append( t )
1973 t.start()
1974
1975 for t in threads:
1976 t.join()
1977 rolesNotNull = rolesNotNull and t.result
1978 utilities.assert_equals(
1979 expect=main.TRUE,
1980 actual=rolesNotNull,
1981 onpass="Each device has a master",
1982 onfail="Some devices don't have a master assigned" )
1983
1984 main.step( "Read device roles from ONOS" )
1985 ONOSMastership = []
Jon Halla440e872016-03-31 15:15:50 -07001986 mastershipCheck = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001987 consistentMastership = True
1988 rolesResults = True
1989 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001990 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001991 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001992 name="roles-" + str( i ),
1993 args=[] )
1994 threads.append( t )
1995 t.start()
1996
1997 for t in threads:
1998 t.join()
1999 ONOSMastership.append( t.result )
2000
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002001 for i in range( len( ONOSMastership ) ):
2002 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002003 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002004 main.log.error( "Error in getting ONOS" + node + " roles" )
2005 main.log.warn( "ONOS" + node + " mastership response: " +
2006 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002007 rolesResults = False
2008 utilities.assert_equals(
2009 expect=True,
2010 actual=rolesResults,
2011 onpass="No error in reading roles output",
2012 onfail="Error in reading roles from ONOS" )
2013
2014 main.step( "Check for consistency in roles from each controller" )
2015 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
2016 main.log.info(
2017 "Switch roles are consistent across all ONOS nodes" )
2018 else:
2019 consistentMastership = False
2020 utilities.assert_equals(
2021 expect=True,
2022 actual=consistentMastership,
2023 onpass="Switch roles are consistent across all ONOS nodes",
2024 onfail="ONOS nodes have different views of switch roles" )
2025
2026 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002027 for i in range( len( ONOSMastership ) ):
2028 node = str( main.activeNodes[i] + 1 )
2029 main.log.warn( "ONOS" + node + " roles: ",
2030 json.dumps( json.loads( ONOSMastership[ i ] ),
2031 sort_keys=True,
2032 indent=4,
2033 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07002034 elif rolesResults and consistentMastership:
2035 mastershipCheck = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002036
2037 # NOTE: we expect mastership to change on controller failure
Jon Hall5cf14d52015-07-16 12:15:19 -07002038
2039 main.step( "Get the intents and compare across all nodes" )
2040 ONOSIntents = []
2041 intentCheck = main.FALSE
2042 consistentIntents = True
2043 intentsResults = True
2044 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002045 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002046 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07002047 name="intents-" + str( i ),
2048 args=[],
2049 kwargs={ 'jsonFormat': True } )
2050 threads.append( t )
2051 t.start()
2052
2053 for t in threads:
2054 t.join()
2055 ONOSIntents.append( t.result )
2056
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002057 for i in range( len( ONOSIntents) ):
2058 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002059 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002060 main.log.error( "Error in getting ONOS" + node + " intents" )
2061 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002062 repr( ONOSIntents[ i ] ) )
2063 intentsResults = False
2064 utilities.assert_equals(
2065 expect=True,
2066 actual=intentsResults,
2067 onpass="No error in reading intents output",
2068 onfail="Error in reading intents from ONOS" )
2069
2070 main.step( "Check for consistency in Intents from each controller" )
2071 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2072 main.log.info( "Intents are consistent across all ONOS " +
2073 "nodes" )
2074 else:
2075 consistentIntents = False
2076
2077 # Try to make it easy to figure out what is happening
2078 #
2079 # Intent ONOS1 ONOS2 ...
2080 # 0x01 INSTALLED INSTALLING
2081 # ... ... ...
2082 # ... ... ...
2083 title = " ID"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002084 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002085 title += " " * 10 + "ONOS" + str( n + 1 )
2086 main.log.warn( title )
2087 # get all intent keys in the cluster
2088 keys = []
2089 for nodeStr in ONOSIntents:
2090 node = json.loads( nodeStr )
2091 for intent in node:
2092 keys.append( intent.get( 'id' ) )
2093 keys = set( keys )
2094 for key in keys:
2095 row = "%-13s" % key
2096 for nodeStr in ONOSIntents:
2097 node = json.loads( nodeStr )
2098 for intent in node:
2099 if intent.get( 'id' ) == key:
2100 row += "%-15s" % intent.get( 'state' )
2101 main.log.warn( row )
2102 # End table view
2103
2104 utilities.assert_equals(
2105 expect=True,
2106 actual=consistentIntents,
2107 onpass="Intents are consistent across all ONOS nodes",
2108 onfail="ONOS nodes have different views of intents" )
2109 intentStates = []
2110 for node in ONOSIntents: # Iter through ONOS nodes
2111 nodeStates = []
2112 # Iter through intents of a node
2113 try:
2114 for intent in json.loads( node ):
2115 nodeStates.append( intent[ 'state' ] )
2116 except ( ValueError, TypeError ):
2117 main.log.exception( "Error in parsing intents" )
2118 main.log.error( repr( node ) )
2119 intentStates.append( nodeStates )
2120 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2121 main.log.info( dict( out ) )
2122
2123 if intentsResults and not consistentIntents:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002124 for i in range( len( main.activeNodes ) ):
2125 node = str( main.activeNodes[i] + 1 )
2126 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002127 main.log.warn( json.dumps(
2128 json.loads( ONOSIntents[ i ] ),
2129 sort_keys=True,
2130 indent=4,
2131 separators=( ',', ': ' ) ) )
2132 elif intentsResults and consistentIntents:
2133 intentCheck = main.TRUE
2134
2135 # NOTE: Store has no durability, so intents are lost across system
2136 # restarts
2137 main.step( "Compare current intents with intents before the failure" )
2138 # NOTE: this requires case 5 to pass for intentState to be set.
2139 # maybe we should stop the test if that fails?
2140 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002141 try:
2142 intentState
2143 except NameError:
2144 main.log.warn( "No previous intent state was saved" )
2145 else:
2146 if intentState and intentState == ONOSIntents[ 0 ]:
2147 sameIntents = main.TRUE
2148 main.log.info( "Intents are consistent with before failure" )
2149 # TODO: possibly the states have changed? we may need to figure out
2150 # what the acceptable states are
2151 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2152 sameIntents = main.TRUE
2153 try:
2154 before = json.loads( intentState )
2155 after = json.loads( ONOSIntents[ 0 ] )
2156 for intent in before:
2157 if intent not in after:
2158 sameIntents = main.FALSE
2159 main.log.debug( "Intent is not currently in ONOS " +
2160 "(at least in the same form):" )
2161 main.log.debug( json.dumps( intent ) )
2162 except ( ValueError, TypeError ):
2163 main.log.exception( "Exception printing intents" )
2164 main.log.debug( repr( ONOSIntents[0] ) )
2165 main.log.debug( repr( intentState ) )
2166 if sameIntents == main.FALSE:
2167 try:
2168 main.log.debug( "ONOS intents before: " )
2169 main.log.debug( json.dumps( json.loads( intentState ),
2170 sort_keys=True, indent=4,
2171 separators=( ',', ': ' ) ) )
2172 main.log.debug( "Current ONOS intents: " )
2173 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2174 sort_keys=True, indent=4,
2175 separators=( ',', ': ' ) ) )
2176 except ( ValueError, TypeError ):
2177 main.log.exception( "Exception printing intents" )
2178 main.log.debug( repr( ONOSIntents[0] ) )
2179 main.log.debug( repr( intentState ) )
2180 utilities.assert_equals(
2181 expect=main.TRUE,
2182 actual=sameIntents,
2183 onpass="Intents are consistent with before failure",
2184 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002185 intentCheck = intentCheck and sameIntents
2186
2187 main.step( "Get the OF Table entries and compare to before " +
2188 "component failure" )
2189 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002190 for i in range( 28 ):
2191 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002192 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002193 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2194 FlowTables = FlowTables and curSwitch
2195 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002196 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002197 utilities.assert_equals(
2198 expect=main.TRUE,
2199 actual=FlowTables,
2200 onpass="No changes were found in the flow tables",
2201 onfail="Changes were found in the flow tables" )
2202
2203 main.Mininet2.pingLongKill()
2204 '''
2205 main.step( "Check the continuous pings to ensure that no packets " +
2206 "were dropped during component failure" )
2207 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2208 main.params[ 'TESTONIP' ] )
2209 LossInPings = main.FALSE
2210 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2211 for i in range( 8, 18 ):
2212 main.log.info(
2213 "Checking for a loss in pings along flow from s" +
2214 str( i ) )
2215 LossInPings = main.Mininet2.checkForLoss(
2216 "/tmp/ping.h" +
2217 str( i ) ) or LossInPings
2218 if LossInPings == main.TRUE:
2219 main.log.info( "Loss in ping detected" )
2220 elif LossInPings == main.ERROR:
2221 main.log.info( "There are multiple mininet process running" )
2222 elif LossInPings == main.FALSE:
2223 main.log.info( "No Loss in the pings" )
2224 main.log.info( "No loss of dataplane connectivity" )
2225 utilities.assert_equals(
2226 expect=main.FALSE,
2227 actual=LossInPings,
2228 onpass="No Loss of connectivity",
2229 onfail="Loss of dataplane connectivity detected" )
2230 '''
2231
2232 main.step( "Leadership Election is still functional" )
2233 # Test of LeadershipElection
2234 leaderList = []
Jon Hall5cf14d52015-07-16 12:15:19 -07002235
Jon Hall3b489db2015-10-05 14:38:37 -07002236 restarted = []
2237 for i in main.kill:
2238 restarted.append( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002239 leaderResult = main.TRUE
Jon Hall3b489db2015-10-05 14:38:37 -07002240
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002241 for i in main.activeNodes:
2242 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002243 leaderN = cli.electionTestLeader()
2244 leaderList.append( leaderN )
2245 if leaderN == main.FALSE:
2246 # error in response
2247 main.log.error( "Something is wrong with " +
2248 "electionTestLeader function, check the" +
2249 " error logs" )
2250 leaderResult = main.FALSE
2251 elif leaderN is None:
2252 main.log.error( cli.name +
2253 " shows no leader for the election-app was" +
2254 " elected after the old one died" )
2255 leaderResult = main.FALSE
2256 elif leaderN in restarted:
2257 main.log.error( cli.name + " shows " + str( leaderN ) +
2258 " as leader for the election-app, but it " +
2259 "was restarted" )
2260 leaderResult = main.FALSE
2261 if len( set( leaderList ) ) != 1:
2262 leaderResult = main.FALSE
2263 main.log.error(
2264 "Inconsistent view of leader for the election test app" )
2265 # TODO: print the list
2266 utilities.assert_equals(
2267 expect=main.TRUE,
2268 actual=leaderResult,
2269 onpass="Leadership election passed",
2270 onfail="Something went wrong with Leadership election" )
2271
2272 def CASE8( self, main ):
2273 """
2274 Compare topo
2275 """
2276 import json
2277 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002278 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002279 assert main, "main not defined"
2280 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002281 assert main.CLIs, "main.CLIs not defined"
2282 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002283
2284 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002285 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002286 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002287 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002288 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002289 elapsed = 0
2290 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002291 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002292 startTime = time.time()
2293 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002294 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002295 devicesResults = main.TRUE
2296 linksResults = main.TRUE
2297 hostsResults = main.TRUE
2298 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002299 count += 1
2300 cliStart = time.time()
2301 devices = []
2302 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002303 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002304 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002305 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002306 args=[ main.CLIs[i].devices, [ None ] ],
2307 kwargs= { 'sleep': 5, 'attempts': 5,
2308 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002309 threads.append( t )
2310 t.start()
2311
2312 for t in threads:
2313 t.join()
2314 devices.append( t.result )
2315 hosts = []
2316 ipResult = main.TRUE
2317 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002318 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002319 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002320 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002321 args=[ main.CLIs[i].hosts, [ None ] ],
2322 kwargs= { 'sleep': 5, 'attempts': 5,
2323 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002324 threads.append( t )
2325 t.start()
2326
2327 for t in threads:
2328 t.join()
2329 try:
2330 hosts.append( json.loads( t.result ) )
2331 except ( ValueError, TypeError ):
2332 main.log.exception( "Error parsing hosts results" )
2333 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002334 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002335 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002336 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002337 if hosts[ controller ]:
2338 for host in hosts[ controller ]:
2339 if host is None or host.get( 'ipAddresses', [] ) == []:
2340 main.log.error(
2341 "Error with host ipAddresses on controller" +
2342 controllerStr + ": " + str( host ) )
2343 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002344 ports = []
2345 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002346 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002347 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002348 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002349 args=[ main.CLIs[i].ports, [ None ] ],
2350 kwargs= { 'sleep': 5, 'attempts': 5,
2351 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002352 threads.append( t )
2353 t.start()
2354
2355 for t in threads:
2356 t.join()
2357 ports.append( t.result )
2358 links = []
2359 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002360 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002361 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002362 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002363 args=[ main.CLIs[i].links, [ None ] ],
2364 kwargs= { 'sleep': 5, 'attempts': 5,
2365 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002366 threads.append( t )
2367 t.start()
2368
2369 for t in threads:
2370 t.join()
2371 links.append( t.result )
2372 clusters = []
2373 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002374 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002375 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002376 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002377 args=[ main.CLIs[i].clusters, [ None ] ],
2378 kwargs= { 'sleep': 5, 'attempts': 5,
2379 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002380 threads.append( t )
2381 t.start()
2382
2383 for t in threads:
2384 t.join()
2385 clusters.append( t.result )
2386
2387 elapsed = time.time() - startTime
2388 cliTime = time.time() - cliStart
2389 print "Elapsed time: " + str( elapsed )
2390 print "CLI time: " + str( cliTime )
2391
Jon Hall6e709752016-02-01 13:38:46 -08002392 if all( e is None for e in devices ) and\
2393 all( e is None for e in hosts ) and\
2394 all( e is None for e in ports ) and\
2395 all( e is None for e in links ) and\
2396 all( e is None for e in clusters ):
2397 topoFailMsg = "Could not get topology from ONOS"
2398 main.log.error( topoFailMsg )
2399 continue # Try again, No use trying to compare
2400
Jon Hall5cf14d52015-07-16 12:15:19 -07002401 mnSwitches = main.Mininet1.getSwitches()
2402 mnLinks = main.Mininet1.getLinks()
2403 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002404 for controller in range( len( main.activeNodes ) ):
2405 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002406 if devices[ controller ] and ports[ controller ] and\
2407 "Error" not in devices[ controller ] and\
2408 "Error" not in ports[ controller ]:
2409
Jon Hallc6793552016-01-19 14:18:37 -08002410 try:
2411 currentDevicesResult = main.Mininet1.compareSwitches(
2412 mnSwitches,
2413 json.loads( devices[ controller ] ),
2414 json.loads( ports[ controller ] ) )
2415 except ( TypeError, ValueError ) as e:
2416 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2417 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002418 else:
2419 currentDevicesResult = main.FALSE
2420 utilities.assert_equals( expect=main.TRUE,
2421 actual=currentDevicesResult,
2422 onpass="ONOS" + controllerStr +
2423 " Switches view is correct",
2424 onfail="ONOS" + controllerStr +
2425 " Switches view is incorrect" )
2426
2427 if links[ controller ] and "Error" not in links[ controller ]:
2428 currentLinksResult = main.Mininet1.compareLinks(
2429 mnSwitches, mnLinks,
2430 json.loads( links[ controller ] ) )
2431 else:
2432 currentLinksResult = main.FALSE
2433 utilities.assert_equals( expect=main.TRUE,
2434 actual=currentLinksResult,
2435 onpass="ONOS" + controllerStr +
2436 " links view is correct",
2437 onfail="ONOS" + controllerStr +
2438 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002439 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002440 currentHostsResult = main.Mininet1.compareHosts(
2441 mnHosts,
2442 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002443 elif hosts[ controller ] == []:
2444 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002445 else:
2446 currentHostsResult = main.FALSE
2447 utilities.assert_equals( expect=main.TRUE,
2448 actual=currentHostsResult,
2449 onpass="ONOS" + controllerStr +
2450 " hosts exist in Mininet",
2451 onfail="ONOS" + controllerStr +
2452 " hosts don't match Mininet" )
2453 # CHECKING HOST ATTACHMENT POINTS
2454 hostAttachment = True
2455 zeroHosts = False
2456 # FIXME: topo-HA/obelisk specific mappings:
2457 # key is mac and value is dpid
2458 mappings = {}
2459 for i in range( 1, 29 ): # hosts 1 through 28
2460 # set up correct variables:
2461 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2462 if i == 1:
2463 deviceId = "1000".zfill(16)
2464 elif i == 2:
2465 deviceId = "2000".zfill(16)
2466 elif i == 3:
2467 deviceId = "3000".zfill(16)
2468 elif i == 4:
2469 deviceId = "3004".zfill(16)
2470 elif i == 5:
2471 deviceId = "5000".zfill(16)
2472 elif i == 6:
2473 deviceId = "6000".zfill(16)
2474 elif i == 7:
2475 deviceId = "6007".zfill(16)
2476 elif i >= 8 and i <= 17:
2477 dpid = '3' + str( i ).zfill( 3 )
2478 deviceId = dpid.zfill(16)
2479 elif i >= 18 and i <= 27:
2480 dpid = '6' + str( i ).zfill( 3 )
2481 deviceId = dpid.zfill(16)
2482 elif i == 28:
2483 deviceId = "2800".zfill(16)
2484 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002485 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002486 if hosts[ controller ] == []:
2487 main.log.warn( "There are no hosts discovered" )
2488 zeroHosts = True
2489 else:
2490 for host in hosts[ controller ]:
2491 mac = None
2492 location = None
2493 device = None
2494 port = None
2495 try:
2496 mac = host.get( 'mac' )
2497 assert mac, "mac field could not be found for this host object"
2498
2499 location = host.get( 'location' )
2500 assert location, "location field could not be found for this host object"
2501
2502 # Trim the protocol identifier off deviceId
2503 device = str( location.get( 'elementId' ) ).split(':')[1]
2504 assert device, "elementId field could not be found for this host location object"
2505
2506 port = location.get( 'port' )
2507 assert port, "port field could not be found for this host location object"
2508
2509 # Now check if this matches where they should be
2510 if mac and device and port:
2511 if str( port ) != "1":
2512 main.log.error( "The attachment port is incorrect for " +
2513 "host " + str( mac ) +
2514 ". Expected: 1 Actual: " + str( port) )
2515 hostAttachment = False
2516 if device != mappings[ str( mac ) ]:
2517 main.log.error( "The attachment device is incorrect for " +
2518 "host " + str( mac ) +
2519 ". Expected: " + mappings[ str( mac ) ] +
2520 " Actual: " + device )
2521 hostAttachment = False
2522 else:
2523 hostAttachment = False
2524 except AssertionError:
2525 main.log.exception( "Json object not as expected" )
2526 main.log.error( repr( host ) )
2527 hostAttachment = False
2528 else:
2529 main.log.error( "No hosts json output or \"Error\"" +
2530 " in output. hosts = " +
2531 repr( hosts[ controller ] ) )
2532 if zeroHosts is False:
2533 hostAttachment = True
2534
2535 # END CHECKING HOST ATTACHMENT POINTS
2536 devicesResults = devicesResults and currentDevicesResult
2537 linksResults = linksResults and currentLinksResult
2538 hostsResults = hostsResults and currentHostsResult
2539 hostAttachmentResults = hostAttachmentResults and\
2540 hostAttachment
Jon Halla440e872016-03-31 15:15:50 -07002541 topoResult = ( devicesResults and linksResults
2542 and hostsResults and ipResult and
2543 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002544 utilities.assert_equals( expect=True,
2545 actual=topoResult,
2546 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002547 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002548 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002549
2550 # Compare json objects for hosts and dataplane clusters
2551
2552 # hosts
2553 main.step( "Hosts view is consistent across all ONOS nodes" )
2554 consistentHostsResult = main.TRUE
2555 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002556 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002557 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002558 if hosts[ controller ] == hosts[ 0 ]:
2559 continue
2560 else: # hosts not consistent
2561 main.log.error( "hosts from ONOS" + controllerStr +
2562 " is inconsistent with ONOS1" )
2563 main.log.warn( repr( hosts[ controller ] ) )
2564 consistentHostsResult = main.FALSE
2565
2566 else:
2567 main.log.error( "Error in getting ONOS hosts from ONOS" +
2568 controllerStr )
2569 consistentHostsResult = main.FALSE
2570 main.log.warn( "ONOS" + controllerStr +
2571 " hosts response: " +
2572 repr( hosts[ controller ] ) )
2573 utilities.assert_equals(
2574 expect=main.TRUE,
2575 actual=consistentHostsResult,
2576 onpass="Hosts view is consistent across all ONOS nodes",
2577 onfail="ONOS nodes have different views of hosts" )
2578
2579 main.step( "Hosts information is correct" )
2580 hostsResults = hostsResults and ipResult
2581 utilities.assert_equals(
2582 expect=main.TRUE,
2583 actual=hostsResults,
2584 onpass="Host information is correct",
2585 onfail="Host information is incorrect" )
2586
2587 main.step( "Host attachment points to the network" )
2588 utilities.assert_equals(
2589 expect=True,
2590 actual=hostAttachmentResults,
2591 onpass="Hosts are correctly attached to the network",
2592 onfail="ONOS did not correctly attach hosts to the network" )
2593
2594 # Strongly connected clusters of devices
2595 main.step( "Clusters view is consistent across all ONOS nodes" )
2596 consistentClustersResult = main.TRUE
2597 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002598 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002599 if "Error" not in clusters[ controller ]:
2600 if clusters[ controller ] == clusters[ 0 ]:
2601 continue
2602 else: # clusters not consistent
2603 main.log.error( "clusters from ONOS" +
2604 controllerStr +
2605 " is inconsistent with ONOS1" )
2606 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002607 else:
2608 main.log.error( "Error in getting dataplane clusters " +
2609 "from ONOS" + controllerStr )
2610 consistentClustersResult = main.FALSE
2611 main.log.warn( "ONOS" + controllerStr +
2612 " clusters response: " +
2613 repr( clusters[ controller ] ) )
2614 utilities.assert_equals(
2615 expect=main.TRUE,
2616 actual=consistentClustersResult,
2617 onpass="Clusters view is consistent across all ONOS nodes",
2618 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002619 if not consistentClustersResult:
2620 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002621
2622 main.step( "There is only one SCC" )
2623 # there should always only be one cluster
2624 try:
2625 numClusters = len( json.loads( clusters[ 0 ] ) )
2626 except ( ValueError, TypeError ):
2627 main.log.exception( "Error parsing clusters[0]: " +
2628 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002629 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002630 clusterResults = main.FALSE
2631 if numClusters == 1:
2632 clusterResults = main.TRUE
2633 utilities.assert_equals(
2634 expect=1,
2635 actual=numClusters,
2636 onpass="ONOS shows 1 SCC",
2637 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2638
2639 topoResult = ( devicesResults and linksResults
2640 and hostsResults and consistentHostsResult
2641 and consistentClustersResult and clusterResults
2642 and ipResult and hostAttachmentResults )
2643
2644 topoResult = topoResult and int( count <= 2 )
2645 note = "note it takes about " + str( int( cliTime ) ) + \
2646 " seconds for the test to make all the cli calls to fetch " +\
2647 "the topology from each ONOS instance"
2648 main.log.info(
2649 "Very crass estimate for topology discovery/convergence( " +
2650 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2651 str( count ) + " tries" )
2652
2653 main.step( "Device information is correct" )
2654 utilities.assert_equals(
2655 expect=main.TRUE,
2656 actual=devicesResults,
2657 onpass="Device information is correct",
2658 onfail="Device information is incorrect" )
2659
2660 main.step( "Links are correct" )
2661 utilities.assert_equals(
2662 expect=main.TRUE,
2663 actual=linksResults,
2664 onpass="Link are correct",
2665 onfail="Links are incorrect" )
2666
Jon Halla440e872016-03-31 15:15:50 -07002667 main.step( "Hosts are correct" )
2668 utilities.assert_equals(
2669 expect=main.TRUE,
2670 actual=hostsResults,
2671 onpass="Hosts are correct",
2672 onfail="Hosts are incorrect" )
2673
Jon Hall5cf14d52015-07-16 12:15:19 -07002674 # FIXME: move this to an ONOS state case
2675 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002676 nodeResults = utilities.retry( main.HA.nodesCheck,
2677 False,
2678 args=[main.activeNodes],
2679 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002680
Jon Hall41d39f12016-04-11 22:54:35 -07002681 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002682 onpass="Nodes check successful",
2683 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002684 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002685 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002686 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002687 main.CLIs[i].name,
2688 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002689
Jon Halld2871c22016-07-26 11:01:14 -07002690 if not topoResult:
2691 main.cleanup()
2692 main.exit()
2693
Jon Hall5cf14d52015-07-16 12:15:19 -07002694 def CASE9( self, main ):
2695 """
2696 Link s3-s28 down
2697 """
2698 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002699 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002700 assert main, "main not defined"
2701 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002702 assert main.CLIs, "main.CLIs not defined"
2703 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002704 # NOTE: You should probably run a topology check after this
2705
2706 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2707
2708 description = "Turn off a link to ensure that Link Discovery " +\
2709 "is working properly"
2710 main.case( description )
2711
2712 main.step( "Kill Link between s3 and s28" )
2713 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2714 main.log.info( "Waiting " + str( linkSleep ) +
2715 " seconds for link down to be discovered" )
2716 time.sleep( linkSleep )
2717 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2718 onpass="Link down successful",
2719 onfail="Failed to bring link down" )
2720 # TODO do some sort of check here
2721
2722 def CASE10( self, main ):
2723 """
2724 Link s3-s28 up
2725 """
2726 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002727 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002728 assert main, "main not defined"
2729 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002730 assert main.CLIs, "main.CLIs not defined"
2731 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002732 # NOTE: You should probably run a topology check after this
2733
2734 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2735
2736 description = "Restore a link to ensure that Link Discovery is " + \
2737 "working properly"
2738 main.case( description )
2739
2740 main.step( "Bring link between s3 and s28 back up" )
2741 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2742 main.log.info( "Waiting " + str( linkSleep ) +
2743 " seconds for link up to be discovered" )
2744 time.sleep( linkSleep )
2745 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2746 onpass="Link up successful",
2747 onfail="Failed to bring link up" )
2748 # TODO do some sort of check here
2749
2750 def CASE11( self, main ):
2751 """
2752 Switch Down
2753 """
2754 # NOTE: You should probably run a topology check after this
2755 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002756 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002757 assert main, "main not defined"
2758 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002759 assert main.CLIs, "main.CLIs not defined"
2760 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002761
2762 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2763
2764 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002765 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002766 main.case( description )
2767 switch = main.params[ 'kill' ][ 'switch' ]
2768 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2769
2770 # TODO: Make this switch parameterizable
2771 main.step( "Kill " + switch )
2772 main.log.info( "Deleting " + switch )
2773 main.Mininet1.delSwitch( switch )
2774 main.log.info( "Waiting " + str( switchSleep ) +
2775 " seconds for switch down to be discovered" )
2776 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002777 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002778 # Peek at the deleted switch
2779 main.log.warn( str( device ) )
2780 result = main.FALSE
2781 if device and device[ 'available' ] is False:
2782 result = main.TRUE
2783 utilities.assert_equals( expect=main.TRUE, actual=result,
2784 onpass="Kill switch successful",
2785 onfail="Failed to kill switch?" )
2786
2787 def CASE12( self, main ):
2788 """
2789 Switch Up
2790 """
2791 # NOTE: You should probably run a topology check after this
2792 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002793 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002794 assert main, "main not defined"
2795 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002796 assert main.CLIs, "main.CLIs not defined"
2797 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002798 assert ONOS1Port, "ONOS1Port not defined"
2799 assert ONOS2Port, "ONOS2Port not defined"
2800 assert ONOS3Port, "ONOS3Port not defined"
2801 assert ONOS4Port, "ONOS4Port not defined"
2802 assert ONOS5Port, "ONOS5Port not defined"
2803 assert ONOS6Port, "ONOS6Port not defined"
2804 assert ONOS7Port, "ONOS7Port not defined"
2805
2806 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2807 switch = main.params[ 'kill' ][ 'switch' ]
2808 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2809 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002810 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002811 description = "Adding a switch to ensure it is discovered correctly"
2812 main.case( description )
2813
2814 main.step( "Add back " + switch )
2815 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2816 for peer in links:
2817 main.Mininet1.addLink( switch, peer )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002818 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002819 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2820 main.log.info( "Waiting " + str( switchSleep ) +
2821 " seconds for switch up to be discovered" )
2822 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002823 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002824 # Peek at the deleted switch
2825 main.log.warn( str( device ) )
2826 result = main.FALSE
2827 if device and device[ 'available' ]:
2828 result = main.TRUE
2829 utilities.assert_equals( expect=main.TRUE, actual=result,
2830 onpass="add switch successful",
2831 onfail="Failed to add switch?" )
2832
2833 def CASE13( self, main ):
2834 """
2835 Clean up
2836 """
2837 import os
2838 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002839 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002840 assert main, "main not defined"
2841 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002842 assert main.CLIs, "main.CLIs not defined"
2843 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002844
2845 # printing colors to terminal
2846 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2847 'blue': '\033[94m', 'green': '\033[92m',
2848 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2849 main.case( "Test Cleanup" )
2850 main.step( "Killing tcpdumps" )
2851 main.Mininet2.stopTcpdump()
2852
2853 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002854 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002855 main.step( "Copying MN pcap and ONOS log files to test station" )
2856 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2857 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002858 # NOTE: MN Pcap file is being saved to logdir.
2859 # We scp this file as MN and TestON aren't necessarily the same vm
2860
2861 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002862 # TODO: Load these from params
2863 # NOTE: must end in /
2864 logFolder = "/opt/onos/log/"
2865 logFiles = [ "karaf.log", "karaf.log.1" ]
2866 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002867 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002868 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002869 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002870 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2871 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002872 # std*.log's
2873 # NOTE: must end in /
2874 logFolder = "/opt/onos/var/"
2875 logFiles = [ "stderr.log", "stdout.log" ]
2876 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002877 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002878 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002879 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002880 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2881 logFolder + f, dstName )
2882 else:
2883 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002884
2885 main.step( "Stopping Mininet" )
2886 mnResult = main.Mininet1.stopNet()
2887 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2888 onpass="Mininet stopped",
2889 onfail="MN cleanup NOT successful" )
2890
2891 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002892 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002893 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2894 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002895
2896 try:
2897 timerLog = open( main.logdir + "/Timers.csv", 'w')
2898 # Overwrite with empty line and close
2899 labels = "Gossip Intents, Restart"
2900 data = str( gossipTime ) + ", " + str( main.restartTime )
2901 timerLog.write( labels + "\n" + data )
2902 timerLog.close()
2903 except NameError, e:
2904 main.log.exception(e)
2905
2906 def CASE14( self, main ):
2907 """
2908 start election app on all onos nodes
2909 """
Jon Halle1a3b752015-07-22 13:02:46 -07002910 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002911 assert main, "main not defined"
2912 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002913 assert main.CLIs, "main.CLIs not defined"
2914 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002915
2916 main.case("Start Leadership Election app")
2917 main.step( "Install leadership election app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002918 onosCli = main.CLIs[ main.activeNodes[0] ]
2919 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002920 utilities.assert_equals(
2921 expect=main.TRUE,
2922 actual=appResult,
2923 onpass="Election app installed",
2924 onfail="Something went wrong with installing Leadership election" )
2925
2926 main.step( "Run for election on each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002927 for i in main.activeNodes:
2928 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002929 time.sleep(5)
2930 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2931 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002932 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002933 expect=True,
2934 actual=sameResult,
2935 onpass="All nodes see the same leaderboards",
2936 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002937
Jon Hall25463a82016-04-13 14:03:52 -07002938 if sameResult:
2939 leader = leaders[ 0 ][ 0 ]
2940 if main.nodes[main.activeNodes[0]].ip_address in leader:
2941 correctLeader = True
2942 else:
2943 correctLeader = False
2944 main.step( "First node was elected leader" )
2945 utilities.assert_equals(
2946 expect=True,
2947 actual=correctLeader,
2948 onpass="Correct leader was elected",
2949 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002950
2951 def CASE15( self, main ):
2952 """
2953 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002954 15.1 Run election on each node
2955 15.2 Check that each node has the same leaders and candidates
2956 15.3 Find current leader and withdraw
2957 15.4 Check that a new node was elected leader
2958 15.5 Check that that new leader was the candidate of old leader
2959 15.6 Run for election on old leader
2960 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2961 15.8 Make sure that the old leader was added to the candidate list
2962
2963 old and new variable prefixes refer to data from before vs after
2964 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002965 """
2966 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002967 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002968 assert main, "main not defined"
2969 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002970 assert main.CLIs, "main.CLIs not defined"
2971 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002972
Jon Hall5cf14d52015-07-16 12:15:19 -07002973 description = "Check that Leadership Election is still functional"
2974 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002975 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002976
Jon Halla440e872016-03-31 15:15:50 -07002977 oldLeaders = [] # list of lists of each nodes' candidates before
2978 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002979 oldLeader = '' # the old leader from oldLeaders, None if not same
2980 newLeader = '' # the new leaders fron newLoeaders, None if not same
2981 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2982 expectNoLeader = False # True when there is only one leader
2983 if main.numCtrls == 1:
2984 expectNoLeader = True
2985
2986 main.step( "Run for election on each node" )
2987 electionResult = main.TRUE
2988
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002989 for i in main.activeNodes: # run test election on each node
2990 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002991 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002992 utilities.assert_equals(
2993 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002994 actual=electionResult,
2995 onpass="All nodes successfully ran for leadership",
2996 onfail="At least one node failed to run for leadership" )
2997
acsmars3a72bde2015-09-02 14:16:22 -07002998 if electionResult == main.FALSE:
2999 main.log.error(
3000 "Skipping Test Case because Election Test App isn't loaded" )
3001 main.skipCase()
3002
acsmars71adceb2015-08-31 15:09:26 -07003003 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07003004 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07003005 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07003006 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07003007 if sameResult:
3008 oldLeader = oldLeaders[ 0 ][ 0 ]
3009 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07003010 else:
Jon Halla440e872016-03-31 15:15:50 -07003011 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07003012 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003013 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003014 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07003015 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07003016 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07003017
3018 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07003019 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07003020 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07003021 if oldLeader is None:
3022 main.log.error( "Leadership isn't consistent." )
3023 withdrawResult = main.FALSE
3024 # Get the CLI of the oldLeader
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003025 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07003026 if oldLeader == main.nodes[ i ].ip_address:
3027 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07003028 break
3029 else: # FOR/ELSE statement
3030 main.log.error( "Leader election, could not find current leader" )
3031 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07003032 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07003033 utilities.assert_equals(
3034 expect=main.TRUE,
3035 actual=withdrawResult,
3036 onpass="Node was withdrawn from election",
3037 onfail="Node was not withdrawn from election" )
3038
acsmars71adceb2015-08-31 15:09:26 -07003039 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07003040 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07003041 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07003042 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07003043 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07003044 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07003045 if newLeaders[ 0 ][ 0 ] == 'none':
3046 main.log.error( "No leader was elected on at least 1 node" )
3047 if not expectNoLeader:
3048 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07003049 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07003050
3051 # Check that the new leader is not the older leader, which was withdrawn
3052 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07003053 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08003054 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07003055 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003056 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003057 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003058 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003059 onpass="Leadership election passed",
3060 onfail="Something went wrong with Leadership election" )
3061
Jon Halla440e872016-03-31 15:15:50 -07003062 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08003063 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07003064 correctCandidateResult = main.TRUE
3065 if expectNoLeader:
3066 if newLeader == 'none':
3067 main.log.info( "No leader expected. None found. Pass" )
3068 correctCandidateResult = main.TRUE
3069 else:
3070 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3071 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003072 elif len( oldLeaders[0] ) >= 3:
3073 if newLeader == oldLeaders[ 0 ][ 2 ]:
3074 # correct leader was elected
3075 correctCandidateResult = main.TRUE
3076 else:
3077 correctCandidateResult = main.FALSE
3078 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3079 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003080 else:
3081 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003082 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003083 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07003084 utilities.assert_equals(
3085 expect=main.TRUE,
3086 actual=correctCandidateResult,
3087 onpass="Correct Candidate Elected",
3088 onfail="Incorrect Candidate Elected" )
3089
Jon Hall5cf14d52015-07-16 12:15:19 -07003090 main.step( "Run for election on old leader( just so everyone " +
3091 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07003092 if oldLeaderCLI is not None:
3093 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003094 else:
acsmars71adceb2015-08-31 15:09:26 -07003095 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003096 runResult = main.FALSE
3097 utilities.assert_equals(
3098 expect=main.TRUE,
3099 actual=runResult,
3100 onpass="App re-ran for election",
3101 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003102
acsmars71adceb2015-08-31 15:09:26 -07003103 main.step(
3104 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003105 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003106 # Get new leaders and candidates
3107 reRunLeaders = []
3108 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003109 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003110
3111 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07003112 if not reRunLeaders[0]:
3113 positionResult = main.FALSE
3114 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07003115 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3116 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07003117 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003118 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003119 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003120 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003121 onpass="Old leader successfully re-ran for election",
3122 onfail="Something went wrong with Leadership election after " +
3123 "the old leader re-ran for election" )
3124
3125 def CASE16( self, main ):
3126 """
3127 Install Distributed Primitives app
3128 """
3129 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003130 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003131 assert main, "main not defined"
3132 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003133 assert main.CLIs, "main.CLIs not defined"
3134 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003135
3136 # Variables for the distributed primitives tests
3137 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003138 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003139 global onosSet
3140 global onosSetName
3141 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003142 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003143 onosSet = set([])
3144 onosSetName = "TestON-set"
3145
3146 description = "Install Primitives app"
3147 main.case( description )
3148 main.step( "Install Primitives app" )
3149 appName = "org.onosproject.distributedprimitives"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003150 node = main.activeNodes[0]
3151 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003152 utilities.assert_equals( expect=main.TRUE,
3153 actual=appResults,
3154 onpass="Primitives app activated",
3155 onfail="Primitives app not activated" )
3156 time.sleep( 5 ) # To allow all nodes to activate
3157
3158 def CASE17( self, main ):
3159 """
3160 Check for basic functionality with distributed primitives
3161 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003162 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003163 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003164 assert main, "main not defined"
3165 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003166 assert main.CLIs, "main.CLIs not defined"
3167 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003168 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003169 assert onosSetName, "onosSetName not defined"
3170 # NOTE: assert fails if value is 0/None/Empty/False
3171 try:
3172 pCounterValue
3173 except NameError:
3174 main.log.error( "pCounterValue not defined, setting to 0" )
3175 pCounterValue = 0
3176 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003177 onosSet
3178 except NameError:
3179 main.log.error( "onosSet not defined, setting to empty Set" )
3180 onosSet = set([])
3181 # Variables for the distributed primitives tests. These are local only
3182 addValue = "a"
3183 addAllValue = "a b c d e f"
3184 retainValue = "c d e f"
3185
3186 description = "Check for basic functionality with distributed " +\
3187 "primitives"
3188 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003189 main.caseExplanation = "Test the methods of the distributed " +\
3190 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003191 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003192 # Partitioned counters
3193 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003194 pCounters = []
3195 threads = []
3196 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003197 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003198 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3199 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003200 args=[ pCounterName ] )
3201 pCounterValue += 1
3202 addedPValues.append( pCounterValue )
3203 threads.append( t )
3204 t.start()
3205
3206 for t in threads:
3207 t.join()
3208 pCounters.append( t.result )
3209 # Check that counter incremented numController times
3210 pCounterResults = True
3211 for i in addedPValues:
3212 tmpResult = i in pCounters
3213 pCounterResults = pCounterResults and tmpResult
3214 if not tmpResult:
3215 main.log.error( str( i ) + " is not in partitioned "
3216 "counter incremented results" )
3217 utilities.assert_equals( expect=True,
3218 actual=pCounterResults,
3219 onpass="Default counter incremented",
3220 onfail="Error incrementing default" +
3221 " counter" )
3222
Jon Halle1a3b752015-07-22 13:02:46 -07003223 main.step( "Get then Increment a default counter on each node" )
3224 pCounters = []
3225 threads = []
3226 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003227 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003228 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3229 name="counterGetAndAdd-" + str( i ),
3230 args=[ pCounterName ] )
3231 addedPValues.append( pCounterValue )
3232 pCounterValue += 1
3233 threads.append( t )
3234 t.start()
3235
3236 for t in threads:
3237 t.join()
3238 pCounters.append( t.result )
3239 # Check that counter incremented numController times
3240 pCounterResults = True
3241 for i in addedPValues:
3242 tmpResult = i in pCounters
3243 pCounterResults = pCounterResults and tmpResult
3244 if not tmpResult:
3245 main.log.error( str( i ) + " is not in partitioned "
3246 "counter incremented results" )
3247 utilities.assert_equals( expect=True,
3248 actual=pCounterResults,
3249 onpass="Default counter incremented",
3250 onfail="Error incrementing default" +
3251 " counter" )
3252
3253 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003254 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003255 utilities.assert_equals( expect=main.TRUE,
3256 actual=incrementCheck,
3257 onpass="Added counters are correct",
3258 onfail="Added counters are incorrect" )
3259
3260 main.step( "Add -8 to then get a default counter on each node" )
3261 pCounters = []
3262 threads = []
3263 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003264 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003265 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3266 name="counterIncrement-" + str( i ),
3267 args=[ pCounterName ],
3268 kwargs={ "delta": -8 } )
3269 pCounterValue += -8
3270 addedPValues.append( pCounterValue )
3271 threads.append( t )
3272 t.start()
3273
3274 for t in threads:
3275 t.join()
3276 pCounters.append( t.result )
3277 # Check that counter incremented numController times
3278 pCounterResults = True
3279 for i in addedPValues:
3280 tmpResult = i in pCounters
3281 pCounterResults = pCounterResults and tmpResult
3282 if not tmpResult:
3283 main.log.error( str( i ) + " is not in partitioned "
3284 "counter incremented results" )
3285 utilities.assert_equals( expect=True,
3286 actual=pCounterResults,
3287 onpass="Default counter incremented",
3288 onfail="Error incrementing default" +
3289 " counter" )
3290
3291 main.step( "Add 5 to then get a default counter on each node" )
3292 pCounters = []
3293 threads = []
3294 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003295 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003296 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3297 name="counterIncrement-" + str( i ),
3298 args=[ pCounterName ],
3299 kwargs={ "delta": 5 } )
3300 pCounterValue += 5
3301 addedPValues.append( pCounterValue )
3302 threads.append( t )
3303 t.start()
3304
3305 for t in threads:
3306 t.join()
3307 pCounters.append( t.result )
3308 # Check that counter incremented numController times
3309 pCounterResults = True
3310 for i in addedPValues:
3311 tmpResult = i in pCounters
3312 pCounterResults = pCounterResults and tmpResult
3313 if not tmpResult:
3314 main.log.error( str( i ) + " is not in partitioned "
3315 "counter incremented results" )
3316 utilities.assert_equals( expect=True,
3317 actual=pCounterResults,
3318 onpass="Default counter incremented",
3319 onfail="Error incrementing default" +
3320 " counter" )
3321
3322 main.step( "Get then add 5 to a default counter on each node" )
3323 pCounters = []
3324 threads = []
3325 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003326 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003327 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3328 name="counterIncrement-" + str( i ),
3329 args=[ pCounterName ],
3330 kwargs={ "delta": 5 } )
3331 addedPValues.append( pCounterValue )
3332 pCounterValue += 5
3333 threads.append( t )
3334 t.start()
3335
3336 for t in threads:
3337 t.join()
3338 pCounters.append( t.result )
3339 # Check that counter incremented numController times
3340 pCounterResults = True
3341 for i in addedPValues:
3342 tmpResult = i in pCounters
3343 pCounterResults = pCounterResults and tmpResult
3344 if not tmpResult:
3345 main.log.error( str( i ) + " is not in partitioned "
3346 "counter incremented results" )
3347 utilities.assert_equals( expect=True,
3348 actual=pCounterResults,
3349 onpass="Default counter incremented",
3350 onfail="Error incrementing default" +
3351 " counter" )
3352
3353 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003354 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003355 utilities.assert_equals( expect=main.TRUE,
3356 actual=incrementCheck,
3357 onpass="Added counters are correct",
3358 onfail="Added counters are incorrect" )
3359
Jon Hall5cf14d52015-07-16 12:15:19 -07003360 # DISTRIBUTED SETS
3361 main.step( "Distributed Set get" )
3362 size = len( onosSet )
3363 getResponses = []
3364 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003365 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003366 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003367 name="setTestGet-" + str( i ),
3368 args=[ onosSetName ] )
3369 threads.append( t )
3370 t.start()
3371 for t in threads:
3372 t.join()
3373 getResponses.append( t.result )
3374
3375 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003376 for i in range( len( main.activeNodes ) ):
3377 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003378 if isinstance( getResponses[ i ], list):
3379 current = set( getResponses[ i ] )
3380 if len( current ) == len( getResponses[ i ] ):
3381 # no repeats
3382 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003383 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003384 " has incorrect view" +
3385 " of set " + onosSetName + ":\n" +
3386 str( getResponses[ i ] ) )
3387 main.log.debug( "Expected: " + str( onosSet ) )
3388 main.log.debug( "Actual: " + str( current ) )
3389 getResults = main.FALSE
3390 else:
3391 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003392 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003393 " has repeat elements in" +
3394 " set " + onosSetName + ":\n" +
3395 str( getResponses[ i ] ) )
3396 getResults = main.FALSE
3397 elif getResponses[ i ] == main.ERROR:
3398 getResults = main.FALSE
3399 utilities.assert_equals( expect=main.TRUE,
3400 actual=getResults,
3401 onpass="Set elements are correct",
3402 onfail="Set elements are incorrect" )
3403
3404 main.step( "Distributed Set size" )
3405 sizeResponses = []
3406 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003407 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003408 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003409 name="setTestSize-" + str( i ),
3410 args=[ onosSetName ] )
3411 threads.append( t )
3412 t.start()
3413 for t in threads:
3414 t.join()
3415 sizeResponses.append( t.result )
3416
3417 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003418 for i in range( len( main.activeNodes ) ):
3419 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003420 if size != sizeResponses[ i ]:
3421 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003422 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003423 " expected a size of " + str( size ) +
3424 " for set " + onosSetName +
3425 " but got " + str( sizeResponses[ i ] ) )
3426 utilities.assert_equals( expect=main.TRUE,
3427 actual=sizeResults,
3428 onpass="Set sizes are correct",
3429 onfail="Set sizes are incorrect" )
3430
3431 main.step( "Distributed Set add()" )
3432 onosSet.add( addValue )
3433 addResponses = []
3434 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003435 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003436 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003437 name="setTestAdd-" + str( i ),
3438 args=[ onosSetName, addValue ] )
3439 threads.append( t )
3440 t.start()
3441 for t in threads:
3442 t.join()
3443 addResponses.append( t.result )
3444
3445 # main.TRUE = successfully changed the set
3446 # main.FALSE = action resulted in no change in set
3447 # main.ERROR - Some error in executing the function
3448 addResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003449 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003450 if addResponses[ i ] == main.TRUE:
3451 # All is well
3452 pass
3453 elif addResponses[ i ] == main.FALSE:
3454 # Already in set, probably fine
3455 pass
3456 elif addResponses[ i ] == main.ERROR:
3457 # Error in execution
3458 addResults = main.FALSE
3459 else:
3460 # unexpected result
3461 addResults = main.FALSE
3462 if addResults != main.TRUE:
3463 main.log.error( "Error executing set add" )
3464
3465 # Check if set is still correct
3466 size = len( onosSet )
3467 getResponses = []
3468 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003469 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003470 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003471 name="setTestGet-" + str( i ),
3472 args=[ onosSetName ] )
3473 threads.append( t )
3474 t.start()
3475 for t in threads:
3476 t.join()
3477 getResponses.append( t.result )
3478 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003479 for i in range( len( main.activeNodes ) ):
3480 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003481 if isinstance( getResponses[ i ], list):
3482 current = set( getResponses[ i ] )
3483 if len( current ) == len( getResponses[ i ] ):
3484 # no repeats
3485 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003486 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003487 " of set " + onosSetName + ":\n" +
3488 str( getResponses[ i ] ) )
3489 main.log.debug( "Expected: " + str( onosSet ) )
3490 main.log.debug( "Actual: " + str( current ) )
3491 getResults = main.FALSE
3492 else:
3493 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003494 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003495 " set " + onosSetName + ":\n" +
3496 str( getResponses[ i ] ) )
3497 getResults = main.FALSE
3498 elif getResponses[ i ] == main.ERROR:
3499 getResults = main.FALSE
3500 sizeResponses = []
3501 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003502 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003503 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003504 name="setTestSize-" + str( i ),
3505 args=[ onosSetName ] )
3506 threads.append( t )
3507 t.start()
3508 for t in threads:
3509 t.join()
3510 sizeResponses.append( t.result )
3511 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003512 for i in range( len( main.activeNodes ) ):
3513 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003514 if size != sizeResponses[ i ]:
3515 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003516 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003517 " expected a size of " + str( size ) +
3518 " for set " + onosSetName +
3519 " but got " + str( sizeResponses[ i ] ) )
3520 addResults = addResults and getResults and sizeResults
3521 utilities.assert_equals( expect=main.TRUE,
3522 actual=addResults,
3523 onpass="Set add correct",
3524 onfail="Set add was incorrect" )
3525
3526 main.step( "Distributed Set addAll()" )
3527 onosSet.update( addAllValue.split() )
3528 addResponses = []
3529 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003530 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003531 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003532 name="setTestAddAll-" + str( i ),
3533 args=[ onosSetName, addAllValue ] )
3534 threads.append( t )
3535 t.start()
3536 for t in threads:
3537 t.join()
3538 addResponses.append( t.result )
3539
3540 # main.TRUE = successfully changed the set
3541 # main.FALSE = action resulted in no change in set
3542 # main.ERROR - Some error in executing the function
3543 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003544 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003545 if addResponses[ i ] == main.TRUE:
3546 # All is well
3547 pass
3548 elif addResponses[ i ] == main.FALSE:
3549 # Already in set, probably fine
3550 pass
3551 elif addResponses[ i ] == main.ERROR:
3552 # Error in execution
3553 addAllResults = main.FALSE
3554 else:
3555 # unexpected result
3556 addAllResults = main.FALSE
3557 if addAllResults != main.TRUE:
3558 main.log.error( "Error executing set addAll" )
3559
3560 # Check if set is still correct
3561 size = len( onosSet )
3562 getResponses = []
3563 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003564 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003565 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003566 name="setTestGet-" + str( i ),
3567 args=[ onosSetName ] )
3568 threads.append( t )
3569 t.start()
3570 for t in threads:
3571 t.join()
3572 getResponses.append( t.result )
3573 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003574 for i in range( len( main.activeNodes ) ):
3575 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003576 if isinstance( getResponses[ i ], list):
3577 current = set( getResponses[ i ] )
3578 if len( current ) == len( getResponses[ i ] ):
3579 # no repeats
3580 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003581 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003582 " has incorrect view" +
3583 " of set " + onosSetName + ":\n" +
3584 str( getResponses[ i ] ) )
3585 main.log.debug( "Expected: " + str( onosSet ) )
3586 main.log.debug( "Actual: " + str( current ) )
3587 getResults = main.FALSE
3588 else:
3589 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003590 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003591 " has repeat elements in" +
3592 " set " + onosSetName + ":\n" +
3593 str( getResponses[ i ] ) )
3594 getResults = main.FALSE
3595 elif getResponses[ i ] == main.ERROR:
3596 getResults = main.FALSE
3597 sizeResponses = []
3598 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003599 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003600 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003601 name="setTestSize-" + str( i ),
3602 args=[ onosSetName ] )
3603 threads.append( t )
3604 t.start()
3605 for t in threads:
3606 t.join()
3607 sizeResponses.append( t.result )
3608 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003609 for i in range( len( main.activeNodes ) ):
3610 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003611 if size != sizeResponses[ i ]:
3612 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003613 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003614 " expected a size of " + str( size ) +
3615 " for set " + onosSetName +
3616 " but got " + str( sizeResponses[ i ] ) )
3617 addAllResults = addAllResults and getResults and sizeResults
3618 utilities.assert_equals( expect=main.TRUE,
3619 actual=addAllResults,
3620 onpass="Set addAll correct",
3621 onfail="Set addAll was incorrect" )
3622
3623 main.step( "Distributed Set contains()" )
3624 containsResponses = []
3625 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003626 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003627 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003628 name="setContains-" + str( i ),
3629 args=[ onosSetName ],
3630 kwargs={ "values": addValue } )
3631 threads.append( t )
3632 t.start()
3633 for t in threads:
3634 t.join()
3635 # NOTE: This is the tuple
3636 containsResponses.append( t.result )
3637
3638 containsResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003639 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003640 if containsResponses[ i ] == main.ERROR:
3641 containsResults = main.FALSE
3642 else:
3643 containsResults = containsResults and\
3644 containsResponses[ i ][ 1 ]
3645 utilities.assert_equals( expect=main.TRUE,
3646 actual=containsResults,
3647 onpass="Set contains is functional",
3648 onfail="Set contains failed" )
3649
3650 main.step( "Distributed Set containsAll()" )
3651 containsAllResponses = []
3652 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003653 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003654 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003655 name="setContainsAll-" + str( i ),
3656 args=[ onosSetName ],
3657 kwargs={ "values": addAllValue } )
3658 threads.append( t )
3659 t.start()
3660 for t in threads:
3661 t.join()
3662 # NOTE: This is the tuple
3663 containsAllResponses.append( t.result )
3664
3665 containsAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003666 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003667 if containsResponses[ i ] == main.ERROR:
3668 containsResults = main.FALSE
3669 else:
3670 containsResults = containsResults and\
3671 containsResponses[ i ][ 1 ]
3672 utilities.assert_equals( expect=main.TRUE,
3673 actual=containsAllResults,
3674 onpass="Set containsAll is functional",
3675 onfail="Set containsAll failed" )
3676
3677 main.step( "Distributed Set remove()" )
3678 onosSet.remove( addValue )
3679 removeResponses = []
3680 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003681 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003682 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003683 name="setTestRemove-" + str( i ),
3684 args=[ onosSetName, addValue ] )
3685 threads.append( t )
3686 t.start()
3687 for t in threads:
3688 t.join()
3689 removeResponses.append( t.result )
3690
3691 # main.TRUE = successfully changed the set
3692 # main.FALSE = action resulted in no change in set
3693 # main.ERROR - Some error in executing the function
3694 removeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003695 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003696 if removeResponses[ i ] == main.TRUE:
3697 # All is well
3698 pass
3699 elif removeResponses[ i ] == main.FALSE:
3700 # not in set, probably fine
3701 pass
3702 elif removeResponses[ i ] == main.ERROR:
3703 # Error in execution
3704 removeResults = main.FALSE
3705 else:
3706 # unexpected result
3707 removeResults = main.FALSE
3708 if removeResults != main.TRUE:
3709 main.log.error( "Error executing set remove" )
3710
3711 # Check if set is still correct
3712 size = len( onosSet )
3713 getResponses = []
3714 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003715 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003716 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003717 name="setTestGet-" + str( i ),
3718 args=[ onosSetName ] )
3719 threads.append( t )
3720 t.start()
3721 for t in threads:
3722 t.join()
3723 getResponses.append( t.result )
3724 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003725 for i in range( len( main.activeNodes ) ):
3726 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003727 if isinstance( getResponses[ i ], list):
3728 current = set( getResponses[ i ] )
3729 if len( current ) == len( getResponses[ i ] ):
3730 # no repeats
3731 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003732 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003733 " has incorrect view" +
3734 " of set " + onosSetName + ":\n" +
3735 str( getResponses[ i ] ) )
3736 main.log.debug( "Expected: " + str( onosSet ) )
3737 main.log.debug( "Actual: " + str( current ) )
3738 getResults = main.FALSE
3739 else:
3740 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003741 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003742 " has repeat elements in" +
3743 " set " + onosSetName + ":\n" +
3744 str( getResponses[ i ] ) )
3745 getResults = main.FALSE
3746 elif getResponses[ i ] == main.ERROR:
3747 getResults = main.FALSE
3748 sizeResponses = []
3749 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003750 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003751 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003752 name="setTestSize-" + str( i ),
3753 args=[ onosSetName ] )
3754 threads.append( t )
3755 t.start()
3756 for t in threads:
3757 t.join()
3758 sizeResponses.append( t.result )
3759 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003760 for i in range( len( main.activeNodes ) ):
3761 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003762 if size != sizeResponses[ i ]:
3763 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003764 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003765 " expected a size of " + str( size ) +
3766 " for set " + onosSetName +
3767 " but got " + str( sizeResponses[ i ] ) )
3768 removeResults = removeResults and getResults and sizeResults
3769 utilities.assert_equals( expect=main.TRUE,
3770 actual=removeResults,
3771 onpass="Set remove correct",
3772 onfail="Set remove was incorrect" )
3773
3774 main.step( "Distributed Set removeAll()" )
3775 onosSet.difference_update( addAllValue.split() )
3776 removeAllResponses = []
3777 threads = []
3778 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003779 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003780 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003781 name="setTestRemoveAll-" + str( i ),
3782 args=[ onosSetName, addAllValue ] )
3783 threads.append( t )
3784 t.start()
3785 for t in threads:
3786 t.join()
3787 removeAllResponses.append( t.result )
3788 except Exception, e:
3789 main.log.exception(e)
3790
3791 # main.TRUE = successfully changed the set
3792 # main.FALSE = action resulted in no change in set
3793 # main.ERROR - Some error in executing the function
3794 removeAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003795 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003796 if removeAllResponses[ i ] == main.TRUE:
3797 # All is well
3798 pass
3799 elif removeAllResponses[ i ] == main.FALSE:
3800 # not in set, probably fine
3801 pass
3802 elif removeAllResponses[ i ] == main.ERROR:
3803 # Error in execution
3804 removeAllResults = main.FALSE
3805 else:
3806 # unexpected result
3807 removeAllResults = main.FALSE
3808 if removeAllResults != main.TRUE:
3809 main.log.error( "Error executing set removeAll" )
3810
3811 # Check if set is still correct
3812 size = len( onosSet )
3813 getResponses = []
3814 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003815 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003816 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003817 name="setTestGet-" + str( i ),
3818 args=[ onosSetName ] )
3819 threads.append( t )
3820 t.start()
3821 for t in threads:
3822 t.join()
3823 getResponses.append( t.result )
3824 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003825 for i in range( len( main.activeNodes ) ):
3826 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003827 if isinstance( getResponses[ i ], list):
3828 current = set( getResponses[ i ] )
3829 if len( current ) == len( getResponses[ i ] ):
3830 # no repeats
3831 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003832 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003833 " has incorrect view" +
3834 " of set " + onosSetName + ":\n" +
3835 str( getResponses[ i ] ) )
3836 main.log.debug( "Expected: " + str( onosSet ) )
3837 main.log.debug( "Actual: " + str( current ) )
3838 getResults = main.FALSE
3839 else:
3840 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003841 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003842 " has repeat elements in" +
3843 " set " + onosSetName + ":\n" +
3844 str( getResponses[ i ] ) )
3845 getResults = main.FALSE
3846 elif getResponses[ i ] == main.ERROR:
3847 getResults = main.FALSE
3848 sizeResponses = []
3849 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003850 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003851 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003852 name="setTestSize-" + str( i ),
3853 args=[ onosSetName ] )
3854 threads.append( t )
3855 t.start()
3856 for t in threads:
3857 t.join()
3858 sizeResponses.append( t.result )
3859 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003860 for i in range( len( main.activeNodes ) ):
3861 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003862 if size != sizeResponses[ i ]:
3863 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003864 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003865 " expected a size of " + str( size ) +
3866 " for set " + onosSetName +
3867 " but got " + str( sizeResponses[ i ] ) )
3868 removeAllResults = removeAllResults and getResults and sizeResults
3869 utilities.assert_equals( expect=main.TRUE,
3870 actual=removeAllResults,
3871 onpass="Set removeAll correct",
3872 onfail="Set removeAll was incorrect" )
3873
3874 main.step( "Distributed Set addAll()" )
3875 onosSet.update( addAllValue.split() )
3876 addResponses = []
3877 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003878 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003879 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003880 name="setTestAddAll-" + str( i ),
3881 args=[ onosSetName, addAllValue ] )
3882 threads.append( t )
3883 t.start()
3884 for t in threads:
3885 t.join()
3886 addResponses.append( t.result )
3887
3888 # main.TRUE = successfully changed the set
3889 # main.FALSE = action resulted in no change in set
3890 # main.ERROR - Some error in executing the function
3891 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003892 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003893 if addResponses[ i ] == main.TRUE:
3894 # All is well
3895 pass
3896 elif addResponses[ i ] == main.FALSE:
3897 # Already in set, probably fine
3898 pass
3899 elif addResponses[ i ] == main.ERROR:
3900 # Error in execution
3901 addAllResults = main.FALSE
3902 else:
3903 # unexpected result
3904 addAllResults = main.FALSE
3905 if addAllResults != main.TRUE:
3906 main.log.error( "Error executing set addAll" )
3907
3908 # Check if set is still correct
3909 size = len( onosSet )
3910 getResponses = []
3911 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003912 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003913 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003914 name="setTestGet-" + str( i ),
3915 args=[ onosSetName ] )
3916 threads.append( t )
3917 t.start()
3918 for t in threads:
3919 t.join()
3920 getResponses.append( t.result )
3921 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003922 for i in range( len( main.activeNodes ) ):
3923 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003924 if isinstance( getResponses[ i ], list):
3925 current = set( getResponses[ i ] )
3926 if len( current ) == len( getResponses[ i ] ):
3927 # no repeats
3928 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003929 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003930 " has incorrect view" +
3931 " of set " + onosSetName + ":\n" +
3932 str( getResponses[ i ] ) )
3933 main.log.debug( "Expected: " + str( onosSet ) )
3934 main.log.debug( "Actual: " + str( current ) )
3935 getResults = main.FALSE
3936 else:
3937 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003938 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003939 " has repeat elements in" +
3940 " set " + onosSetName + ":\n" +
3941 str( getResponses[ i ] ) )
3942 getResults = main.FALSE
3943 elif getResponses[ i ] == main.ERROR:
3944 getResults = main.FALSE
3945 sizeResponses = []
3946 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003947 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003948 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003949 name="setTestSize-" + str( i ),
3950 args=[ onosSetName ] )
3951 threads.append( t )
3952 t.start()
3953 for t in threads:
3954 t.join()
3955 sizeResponses.append( t.result )
3956 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003957 for i in range( len( main.activeNodes ) ):
3958 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003959 if size != sizeResponses[ i ]:
3960 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003961 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003962 " expected a size of " + str( size ) +
3963 " for set " + onosSetName +
3964 " but got " + str( sizeResponses[ i ] ) )
3965 addAllResults = addAllResults and getResults and sizeResults
3966 utilities.assert_equals( expect=main.TRUE,
3967 actual=addAllResults,
3968 onpass="Set addAll correct",
3969 onfail="Set addAll was incorrect" )
3970
3971 main.step( "Distributed Set clear()" )
3972 onosSet.clear()
3973 clearResponses = []
3974 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003975 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003976 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003977 name="setTestClear-" + str( i ),
3978 args=[ onosSetName, " "], # Values doesn't matter
3979 kwargs={ "clear": True } )
3980 threads.append( t )
3981 t.start()
3982 for t in threads:
3983 t.join()
3984 clearResponses.append( t.result )
3985
3986 # main.TRUE = successfully changed the set
3987 # main.FALSE = action resulted in no change in set
3988 # main.ERROR - Some error in executing the function
3989 clearResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003990 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003991 if clearResponses[ i ] == main.TRUE:
3992 # All is well
3993 pass
3994 elif clearResponses[ i ] == main.FALSE:
3995 # Nothing set, probably fine
3996 pass
3997 elif clearResponses[ i ] == main.ERROR:
3998 # Error in execution
3999 clearResults = main.FALSE
4000 else:
4001 # unexpected result
4002 clearResults = main.FALSE
4003 if clearResults != main.TRUE:
4004 main.log.error( "Error executing set clear" )
4005
4006 # Check if set is still correct
4007 size = len( onosSet )
4008 getResponses = []
4009 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004010 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004011 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004012 name="setTestGet-" + str( i ),
4013 args=[ onosSetName ] )
4014 threads.append( t )
4015 t.start()
4016 for t in threads:
4017 t.join()
4018 getResponses.append( t.result )
4019 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004020 for i in range( len( main.activeNodes ) ):
4021 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004022 if isinstance( getResponses[ i ], list):
4023 current = set( getResponses[ i ] )
4024 if len( current ) == len( getResponses[ i ] ):
4025 # no repeats
4026 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004027 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004028 " has incorrect view" +
4029 " of set " + onosSetName + ":\n" +
4030 str( getResponses[ i ] ) )
4031 main.log.debug( "Expected: " + str( onosSet ) )
4032 main.log.debug( "Actual: " + str( current ) )
4033 getResults = main.FALSE
4034 else:
4035 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004036 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004037 " has repeat elements in" +
4038 " set " + onosSetName + ":\n" +
4039 str( getResponses[ i ] ) )
4040 getResults = main.FALSE
4041 elif getResponses[ i ] == main.ERROR:
4042 getResults = main.FALSE
4043 sizeResponses = []
4044 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004045 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004046 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004047 name="setTestSize-" + str( i ),
4048 args=[ onosSetName ] )
4049 threads.append( t )
4050 t.start()
4051 for t in threads:
4052 t.join()
4053 sizeResponses.append( t.result )
4054 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004055 for i in range( len( main.activeNodes ) ):
4056 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004057 if size != sizeResponses[ i ]:
4058 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004059 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004060 " expected a size of " + str( size ) +
4061 " for set " + onosSetName +
4062 " but got " + str( sizeResponses[ i ] ) )
4063 clearResults = clearResults and getResults and sizeResults
4064 utilities.assert_equals( expect=main.TRUE,
4065 actual=clearResults,
4066 onpass="Set clear correct",
4067 onfail="Set clear was incorrect" )
4068
4069 main.step( "Distributed Set addAll()" )
4070 onosSet.update( addAllValue.split() )
4071 addResponses = []
4072 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004073 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004074 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004075 name="setTestAddAll-" + str( i ),
4076 args=[ onosSetName, addAllValue ] )
4077 threads.append( t )
4078 t.start()
4079 for t in threads:
4080 t.join()
4081 addResponses.append( t.result )
4082
4083 # main.TRUE = successfully changed the set
4084 # main.FALSE = action resulted in no change in set
4085 # main.ERROR - Some error in executing the function
4086 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004087 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004088 if addResponses[ i ] == main.TRUE:
4089 # All is well
4090 pass
4091 elif addResponses[ i ] == main.FALSE:
4092 # Already in set, probably fine
4093 pass
4094 elif addResponses[ i ] == main.ERROR:
4095 # Error in execution
4096 addAllResults = main.FALSE
4097 else:
4098 # unexpected result
4099 addAllResults = main.FALSE
4100 if addAllResults != main.TRUE:
4101 main.log.error( "Error executing set addAll" )
4102
4103 # Check if set is still correct
4104 size = len( onosSet )
4105 getResponses = []
4106 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004107 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004108 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004109 name="setTestGet-" + str( i ),
4110 args=[ onosSetName ] )
4111 threads.append( t )
4112 t.start()
4113 for t in threads:
4114 t.join()
4115 getResponses.append( t.result )
4116 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004117 for i in range( len( main.activeNodes ) ):
4118 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004119 if isinstance( getResponses[ i ], list):
4120 current = set( getResponses[ i ] )
4121 if len( current ) == len( getResponses[ i ] ):
4122 # no repeats
4123 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004124 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004125 " has incorrect view" +
4126 " of set " + onosSetName + ":\n" +
4127 str( getResponses[ i ] ) )
4128 main.log.debug( "Expected: " + str( onosSet ) )
4129 main.log.debug( "Actual: " + str( current ) )
4130 getResults = main.FALSE
4131 else:
4132 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004133 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004134 " has repeat elements in" +
4135 " set " + onosSetName + ":\n" +
4136 str( getResponses[ i ] ) )
4137 getResults = main.FALSE
4138 elif getResponses[ i ] == main.ERROR:
4139 getResults = main.FALSE
4140 sizeResponses = []
4141 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004142 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004143 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004144 name="setTestSize-" + str( i ),
4145 args=[ onosSetName ] )
4146 threads.append( t )
4147 t.start()
4148 for t in threads:
4149 t.join()
4150 sizeResponses.append( t.result )
4151 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004152 for i in range( len( main.activeNodes ) ):
4153 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004154 if size != sizeResponses[ i ]:
4155 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004156 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004157 " expected a size of " + str( size ) +
4158 " for set " + onosSetName +
4159 " but got " + str( sizeResponses[ i ] ) )
4160 addAllResults = addAllResults and getResults and sizeResults
4161 utilities.assert_equals( expect=main.TRUE,
4162 actual=addAllResults,
4163 onpass="Set addAll correct",
4164 onfail="Set addAll was incorrect" )
4165
4166 main.step( "Distributed Set retain()" )
4167 onosSet.intersection_update( retainValue.split() )
4168 retainResponses = []
4169 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004170 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004171 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004172 name="setTestRetain-" + str( i ),
4173 args=[ onosSetName, retainValue ],
4174 kwargs={ "retain": True } )
4175 threads.append( t )
4176 t.start()
4177 for t in threads:
4178 t.join()
4179 retainResponses.append( t.result )
4180
4181 # main.TRUE = successfully changed the set
4182 # main.FALSE = action resulted in no change in set
4183 # main.ERROR - Some error in executing the function
4184 retainResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004185 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004186 if retainResponses[ i ] == main.TRUE:
4187 # All is well
4188 pass
4189 elif retainResponses[ i ] == main.FALSE:
4190 # Already in set, probably fine
4191 pass
4192 elif retainResponses[ i ] == main.ERROR:
4193 # Error in execution
4194 retainResults = main.FALSE
4195 else:
4196 # unexpected result
4197 retainResults = main.FALSE
4198 if retainResults != main.TRUE:
4199 main.log.error( "Error executing set retain" )
4200
4201 # Check if set is still correct
4202 size = len( onosSet )
4203 getResponses = []
4204 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004205 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004206 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004207 name="setTestGet-" + str( i ),
4208 args=[ onosSetName ] )
4209 threads.append( t )
4210 t.start()
4211 for t in threads:
4212 t.join()
4213 getResponses.append( t.result )
4214 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004215 for i in range( len( main.activeNodes ) ):
4216 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004217 if isinstance( getResponses[ i ], list):
4218 current = set( getResponses[ i ] )
4219 if len( current ) == len( getResponses[ i ] ):
4220 # no repeats
4221 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004222 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004223 " has incorrect view" +
4224 " of set " + onosSetName + ":\n" +
4225 str( getResponses[ i ] ) )
4226 main.log.debug( "Expected: " + str( onosSet ) )
4227 main.log.debug( "Actual: " + str( current ) )
4228 getResults = main.FALSE
4229 else:
4230 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004231 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004232 " has repeat elements in" +
4233 " set " + onosSetName + ":\n" +
4234 str( getResponses[ i ] ) )
4235 getResults = main.FALSE
4236 elif getResponses[ i ] == main.ERROR:
4237 getResults = main.FALSE
4238 sizeResponses = []
4239 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004240 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004241 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004242 name="setTestSize-" + str( i ),
4243 args=[ onosSetName ] )
4244 threads.append( t )
4245 t.start()
4246 for t in threads:
4247 t.join()
4248 sizeResponses.append( t.result )
4249 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004250 for i in range( len( main.activeNodes ) ):
4251 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004252 if size != sizeResponses[ i ]:
4253 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004254 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004255 str( size ) + " for set " + onosSetName +
4256 " but got " + str( sizeResponses[ i ] ) )
4257 retainResults = retainResults and getResults and sizeResults
4258 utilities.assert_equals( expect=main.TRUE,
4259 actual=retainResults,
4260 onpass="Set retain correct",
4261 onfail="Set retain was incorrect" )
4262
Jon Hall2a5002c2015-08-21 16:49:11 -07004263 # Transactional maps
4264 main.step( "Partitioned Transactional maps put" )
4265 tMapValue = "Testing"
4266 numKeys = 100
4267 putResult = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004268 node = main.activeNodes[0]
4269 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004270 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004271 for i in putResponses:
4272 if putResponses[ i ][ 'value' ] != tMapValue:
4273 putResult = False
4274 else:
4275 putResult = False
4276 if not putResult:
4277 main.log.debug( "Put response values: " + str( putResponses ) )
4278 utilities.assert_equals( expect=True,
4279 actual=putResult,
4280 onpass="Partitioned Transactional Map put successful",
4281 onfail="Partitioned Transactional Map put values are incorrect" )
4282
4283 main.step( "Partitioned Transactional maps get" )
Jon Hall9bfadd22016-05-11 14:48:07 -07004284 # FIXME: is this sleep needed?
4285 time.sleep( 5 )
4286
Jon Hall2a5002c2015-08-21 16:49:11 -07004287 getCheck = True
4288 for n in range( 1, numKeys + 1 ):
4289 getResponses = []
4290 threads = []
4291 valueCheck = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004292 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004293 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4294 name="TMap-get-" + str( i ),
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004295 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004296 threads.append( t )
4297 t.start()
4298 for t in threads:
4299 t.join()
4300 getResponses.append( t.result )
4301 for node in getResponses:
4302 if node != tMapValue:
4303 valueCheck = False
4304 if not valueCheck:
4305 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4306 main.log.warn( getResponses )
4307 getCheck = getCheck and valueCheck
4308 utilities.assert_equals( expect=True,
4309 actual=getCheck,
4310 onpass="Partitioned Transactional Map get values were correct",
4311 onfail="Partitioned Transactional Map values incorrect" )