blob: 26f255f3f75649f7331b7428803221bc2ca6a3cd [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
Jon Hallb3ed8ed2015-10-28 16:43:55 -070012CASE61: The Failure inducing case.
13CASE62: The Failure recovery case.
Jon Hall5cf14d52015-07-16 12:15:19 -070014CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
Jon Hallb3ed8ed2015-10-28 16:43:55 -070028class HAstopNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -070029
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall3b489db2015-10-05 14:38:37 -070052 import pexpect
Jon Hallf3d16e72015-12-16 17:45:08 -080053 import time
Jon Halla440e872016-03-31 15:15:50 -070054 import json
Jon Hallb3ed8ed2015-10-28 16:43:55 -070055 main.log.info( "ONOS HA test: Stop a minority of ONOS nodes - " +
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "initialization" )
57 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070058 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070059 "installing ONOS, starting Mininet and ONOS" +\
60 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070061
62 # load some variables from the params file
63 PULLCODE = False
64 if main.params[ 'Git' ] == 'True':
65 PULLCODE = True
66 gitBranch = main.params[ 'branch' ]
67 cellName = main.params[ 'ENV' ][ 'cellName' ]
68
Jon Halle1a3b752015-07-22 13:02:46 -070069 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070071 if main.ONOSbench.maxNodes < main.numCtrls:
72 main.numCtrls = int( main.ONOSbench.maxNodes )
73 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070074 global ONOS1Port
75 global ONOS2Port
76 global ONOS3Port
77 global ONOS4Port
78 global ONOS5Port
79 global ONOS6Port
80 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070081 # These are for csv plotting in jenkins
82 global labels
83 global data
84 labels = []
85 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070086
87 # FIXME: just get controller port from params?
88 # TODO: do we really need all these?
89 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
90 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
91 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
92 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
93 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
94 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
95 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
96
Jon Halle1a3b752015-07-22 13:02:46 -070097 try:
Jon Hall53c5e662016-04-13 16:06:56 -070098 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070099 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -0700100 except Exception as e:
101 main.log.exception( e )
102 main.cleanup()
103 main.exit()
104
105 main.CLIs = []
106 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700107 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700108 for i in range( 1, main.numCtrls + 1 ):
109 try:
110 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
111 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
112 ipList.append( main.nodes[ -1 ].ip_address )
113 except AttributeError:
114 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700115
116 main.step( "Create cell file" )
117 cellAppString = main.params[ 'ENV' ][ 'appString' ]
118 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
119 main.Mininet1.ip_address,
120 cellAppString, ipList )
121 main.step( "Applying cell variable to environment" )
122 cellResult = main.ONOSbench.setCell( cellName )
123 verifyResult = main.ONOSbench.verifyCell()
124
125 # FIXME:this is short term fix
126 main.log.info( "Removing raft logs" )
127 main.ONOSbench.onosRemoveRaftLogs()
128
129 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700130 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700131 main.ONOSbench.onosUninstall( node.ip_address )
132
133 # Make sure ONOS is DEAD
134 main.log.info( "Killing any ONOS processes" )
135 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700136 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700137 killed = main.ONOSbench.onosKill( node.ip_address )
138 killResults = killResults and killed
139
140 cleanInstallResult = main.TRUE
141 gitPullResult = main.TRUE
142
143 main.step( "Starting Mininet" )
144 # scp topo file to mininet
145 # TODO: move to params?
146 topoName = "obelisk.py"
147 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700148 main.ONOSbench.scp( main.Mininet1,
149 filePath + topoName,
150 main.Mininet1.home,
151 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700152 mnResult = main.Mininet1.startNet( )
153 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
154 onpass="Mininet Started",
155 onfail="Error starting Mininet" )
156
157 main.step( "Git checkout and pull " + gitBranch )
158 if PULLCODE:
159 main.ONOSbench.gitCheckout( gitBranch )
160 gitPullResult = main.ONOSbench.gitPull()
161 # values of 1 or 3 are good
162 utilities.assert_lesser( expect=0, actual=gitPullResult,
163 onpass="Git pull successful",
164 onfail="Git pull failed" )
165 main.ONOSbench.getVersion( report=True )
166
167 main.step( "Using mvn clean install" )
168 cleanInstallResult = main.TRUE
169 if PULLCODE and gitPullResult == main.TRUE:
170 cleanInstallResult = main.ONOSbench.cleanInstall()
171 else:
172 main.log.warn( "Did not pull new code so skipping mvn " +
173 "clean install" )
174 utilities.assert_equals( expect=main.TRUE,
175 actual=cleanInstallResult,
176 onpass="MCI successful",
177 onfail="MCI failed" )
178 # GRAPHS
179 # NOTE: important params here:
180 # job = name of Jenkins job
181 # Plot Name = Plot-HA, only can be used if multiple plots
182 # index = The number of the graph under plot name
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700183 job = "HAstopNodes"
Jon Hall5cf14d52015-07-16 12:15:19 -0700184 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700185 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700186 graphs = '<ac:structured-macro ac:name="html">\n'
187 graphs += '<ac:plain-text-body><![CDATA[\n'
188 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800189 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700190 '&width=500&height=300"' +\
191 'noborder="0" width="500" height="300" scrolling="yes" ' +\
192 'seamless="seamless"></iframe>\n'
193 graphs += ']]></ac:plain-text-body>\n'
194 graphs += '</ac:structured-macro>\n'
195 main.log.wiki(graphs)
196
197 main.step( "Creating ONOS package" )
Jon Hall3b489db2015-10-05 14:38:37 -0700198 # copy gen-partions file to ONOS
199 # NOTE: this assumes TestON and ONOS are on the same machine
Jon Hall53c5e662016-04-13 16:06:56 -0700200 srcFile = main.testDir + "/HA/dependencies/onos-gen-partitions"
Jon Hall3b489db2015-10-05 14:38:37 -0700201 dstDir = main.ONOSbench.home + "/tools/test/bin/onos-gen-partitions"
202 cpResult = main.ONOSbench.secureCopy( main.ONOSbench.user_name,
203 main.ONOSbench.ip_address,
204 srcFile,
205 dstDir,
206 pwd=main.ONOSbench.pwd,
207 direction="from" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700208 packageResult = main.ONOSbench.onosPackage()
209 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
210 onpass="ONOS package successful",
211 onfail="ONOS package failed" )
212
213 main.step( "Installing ONOS package" )
214 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700215 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700216 tmpResult = main.ONOSbench.onosInstall( options="-f",
217 node=node.ip_address )
218 onosInstallResult = onosInstallResult and tmpResult
219 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
220 onpass="ONOS install successful",
221 onfail="ONOS install failed" )
Jon Hall3b489db2015-10-05 14:38:37 -0700222 # clean up gen-partitions file
223 try:
224 main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
225 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
226 main.ONOSbench.handle.sendline( "git checkout -- tools/test/bin/onos-gen-partitions" )
227 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
228 main.log.info( " Cleaning custom gen partitions file, response was: \n" +
229 str( main.ONOSbench.handle.before ) )
230 except ( pexpect.TIMEOUT, pexpect.EOF ):
231 main.log.exception( "ONOSbench: pexpect exception found:" +
232 main.ONOSbench.handle.before )
233 main.cleanup()
234 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700235
236 main.step( "Checking if ONOS is up yet" )
237 for i in range( 2 ):
238 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700239 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700240 started = main.ONOSbench.isup( node.ip_address )
241 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800242 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700243 onosIsupResult = onosIsupResult and started
244 if onosIsupResult == main.TRUE:
245 break
246 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
247 onpass="ONOS startup successful",
248 onfail="ONOS startup failed" )
249
250 main.log.step( "Starting ONOS CLI sessions" )
251 cliResults = main.TRUE
252 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700253 for i in range( main.numCtrls ):
254 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700255 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700256 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700257 threads.append( t )
258 t.start()
259
260 for t in threads:
261 t.join()
262 cliResults = cliResults and t.result
263 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
264 onpass="ONOS cli startup successful",
265 onfail="ONOS cli startup failed" )
266
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700267 # Create a list of active nodes for use when some nodes are stopped
268 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
269
Jon Hall5cf14d52015-07-16 12:15:19 -0700270 if main.params[ 'tcpdump' ].lower() == "true":
271 main.step( "Start Packet Capture MN" )
272 main.Mininet2.startTcpdump(
273 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
274 + "-MN.pcap",
275 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
276 port=main.params[ 'MNtcpdump' ][ 'port' ] )
277
Jon Halla440e872016-03-31 15:15:50 -0700278 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700279 nodeResults = utilities.retry( main.HA.nodesCheck,
280 False,
281 args=[main.activeNodes],
282 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700283
Jon Hall41d39f12016-04-11 22:54:35 -0700284 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700285 onpass="Nodes check successful",
286 onfail="Nodes check NOT successful" )
287
288 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700289 for i in main.activeNodes:
290 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700291 main.log.debug( "{} components not ACTIVE: \n{}".format(
292 cli.name,
293 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700294 main.log.error( "Failed to start ONOS, stopping test" )
295 main.cleanup()
296 main.exit()
297
Jon Hall172b7ba2016-04-07 18:12:20 -0700298 main.step( "Activate apps defined in the params file" )
299 # get data from the params
300 apps = main.params.get( 'apps' )
301 if apps:
302 apps = apps.split(',')
303 main.log.warn( apps )
304 activateResult = True
305 for app in apps:
306 main.CLIs[ 0 ].app( app, "Activate" )
307 # TODO: check this worked
308 time.sleep( 10 ) # wait for apps to activate
309 for app in apps:
310 state = main.CLIs[ 0 ].appStatus( app )
311 if state == "ACTIVE":
312 activateResult = activeResult and True
313 else:
314 main.log.error( "{} is in {} state".format( app, state ) )
315 activeResult = False
316 utilities.assert_equals( expect=True,
317 actual=activateResult,
318 onpass="Successfully activated apps",
319 onfail="Failed to activate apps" )
320 else:
321 main.log.warn( "No apps were specified to be loaded after startup" )
322
323 main.step( "Set ONOS configurations" )
324 config = main.params.get( 'ONOS_Configuration' )
325 if config:
326 main.log.debug( config )
327 checkResult = main.TRUE
328 for component in config:
329 for setting in config[component]:
330 value = config[component][setting]
331 check = main.CLIs[ 0 ].setCfg( component, setting, value )
332 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
333 checkResult = check and checkResult
334 utilities.assert_equals( expect=main.TRUE,
335 actual=checkResult,
336 onpass="Successfully set config",
337 onfail="Failed to set config" )
338 else:
339 main.log.warn( "No configurations were specified to be changed after startup" )
340
Jon Hall9d2dcad2016-04-08 10:15:20 -0700341 main.step( "App Ids check" )
342 appCheck = main.TRUE
343 threads = []
344 for i in main.activeNodes:
345 t = main.Thread( target=main.CLIs[i].appToIDCheck,
346 name="appToIDCheck-" + str( i ),
347 args=[] )
348 threads.append( t )
349 t.start()
350
351 for t in threads:
352 t.join()
353 appCheck = appCheck and t.result
354 if appCheck != main.TRUE:
355 node = main.activeNodes[0]
356 main.log.warn( main.CLIs[node].apps() )
357 main.log.warn( main.CLIs[node].appIDs() )
358 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
359 onpass="App Ids seem to be correct",
360 onfail="Something is wrong with app Ids" )
361
Jon Hall5cf14d52015-07-16 12:15:19 -0700362 def CASE2( self, main ):
363 """
364 Assign devices to controllers
365 """
366 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700367 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 assert main, "main not defined"
369 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700370 assert main.CLIs, "main.CLIs not defined"
371 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 assert ONOS1Port, "ONOS1Port not defined"
373 assert ONOS2Port, "ONOS2Port not defined"
374 assert ONOS3Port, "ONOS3Port not defined"
375 assert ONOS4Port, "ONOS4Port not defined"
376 assert ONOS5Port, "ONOS5Port not defined"
377 assert ONOS6Port, "ONOS6Port not defined"
378 assert ONOS7Port, "ONOS7Port not defined"
379
380 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700381 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700382 "and check that an ONOS node becomes the " +\
383 "master of the device."
384 main.step( "Assign switches to controllers" )
385
386 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700387 for i in range( main.numCtrls ):
388 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700389 swList = []
390 for i in range( 1, 29 ):
391 swList.append( "s" + str( i ) )
392 main.Mininet1.assignSwController( sw=swList, ip=ipList )
393
394 mastershipCheck = main.TRUE
395 for i in range( 1, 29 ):
396 response = main.Mininet1.getSwController( "s" + str( i ) )
397 try:
398 main.log.info( str( response ) )
399 except Exception:
400 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700401 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700402 if re.search( "tcp:" + node.ip_address, response ):
403 mastershipCheck = mastershipCheck and main.TRUE
404 else:
405 main.log.error( "Error, node " + node.ip_address + " is " +
406 "not in the list of controllers s" +
407 str( i ) + " is connecting to." )
408 mastershipCheck = main.FALSE
409 utilities.assert_equals(
410 expect=main.TRUE,
411 actual=mastershipCheck,
412 onpass="Switch mastership assigned correctly",
413 onfail="Switches not assigned correctly to controllers" )
414
415 def CASE21( self, main ):
416 """
417 Assign mastership to controllers
418 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700419 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700420 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700421 assert main, "main not defined"
422 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700423 assert main.CLIs, "main.CLIs not defined"
424 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700425 assert ONOS1Port, "ONOS1Port not defined"
426 assert ONOS2Port, "ONOS2Port not defined"
427 assert ONOS3Port, "ONOS3Port not defined"
428 assert ONOS4Port, "ONOS4Port not defined"
429 assert ONOS5Port, "ONOS5Port not defined"
430 assert ONOS6Port, "ONOS6Port not defined"
431 assert ONOS7Port, "ONOS7Port not defined"
432
433 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700434 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700435 "device. Then manually assign" +\
436 " mastership to specific ONOS nodes using" +\
437 " 'device-role'"
438 main.step( "Assign mastership of switches to specific controllers" )
439 # Manually assign mastership to the controller we want
440 roleCall = main.TRUE
441
442 ipList = [ ]
443 deviceList = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700444 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700445 try:
446 # Assign mastership to specific controllers. This assignment was
447 # determined for a 7 node cluser, but will work with any sized
448 # cluster
449 for i in range( 1, 29 ): # switches 1 through 28
450 # set up correct variables:
451 if i == 1:
452 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700453 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700454 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700455 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700456 c = 1 % main.numCtrls
457 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700458 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700460 c = 1 % main.numCtrls
461 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700462 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700464 c = 3 % main.numCtrls
465 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700466 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700467 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700468 c = 2 % main.numCtrls
469 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700470 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700471 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700472 c = 2 % main.numCtrls
473 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700474 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700475 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700476 c = 5 % main.numCtrls
477 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700478 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700479 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700480 c = 4 % main.numCtrls
481 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700482 dpid = '3' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700483 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700484 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700485 c = 6 % main.numCtrls
486 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700487 dpid = '6' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700488 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700489 elif i == 28:
490 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700491 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700492 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700493 else:
494 main.log.error( "You didn't write an else statement for " +
495 "switch s" + str( i ) )
496 roleCall = main.FALSE
497 # Assign switch
498 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
499 # TODO: make this controller dynamic
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700500 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700501 ipList.append( ip )
502 deviceList.append( deviceId )
503 except ( AttributeError, AssertionError ):
504 main.log.exception( "Something is wrong with ONOS device view" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700505 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700506 utilities.assert_equals(
507 expect=main.TRUE,
508 actual=roleCall,
509 onpass="Re-assigned switch mastership to designated controller",
510 onfail="Something wrong with deviceRole calls" )
511
512 main.step( "Check mastership was correctly assigned" )
513 roleCheck = main.TRUE
514 # NOTE: This is due to the fact that device mastership change is not
515 # atomic and is actually a multi step process
516 time.sleep( 5 )
517 for i in range( len( ipList ) ):
518 ip = ipList[i]
519 deviceId = deviceList[i]
520 # Check assignment
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700521 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700522 if ip in master:
523 roleCheck = roleCheck and main.TRUE
524 else:
525 roleCheck = roleCheck and main.FALSE
526 main.log.error( "Error, controller " + ip + " is not" +
527 " master " + "of device " +
528 str( deviceId ) + ". Master is " +
529 repr( master ) + "." )
530 utilities.assert_equals(
531 expect=main.TRUE,
532 actual=roleCheck,
533 onpass="Switches were successfully reassigned to designated " +
534 "controller",
535 onfail="Switches were not successfully reassigned" )
536
537 def CASE3( self, main ):
538 """
539 Assign intents
540 """
541 import time
542 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700543 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700544 assert main, "main not defined"
545 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700546 assert main.CLIs, "main.CLIs not defined"
547 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700548 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700549 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700550 "assign predetermined host-to-host intents." +\
551 " After installation, check that the intent" +\
552 " is distributed to all nodes and the state" +\
553 " is INSTALLED"
554
555 # install onos-app-fwd
556 main.step( "Install reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700557 onosCli = main.CLIs[ main.activeNodes[0] ]
558 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700559 utilities.assert_equals( expect=main.TRUE, actual=installResults,
560 onpass="Install fwd successful",
561 onfail="Install fwd failed" )
562
563 main.step( "Check app ids" )
564 appCheck = main.TRUE
565 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700566 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700567 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700568 name="appToIDCheck-" + str( i ),
569 args=[] )
570 threads.append( t )
571 t.start()
572
573 for t in threads:
574 t.join()
575 appCheck = appCheck and t.result
576 if appCheck != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700577 main.log.warn( onosCli.apps() )
578 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700579 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
580 onpass="App Ids seem to be correct",
581 onfail="Something is wrong with app Ids" )
582
583 main.step( "Discovering Hosts( Via pingall for now )" )
584 # FIXME: Once we have a host discovery mechanism, use that instead
585 # REACTIVE FWD test
586 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700587 passMsg = "Reactive Pingall test passed"
588 time1 = time.time()
589 pingResult = main.Mininet1.pingall()
590 time2 = time.time()
591 if not pingResult:
592 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700593 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700594 passMsg += " on the second try"
595 utilities.assert_equals(
596 expect=main.TRUE,
597 actual=pingResult,
598 onpass= passMsg,
599 onfail="Reactive Pingall failed, " +
600 "one or more ping pairs failed" )
601 main.log.info( "Time for pingall: %2f seconds" %
602 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700603 # timeout for fwd flows
604 time.sleep( 11 )
605 # uninstall onos-app-fwd
606 main.step( "Uninstall reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700607 node = main.activeNodes[0]
608 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700609 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
610 onpass="Uninstall fwd successful",
611 onfail="Uninstall fwd failed" )
612
613 main.step( "Check app ids" )
614 threads = []
615 appCheck2 = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700616 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700617 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700618 name="appToIDCheck-" + str( i ),
619 args=[] )
620 threads.append( t )
621 t.start()
622
623 for t in threads:
624 t.join()
625 appCheck2 = appCheck2 and t.result
626 if appCheck2 != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700627 node = main.activeNodes[0]
628 main.log.warn( main.CLIs[node].apps() )
629 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700630 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
631 onpass="App Ids seem to be correct",
632 onfail="Something is wrong with app Ids" )
633
634 main.step( "Add host intents via cli" )
635 intentIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700636 # TODO: move the host numbers to params
637 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700638 intentAddResult = True
639 hostResult = main.TRUE
640 for i in range( 8, 18 ):
641 main.log.info( "Adding host intent between h" + str( i ) +
642 " and h" + str( i + 10 ) )
643 host1 = "00:00:00:00:00:" + \
644 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
645 host2 = "00:00:00:00:00:" + \
646 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
647 # NOTE: getHost can return None
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700648 host1Dict = onosCli.getHost( host1 )
649 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700650 host1Id = None
651 host2Id = None
652 if host1Dict and host2Dict:
653 host1Id = host1Dict.get( 'id', None )
654 host2Id = host2Dict.get( 'id', None )
655 if host1Id and host2Id:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700656 nodeNum = ( i % len( main.activeNodes ) )
657 node = main.activeNodes[nodeNum]
658 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700659 if tmpId:
660 main.log.info( "Added intent with id: " + tmpId )
661 intentIds.append( tmpId )
662 else:
663 main.log.error( "addHostIntent returned: " +
664 repr( tmpId ) )
665 else:
666 main.log.error( "Error, getHost() failed for h" + str( i ) +
667 " and/or h" + str( i + 10 ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700668 node = main.activeNodes[0]
669 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700670 main.log.warn( "Hosts output: " )
671 try:
672 main.log.warn( json.dumps( json.loads( hosts ),
673 sort_keys=True,
674 indent=4,
675 separators=( ',', ': ' ) ) )
676 except ( ValueError, TypeError ):
677 main.log.warn( repr( hosts ) )
678 hostResult = main.FALSE
679 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
680 onpass="Found a host id for each host",
681 onfail="Error looking up host ids" )
682
683 intentStart = time.time()
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700684 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700685 main.log.info( "Submitted intents: " + str( intentIds ) )
686 main.log.info( "Intents in ONOS: " + str( onosIds ) )
687 for intent in intentIds:
688 if intent in onosIds:
689 pass # intent submitted is in onos
690 else:
691 intentAddResult = False
692 if intentAddResult:
693 intentStop = time.time()
694 else:
695 intentStop = None
696 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700697 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700698 intentStates = []
699 installedCheck = True
700 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
701 count = 0
702 try:
703 for intent in json.loads( intents ):
704 state = intent.get( 'state', None )
705 if "INSTALLED" not in state:
706 installedCheck = False
707 intentId = intent.get( 'id', None )
708 intentStates.append( ( intentId, state ) )
709 except ( ValueError, TypeError ):
710 main.log.exception( "Error parsing intents" )
711 # add submitted intents not in the store
712 tmplist = [ i for i, s in intentStates ]
713 missingIntents = False
714 for i in intentIds:
715 if i not in tmplist:
716 intentStates.append( ( i, " - " ) )
717 missingIntents = True
718 intentStates.sort()
719 for i, s in intentStates:
720 count += 1
721 main.log.info( "%-6s%-15s%-15s" %
722 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700723 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700724 try:
725 missing = False
726 if leaders:
727 parsedLeaders = json.loads( leaders )
728 main.log.warn( json.dumps( parsedLeaders,
729 sort_keys=True,
730 indent=4,
731 separators=( ',', ': ' ) ) )
732 # check for all intent partitions
733 topics = []
734 for i in range( 14 ):
735 topics.append( "intent-partition-" + str( i ) )
736 main.log.debug( topics )
737 ONOStopics = [ j['topic'] for j in parsedLeaders ]
738 for topic in topics:
739 if topic not in ONOStopics:
740 main.log.error( "Error: " + topic +
741 " not in leaders" )
742 missing = True
743 else:
744 main.log.error( "leaders() returned None" )
745 except ( ValueError, TypeError ):
746 main.log.exception( "Error parsing leaders" )
747 main.log.error( repr( leaders ) )
748 # Check all nodes
749 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700750 for i in main.activeNodes:
751 response = main.CLIs[i].leaders( jsonFormat=False)
752 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700753 str( response ) )
754
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700755 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700756 try:
757 if partitions :
758 parsedPartitions = json.loads( partitions )
759 main.log.warn( json.dumps( parsedPartitions,
760 sort_keys=True,
761 indent=4,
762 separators=( ',', ': ' ) ) )
763 # TODO check for a leader in all paritions
764 # TODO check for consistency among nodes
765 else:
766 main.log.error( "partitions() returned None" )
767 except ( ValueError, TypeError ):
768 main.log.exception( "Error parsing partitions" )
769 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700770 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700771 try:
772 if pendingMap :
773 parsedPending = json.loads( pendingMap )
774 main.log.warn( json.dumps( parsedPending,
775 sort_keys=True,
776 indent=4,
777 separators=( ',', ': ' ) ) )
778 # TODO check something here?
779 else:
780 main.log.error( "pendingMap() returned None" )
781 except ( ValueError, TypeError ):
782 main.log.exception( "Error parsing pending map" )
783 main.log.error( repr( pendingMap ) )
784
785 intentAddResult = bool( intentAddResult and not missingIntents and
786 installedCheck )
787 if not intentAddResult:
788 main.log.error( "Error in pushing host intents to ONOS" )
789
790 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700791 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700792 correct = True
793 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700794 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700795 onosIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700796 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700797 onosIds.append( ids )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700798 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700799 str( sorted( onosIds ) ) )
800 if sorted( ids ) != sorted( intentIds ):
801 main.log.warn( "Set of intent IDs doesn't match" )
802 correct = False
803 break
804 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700805 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700806 for intent in intents:
807 if intent[ 'state' ] != "INSTALLED":
808 main.log.warn( "Intent " + intent[ 'id' ] +
809 " is " + intent[ 'state' ] )
810 correct = False
811 break
812 if correct:
813 break
814 else:
815 time.sleep(1)
816 if not intentStop:
817 intentStop = time.time()
818 global gossipTime
819 gossipTime = intentStop - intentStart
820 main.log.info( "It took about " + str( gossipTime ) +
821 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700822 gossipPeriod = int( main.params['timers']['gossip'] )
823 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700824 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700825 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700826 onpass="ECM anti-entropy for intents worked within " +
827 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700828 onfail="Intent ECM anti-entropy took too long. " +
829 "Expected time:{}, Actual time:{}".format( maxGossipTime,
830 gossipTime ) )
831 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700832 intentAddResult = True
833
834 if not intentAddResult or "key" in pendingMap:
835 import time
836 installedCheck = True
837 main.log.info( "Sleeping 60 seconds to see if intents are found" )
838 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700839 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700840 main.log.info( "Submitted intents: " + str( intentIds ) )
841 main.log.info( "Intents in ONOS: " + str( onosIds ) )
842 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700843 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700844 intentStates = []
845 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
846 count = 0
847 try:
848 for intent in json.loads( intents ):
849 # Iter through intents of a node
850 state = intent.get( 'state', None )
851 if "INSTALLED" not in state:
852 installedCheck = False
853 intentId = intent.get( 'id', None )
854 intentStates.append( ( intentId, state ) )
855 except ( ValueError, TypeError ):
856 main.log.exception( "Error parsing intents" )
857 # add submitted intents not in the store
858 tmplist = [ i for i, s in intentStates ]
859 for i in intentIds:
860 if i not in tmplist:
861 intentStates.append( ( i, " - " ) )
862 intentStates.sort()
863 for i, s in intentStates:
864 count += 1
865 main.log.info( "%-6s%-15s%-15s" %
866 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700867 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700868 try:
869 missing = False
870 if leaders:
871 parsedLeaders = json.loads( leaders )
872 main.log.warn( json.dumps( parsedLeaders,
873 sort_keys=True,
874 indent=4,
875 separators=( ',', ': ' ) ) )
876 # check for all intent partitions
877 # check for election
878 topics = []
879 for i in range( 14 ):
880 topics.append( "intent-partition-" + str( i ) )
881 # FIXME: this should only be after we start the app
882 topics.append( "org.onosproject.election" )
883 main.log.debug( topics )
884 ONOStopics = [ j['topic'] for j in parsedLeaders ]
885 for topic in topics:
886 if topic not in ONOStopics:
887 main.log.error( "Error: " + topic +
888 " not in leaders" )
889 missing = True
890 else:
891 main.log.error( "leaders() returned None" )
892 except ( ValueError, TypeError ):
893 main.log.exception( "Error parsing leaders" )
894 main.log.error( repr( leaders ) )
895 # Check all nodes
896 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700897 for i in main.activeNodes:
898 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700899 response = node.leaders( jsonFormat=False)
900 main.log.warn( str( node.name ) + " leaders output: \n" +
901 str( response ) )
902
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700903 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700904 try:
905 if partitions :
906 parsedPartitions = json.loads( partitions )
907 main.log.warn( json.dumps( parsedPartitions,
908 sort_keys=True,
909 indent=4,
910 separators=( ',', ': ' ) ) )
911 # TODO check for a leader in all paritions
912 # TODO check for consistency among nodes
913 else:
914 main.log.error( "partitions() returned None" )
915 except ( ValueError, TypeError ):
916 main.log.exception( "Error parsing partitions" )
917 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700918 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700919 try:
920 if pendingMap :
921 parsedPending = json.loads( pendingMap )
922 main.log.warn( json.dumps( parsedPending,
923 sort_keys=True,
924 indent=4,
925 separators=( ',', ': ' ) ) )
926 # TODO check something here?
927 else:
928 main.log.error( "pendingMap() returned None" )
929 except ( ValueError, TypeError ):
930 main.log.exception( "Error parsing pending map" )
931 main.log.error( repr( pendingMap ) )
932
933 def CASE4( self, main ):
934 """
935 Ping across added host intents
936 """
937 import json
938 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700939 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700940 assert main, "main not defined"
941 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700942 assert main.CLIs, "main.CLIs not defined"
943 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700944 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700945 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700946 "functionality and check the state of " +\
947 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700948
Jon Hall41d39f12016-04-11 22:54:35 -0700949 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700950 main.step( "Check Intent state" )
951 installedCheck = False
952 loopCount = 0
953 while not installedCheck and loopCount < 40:
954 installedCheck = True
955 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700956 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700957 intentStates = []
958 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
959 count = 0
960 # Iter through intents of a node
961 try:
962 for intent in json.loads( intents ):
963 state = intent.get( 'state', None )
964 if "INSTALLED" not in state:
965 installedCheck = False
966 intentId = intent.get( 'id', None )
967 intentStates.append( ( intentId, state ) )
968 except ( ValueError, TypeError ):
969 main.log.exception( "Error parsing intents." )
970 # Print states
971 intentStates.sort()
972 for i, s in intentStates:
973 count += 1
974 main.log.info( "%-6s%-15s%-15s" %
975 ( str( count ), str( i ), str( s ) ) )
976 if not installedCheck:
977 time.sleep( 1 )
978 loopCount += 1
979 utilities.assert_equals( expect=True, actual=installedCheck,
980 onpass="Intents are all INSTALLED",
981 onfail="Intents are not all in " +
982 "INSTALLED state" )
983
Jon Hall9d2dcad2016-04-08 10:15:20 -0700984 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700985 PingResult = main.TRUE
986 for i in range( 8, 18 ):
987 ping = main.Mininet1.pingHost( src="h" + str( i ),
988 target="h" + str( i + 10 ) )
989 PingResult = PingResult and ping
990 if ping == main.FALSE:
991 main.log.warn( "Ping failed between h" + str( i ) +
992 " and h" + str( i + 10 ) )
993 elif ping == main.TRUE:
994 main.log.info( "Ping test passed!" )
995 # Don't set PingResult or you'd override failures
996 if PingResult == main.FALSE:
997 main.log.error(
998 "Intents have not been installed correctly, pings failed." )
999 # TODO: pretty print
1000 main.log.warn( "ONOS1 intents: " )
1001 try:
1002 tmpIntents = onosCli.intents()
1003 main.log.warn( json.dumps( json.loads( tmpIntents ),
1004 sort_keys=True,
1005 indent=4,
1006 separators=( ',', ': ' ) ) )
1007 except ( ValueError, TypeError ):
1008 main.log.warn( repr( tmpIntents ) )
1009 utilities.assert_equals(
1010 expect=main.TRUE,
1011 actual=PingResult,
1012 onpass="Intents have been installed correctly and pings work",
1013 onfail="Intents have not been installed correctly, pings failed." )
1014
Jon Hall5cf14d52015-07-16 12:15:19 -07001015 main.step( "Check leadership of topics" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001016 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001017 topicCheck = main.TRUE
1018 try:
1019 if leaders:
1020 parsedLeaders = json.loads( leaders )
1021 main.log.warn( json.dumps( parsedLeaders,
1022 sort_keys=True,
1023 indent=4,
1024 separators=( ',', ': ' ) ) )
1025 # check for all intent partitions
1026 # check for election
1027 # TODO: Look at Devices as topics now that it uses this system
1028 topics = []
1029 for i in range( 14 ):
1030 topics.append( "intent-partition-" + str( i ) )
1031 # FIXME: this should only be after we start the app
1032 # FIXME: topics.append( "org.onosproject.election" )
1033 # Print leaders output
1034 main.log.debug( topics )
1035 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1036 for topic in topics:
1037 if topic not in ONOStopics:
1038 main.log.error( "Error: " + topic +
1039 " not in leaders" )
1040 topicCheck = main.FALSE
1041 else:
1042 main.log.error( "leaders() returned None" )
1043 topicCheck = main.FALSE
1044 except ( ValueError, TypeError ):
1045 topicCheck = main.FALSE
1046 main.log.exception( "Error parsing leaders" )
1047 main.log.error( repr( leaders ) )
1048 # TODO: Check for a leader of these topics
1049 # Check all nodes
1050 if topicCheck:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001051 for i in main.activeNodes:
1052 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001053 response = node.leaders( jsonFormat=False)
1054 main.log.warn( str( node.name ) + " leaders output: \n" +
1055 str( response ) )
1056
1057 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1058 onpass="intent Partitions is in leaders",
1059 onfail="Some topics were lost " )
1060 # Print partitions
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001061 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001062 try:
1063 if partitions :
1064 parsedPartitions = json.loads( partitions )
1065 main.log.warn( json.dumps( parsedPartitions,
1066 sort_keys=True,
1067 indent=4,
1068 separators=( ',', ': ' ) ) )
1069 # TODO check for a leader in all paritions
1070 # TODO check for consistency among nodes
1071 else:
1072 main.log.error( "partitions() returned None" )
1073 except ( ValueError, TypeError ):
1074 main.log.exception( "Error parsing partitions" )
1075 main.log.error( repr( partitions ) )
1076 # Print Pending Map
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001077 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001078 try:
1079 if pendingMap :
1080 parsedPending = json.loads( pendingMap )
1081 main.log.warn( json.dumps( parsedPending,
1082 sort_keys=True,
1083 indent=4,
1084 separators=( ',', ': ' ) ) )
1085 # TODO check something here?
1086 else:
1087 main.log.error( "pendingMap() returned None" )
1088 except ( ValueError, TypeError ):
1089 main.log.exception( "Error parsing pending map" )
1090 main.log.error( repr( pendingMap ) )
1091
1092 if not installedCheck:
1093 main.log.info( "Waiting 60 seconds to see if the state of " +
1094 "intents change" )
1095 time.sleep( 60 )
1096 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001097 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001098 intentStates = []
1099 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1100 count = 0
1101 # Iter through intents of a node
1102 try:
1103 for intent in json.loads( intents ):
1104 state = intent.get( 'state', None )
1105 if "INSTALLED" not in state:
1106 installedCheck = False
1107 intentId = intent.get( 'id', None )
1108 intentStates.append( ( intentId, state ) )
1109 except ( ValueError, TypeError ):
1110 main.log.exception( "Error parsing intents." )
1111 intentStates.sort()
1112 for i, s in intentStates:
1113 count += 1
1114 main.log.info( "%-6s%-15s%-15s" %
1115 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001116 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001117 try:
1118 missing = False
1119 if leaders:
1120 parsedLeaders = json.loads( leaders )
1121 main.log.warn( json.dumps( parsedLeaders,
1122 sort_keys=True,
1123 indent=4,
1124 separators=( ',', ': ' ) ) )
1125 # check for all intent partitions
1126 # check for election
1127 topics = []
1128 for i in range( 14 ):
1129 topics.append( "intent-partition-" + str( i ) )
1130 # FIXME: this should only be after we start the app
1131 topics.append( "org.onosproject.election" )
1132 main.log.debug( topics )
1133 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1134 for topic in topics:
1135 if topic not in ONOStopics:
1136 main.log.error( "Error: " + topic +
1137 " not in leaders" )
1138 missing = True
1139 else:
1140 main.log.error( "leaders() returned None" )
1141 except ( ValueError, TypeError ):
1142 main.log.exception( "Error parsing leaders" )
1143 main.log.error( repr( leaders ) )
1144 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001145 for i in main.activeNodes:
1146 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001147 response = node.leaders( jsonFormat=False)
1148 main.log.warn( str( node.name ) + " leaders output: \n" +
1149 str( response ) )
1150
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001151 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001152 try:
1153 if partitions :
1154 parsedPartitions = json.loads( partitions )
1155 main.log.warn( json.dumps( parsedPartitions,
1156 sort_keys=True,
1157 indent=4,
1158 separators=( ',', ': ' ) ) )
1159 # TODO check for a leader in all paritions
1160 # TODO check for consistency among nodes
1161 else:
1162 main.log.error( "partitions() returned None" )
1163 except ( ValueError, TypeError ):
1164 main.log.exception( "Error parsing partitions" )
1165 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001166 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001167 try:
1168 if pendingMap :
1169 parsedPending = json.loads( pendingMap )
1170 main.log.warn( json.dumps( parsedPending,
1171 sort_keys=True,
1172 indent=4,
1173 separators=( ',', ': ' ) ) )
1174 # TODO check something here?
1175 else:
1176 main.log.error( "pendingMap() returned None" )
1177 except ( ValueError, TypeError ):
1178 main.log.exception( "Error parsing pending map" )
1179 main.log.error( repr( pendingMap ) )
1180 # Print flowrules
Jon Hall41d39f12016-04-11 22:54:35 -07001181 main.log.debug( onosCli.flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001182 main.step( "Wait a minute then ping again" )
1183 # the wait is above
1184 PingResult = main.TRUE
1185 for i in range( 8, 18 ):
1186 ping = main.Mininet1.pingHost( src="h" + str( i ),
1187 target="h" + str( i + 10 ) )
1188 PingResult = PingResult and ping
1189 if ping == main.FALSE:
1190 main.log.warn( "Ping failed between h" + str( i ) +
1191 " and h" + str( i + 10 ) )
1192 elif ping == main.TRUE:
1193 main.log.info( "Ping test passed!" )
1194 # Don't set PingResult or you'd override failures
1195 if PingResult == main.FALSE:
1196 main.log.error(
1197 "Intents have not been installed correctly, pings failed." )
1198 # TODO: pretty print
1199 main.log.warn( "ONOS1 intents: " )
1200 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001201 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001202 main.log.warn( json.dumps( json.loads( tmpIntents ),
1203 sort_keys=True,
1204 indent=4,
1205 separators=( ',', ': ' ) ) )
1206 except ( ValueError, TypeError ):
1207 main.log.warn( repr( tmpIntents ) )
1208 utilities.assert_equals(
1209 expect=main.TRUE,
1210 actual=PingResult,
1211 onpass="Intents have been installed correctly and pings work",
1212 onfail="Intents have not been installed correctly, pings failed." )
1213
1214 def CASE5( self, main ):
1215 """
1216 Reading state of ONOS
1217 """
1218 import json
1219 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001220 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001221 assert main, "main not defined"
1222 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001223 assert main.CLIs, "main.CLIs not defined"
1224 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001225
1226 main.case( "Setting up and gathering data for current state" )
1227 # The general idea for this test case is to pull the state of
1228 # ( intents,flows, topology,... ) from each ONOS node
1229 # We can then compare them with each other and also with past states
1230
1231 main.step( "Check that each switch has a master" )
1232 global mastershipState
1233 mastershipState = '[]'
1234
1235 # Assert that each device has a master
1236 rolesNotNull = main.TRUE
1237 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001238 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001239 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001240 name="rolesNotNull-" + str( i ),
1241 args=[] )
1242 threads.append( t )
1243 t.start()
1244
1245 for t in threads:
1246 t.join()
1247 rolesNotNull = rolesNotNull and t.result
1248 utilities.assert_equals(
1249 expect=main.TRUE,
1250 actual=rolesNotNull,
1251 onpass="Each device has a master",
1252 onfail="Some devices don't have a master assigned" )
1253
1254 main.step( "Get the Mastership of each switch from each controller" )
1255 ONOSMastership = []
1256 mastershipCheck = main.FALSE
1257 consistentMastership = True
1258 rolesResults = True
1259 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001260 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001261 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001262 name="roles-" + str( i ),
1263 args=[] )
1264 threads.append( t )
1265 t.start()
1266
1267 for t in threads:
1268 t.join()
1269 ONOSMastership.append( t.result )
1270
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001271 for i in range( len( ONOSMastership ) ):
1272 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001273 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001274 main.log.error( "Error in getting ONOS" + node + " roles" )
1275 main.log.warn( "ONOS" + node + " mastership response: " +
1276 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001277 rolesResults = False
1278 utilities.assert_equals(
1279 expect=True,
1280 actual=rolesResults,
1281 onpass="No error in reading roles output",
1282 onfail="Error in reading roles from ONOS" )
1283
1284 main.step( "Check for consistency in roles from each controller" )
1285 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1286 main.log.info(
1287 "Switch roles are consistent across all ONOS nodes" )
1288 else:
1289 consistentMastership = False
1290 utilities.assert_equals(
1291 expect=True,
1292 actual=consistentMastership,
1293 onpass="Switch roles are consistent across all ONOS nodes",
1294 onfail="ONOS nodes have different views of switch roles" )
1295
1296 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001297 for i in range( len( main.activeNodes ) ):
1298 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001299 try:
1300 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001301 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001302 json.dumps(
1303 json.loads( ONOSMastership[ i ] ),
1304 sort_keys=True,
1305 indent=4,
1306 separators=( ',', ': ' ) ) )
1307 except ( ValueError, TypeError ):
1308 main.log.warn( repr( ONOSMastership[ i ] ) )
1309 elif rolesResults and consistentMastership:
1310 mastershipCheck = main.TRUE
1311 mastershipState = ONOSMastership[ 0 ]
1312
1313 main.step( "Get the intents from each controller" )
1314 global intentState
1315 intentState = []
1316 ONOSIntents = []
1317 intentCheck = main.FALSE
1318 consistentIntents = True
1319 intentsResults = True
1320 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001321 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001322 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001323 name="intents-" + str( i ),
1324 args=[],
1325 kwargs={ 'jsonFormat': True } )
1326 threads.append( t )
1327 t.start()
1328
1329 for t in threads:
1330 t.join()
1331 ONOSIntents.append( t.result )
1332
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001333 for i in range( len( ONOSIntents ) ):
1334 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001335 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001336 main.log.error( "Error in getting ONOS" + node + " intents" )
1337 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001338 repr( ONOSIntents[ i ] ) )
1339 intentsResults = False
1340 utilities.assert_equals(
1341 expect=True,
1342 actual=intentsResults,
1343 onpass="No error in reading intents output",
1344 onfail="Error in reading intents from ONOS" )
1345
1346 main.step( "Check for consistency in Intents from each controller" )
1347 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1348 main.log.info( "Intents are consistent across all ONOS " +
1349 "nodes" )
1350 else:
1351 consistentIntents = False
1352 main.log.error( "Intents not consistent" )
1353 utilities.assert_equals(
1354 expect=True,
1355 actual=consistentIntents,
1356 onpass="Intents are consistent across all ONOS nodes",
1357 onfail="ONOS nodes have different views of intents" )
1358
1359 if intentsResults:
1360 # Try to make it easy to figure out what is happening
1361 #
1362 # Intent ONOS1 ONOS2 ...
1363 # 0x01 INSTALLED INSTALLING
1364 # ... ... ...
1365 # ... ... ...
1366 title = " Id"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001367 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001368 title += " " * 10 + "ONOS" + str( n + 1 )
1369 main.log.warn( title )
1370 # get all intent keys in the cluster
1371 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001372 try:
1373 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001374 for nodeStr in ONOSIntents:
1375 node = json.loads( nodeStr )
1376 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001377 keys.append( intent.get( 'id' ) )
1378 keys = set( keys )
1379 # For each intent key, print the state on each node
1380 for key in keys:
1381 row = "%-13s" % key
1382 for nodeStr in ONOSIntents:
1383 node = json.loads( nodeStr )
1384 for intent in node:
1385 if intent.get( 'id', "Error" ) == key:
1386 row += "%-15s" % intent.get( 'state' )
1387 main.log.warn( row )
1388 # End of intent state table
1389 except ValueError as e:
1390 main.log.exception( e )
1391 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001392
1393 if intentsResults and not consistentIntents:
1394 # print the json objects
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001395 n = str( main.activeNodes[-1] + 1 )
1396 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001397 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1398 sort_keys=True,
1399 indent=4,
1400 separators=( ',', ': ' ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001401 for i in range( len( ONOSIntents ) ):
1402 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001403 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001404 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001405 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1406 sort_keys=True,
1407 indent=4,
1408 separators=( ',', ': ' ) ) )
1409 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001410 main.log.debug( "ONOS" + node + " intents match ONOS" +
1411 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001412 elif intentsResults and consistentIntents:
1413 intentCheck = main.TRUE
1414 intentState = ONOSIntents[ 0 ]
1415
1416 main.step( "Get the flows from each controller" )
1417 global flowState
1418 flowState = []
1419 ONOSFlows = []
1420 ONOSFlowsJson = []
1421 flowCheck = main.FALSE
1422 consistentFlows = True
1423 flowsResults = True
1424 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001425 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001426 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001427 name="flows-" + str( i ),
1428 args=[],
1429 kwargs={ 'jsonFormat': True } )
1430 threads.append( t )
1431 t.start()
1432
1433 # NOTE: Flows command can take some time to run
1434 time.sleep(30)
1435 for t in threads:
1436 t.join()
1437 result = t.result
1438 ONOSFlows.append( result )
1439
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001440 for i in range( len( ONOSFlows ) ):
1441 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001442 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1443 main.log.error( "Error in getting ONOS" + num + " flows" )
1444 main.log.warn( "ONOS" + num + " flows response: " +
1445 repr( ONOSFlows[ i ] ) )
1446 flowsResults = False
1447 ONOSFlowsJson.append( None )
1448 else:
1449 try:
1450 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1451 except ( ValueError, TypeError ):
1452 # FIXME: change this to log.error?
1453 main.log.exception( "Error in parsing ONOS" + num +
1454 " response as json." )
1455 main.log.error( repr( ONOSFlows[ i ] ) )
1456 ONOSFlowsJson.append( None )
1457 flowsResults = False
1458 utilities.assert_equals(
1459 expect=True,
1460 actual=flowsResults,
1461 onpass="No error in reading flows output",
1462 onfail="Error in reading flows from ONOS" )
1463
1464 main.step( "Check for consistency in Flows from each controller" )
1465 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1466 if all( tmp ):
1467 main.log.info( "Flow count is consistent across all ONOS nodes" )
1468 else:
1469 consistentFlows = False
1470 utilities.assert_equals(
1471 expect=True,
1472 actual=consistentFlows,
1473 onpass="The flow count is consistent across all ONOS nodes",
1474 onfail="ONOS nodes have different flow counts" )
1475
1476 if flowsResults and not consistentFlows:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001477 for i in range( len( ONOSFlows ) ):
1478 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001479 try:
1480 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001481 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001482 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1483 indent=4, separators=( ',', ': ' ) ) )
1484 except ( ValueError, TypeError ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001485 main.log.warn( "ONOS" + node + " flows: " +
1486 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001487 elif flowsResults and consistentFlows:
1488 flowCheck = main.TRUE
1489 flowState = ONOSFlows[ 0 ]
1490
1491 main.step( "Get the OF Table entries" )
1492 global flows
1493 flows = []
1494 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001495 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001496 if flowCheck == main.FALSE:
1497 for table in flows:
1498 main.log.warn( table )
1499 # TODO: Compare switch flow tables with ONOS flow tables
1500
1501 main.step( "Start continuous pings" )
1502 main.Mininet2.pingLong(
1503 src=main.params[ 'PING' ][ 'source1' ],
1504 target=main.params[ 'PING' ][ 'target1' ],
1505 pingTime=500 )
1506 main.Mininet2.pingLong(
1507 src=main.params[ 'PING' ][ 'source2' ],
1508 target=main.params[ 'PING' ][ 'target2' ],
1509 pingTime=500 )
1510 main.Mininet2.pingLong(
1511 src=main.params[ 'PING' ][ 'source3' ],
1512 target=main.params[ 'PING' ][ 'target3' ],
1513 pingTime=500 )
1514 main.Mininet2.pingLong(
1515 src=main.params[ 'PING' ][ 'source4' ],
1516 target=main.params[ 'PING' ][ 'target4' ],
1517 pingTime=500 )
1518 main.Mininet2.pingLong(
1519 src=main.params[ 'PING' ][ 'source5' ],
1520 target=main.params[ 'PING' ][ 'target5' ],
1521 pingTime=500 )
1522 main.Mininet2.pingLong(
1523 src=main.params[ 'PING' ][ 'source6' ],
1524 target=main.params[ 'PING' ][ 'target6' ],
1525 pingTime=500 )
1526 main.Mininet2.pingLong(
1527 src=main.params[ 'PING' ][ 'source7' ],
1528 target=main.params[ 'PING' ][ 'target7' ],
1529 pingTime=500 )
1530 main.Mininet2.pingLong(
1531 src=main.params[ 'PING' ][ 'source8' ],
1532 target=main.params[ 'PING' ][ 'target8' ],
1533 pingTime=500 )
1534 main.Mininet2.pingLong(
1535 src=main.params[ 'PING' ][ 'source9' ],
1536 target=main.params[ 'PING' ][ 'target9' ],
1537 pingTime=500 )
1538 main.Mininet2.pingLong(
1539 src=main.params[ 'PING' ][ 'source10' ],
1540 target=main.params[ 'PING' ][ 'target10' ],
1541 pingTime=500 )
1542
1543 main.step( "Collecting topology information from ONOS" )
1544 devices = []
1545 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001546 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001547 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001548 name="devices-" + str( i ),
1549 args=[ ] )
1550 threads.append( t )
1551 t.start()
1552
1553 for t in threads:
1554 t.join()
1555 devices.append( t.result )
1556 hosts = []
1557 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001558 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001559 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001560 name="hosts-" + str( i ),
1561 args=[ ] )
1562 threads.append( t )
1563 t.start()
1564
1565 for t in threads:
1566 t.join()
1567 try:
1568 hosts.append( json.loads( t.result ) )
1569 except ( ValueError, TypeError ):
1570 # FIXME: better handling of this, print which node
1571 # Maybe use thread name?
1572 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001573 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001574 hosts.append( None )
1575
1576 ports = []
1577 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001578 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001579 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001580 name="ports-" + str( i ),
1581 args=[ ] )
1582 threads.append( t )
1583 t.start()
1584
1585 for t in threads:
1586 t.join()
1587 ports.append( t.result )
1588 links = []
1589 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001590 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001591 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001592 name="links-" + str( i ),
1593 args=[ ] )
1594 threads.append( t )
1595 t.start()
1596
1597 for t in threads:
1598 t.join()
1599 links.append( t.result )
1600 clusters = []
1601 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001602 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001603 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001604 name="clusters-" + str( i ),
1605 args=[ ] )
1606 threads.append( t )
1607 t.start()
1608
1609 for t in threads:
1610 t.join()
1611 clusters.append( t.result )
1612 # Compare json objects for hosts and dataplane clusters
1613
1614 # hosts
1615 main.step( "Host view is consistent across ONOS nodes" )
1616 consistentHostsResult = main.TRUE
1617 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001618 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001619 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001620 if hosts[ controller ] == hosts[ 0 ]:
1621 continue
1622 else: # hosts not consistent
1623 main.log.error( "hosts from ONOS" +
1624 controllerStr +
1625 " is inconsistent with ONOS1" )
1626 main.log.warn( repr( hosts[ controller ] ) )
1627 consistentHostsResult = main.FALSE
1628
1629 else:
1630 main.log.error( "Error in getting ONOS hosts from ONOS" +
1631 controllerStr )
1632 consistentHostsResult = main.FALSE
1633 main.log.warn( "ONOS" + controllerStr +
1634 " hosts response: " +
1635 repr( hosts[ controller ] ) )
1636 utilities.assert_equals(
1637 expect=main.TRUE,
1638 actual=consistentHostsResult,
1639 onpass="Hosts view is consistent across all ONOS nodes",
1640 onfail="ONOS nodes have different views of hosts" )
1641
1642 main.step( "Each host has an IP address" )
1643 ipResult = main.TRUE
1644 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001645 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001646 if hosts[ controller ]:
1647 for host in hosts[ controller ]:
1648 if not host.get( 'ipAddresses', [ ] ):
1649 main.log.error( "Error with host ips on controller" +
1650 controllerStr + ": " + str( host ) )
1651 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001652 utilities.assert_equals(
1653 expect=main.TRUE,
1654 actual=ipResult,
1655 onpass="The ips of the hosts aren't empty",
1656 onfail="The ip of at least one host is missing" )
1657
1658 # Strongly connected clusters of devices
1659 main.step( "Cluster view is consistent across ONOS nodes" )
1660 consistentClustersResult = main.TRUE
1661 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001662 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001663 if "Error" not in clusters[ controller ]:
1664 if clusters[ controller ] == clusters[ 0 ]:
1665 continue
1666 else: # clusters not consistent
1667 main.log.error( "clusters from ONOS" + controllerStr +
1668 " is inconsistent with ONOS1" )
1669 consistentClustersResult = main.FALSE
1670
1671 else:
1672 main.log.error( "Error in getting dataplane clusters " +
1673 "from ONOS" + controllerStr )
1674 consistentClustersResult = main.FALSE
1675 main.log.warn( "ONOS" + controllerStr +
1676 " clusters response: " +
1677 repr( clusters[ controller ] ) )
1678 utilities.assert_equals(
1679 expect=main.TRUE,
1680 actual=consistentClustersResult,
1681 onpass="Clusters view is consistent across all ONOS nodes",
1682 onfail="ONOS nodes have different views of clusters" )
Jon Hall172b7ba2016-04-07 18:12:20 -07001683 if consistentClustersResult != main.TRUE:
1684 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07001685 # there should always only be one cluster
1686 main.step( "Cluster view correct across ONOS nodes" )
1687 try:
1688 numClusters = len( json.loads( clusters[ 0 ] ) )
1689 except ( ValueError, TypeError ):
1690 main.log.exception( "Error parsing clusters[0]: " +
1691 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001692 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001693 clusterResults = main.FALSE
1694 if numClusters == 1:
1695 clusterResults = main.TRUE
1696 utilities.assert_equals(
1697 expect=1,
1698 actual=numClusters,
1699 onpass="ONOS shows 1 SCC",
1700 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1701
1702 main.step( "Comparing ONOS topology to MN" )
1703 devicesResults = main.TRUE
1704 linksResults = main.TRUE
1705 hostsResults = main.TRUE
1706 mnSwitches = main.Mininet1.getSwitches()
1707 mnLinks = main.Mininet1.getLinks()
1708 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001709 for controller in main.activeNodes:
1710 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001711 if devices[ controller ] and ports[ controller ] and\
1712 "Error" not in devices[ controller ] and\
1713 "Error" not in ports[ controller ]:
Jon Hall6e709752016-02-01 13:38:46 -08001714 currentDevicesResult = main.Mininet1.compareSwitches(
1715 mnSwitches,
1716 json.loads( devices[ controller ] ),
1717 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001718 else:
1719 currentDevicesResult = main.FALSE
1720 utilities.assert_equals( expect=main.TRUE,
1721 actual=currentDevicesResult,
1722 onpass="ONOS" + controllerStr +
1723 " Switches view is correct",
1724 onfail="ONOS" + controllerStr +
1725 " Switches view is incorrect" )
1726 if links[ controller ] and "Error" not in links[ controller ]:
1727 currentLinksResult = main.Mininet1.compareLinks(
1728 mnSwitches, mnLinks,
1729 json.loads( links[ controller ] ) )
1730 else:
1731 currentLinksResult = main.FALSE
1732 utilities.assert_equals( expect=main.TRUE,
1733 actual=currentLinksResult,
1734 onpass="ONOS" + controllerStr +
1735 " links view is correct",
1736 onfail="ONOS" + controllerStr +
1737 " links view is incorrect" )
1738
Jon Hall657cdf62015-12-17 14:40:51 -08001739 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001740 currentHostsResult = main.Mininet1.compareHosts(
1741 mnHosts,
1742 hosts[ controller ] )
1743 else:
1744 currentHostsResult = main.FALSE
1745 utilities.assert_equals( expect=main.TRUE,
1746 actual=currentHostsResult,
1747 onpass="ONOS" + controllerStr +
1748 " hosts exist in Mininet",
1749 onfail="ONOS" + controllerStr +
1750 " hosts don't match Mininet" )
1751
1752 devicesResults = devicesResults and currentDevicesResult
1753 linksResults = linksResults and currentLinksResult
1754 hostsResults = hostsResults and currentHostsResult
1755
1756 main.step( "Device information is correct" )
1757 utilities.assert_equals(
1758 expect=main.TRUE,
1759 actual=devicesResults,
1760 onpass="Device information is correct",
1761 onfail="Device information is incorrect" )
1762
1763 main.step( "Links are correct" )
1764 utilities.assert_equals(
1765 expect=main.TRUE,
1766 actual=linksResults,
1767 onpass="Link are correct",
1768 onfail="Links are incorrect" )
1769
1770 main.step( "Hosts are correct" )
1771 utilities.assert_equals(
1772 expect=main.TRUE,
1773 actual=hostsResults,
1774 onpass="Hosts are correct",
1775 onfail="Hosts are incorrect" )
1776
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001777 def CASE61( self, main ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001778 """
1779 The Failure case.
1780 """
Jon Halle1a3b752015-07-22 13:02:46 -07001781 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001782 assert main, "main not defined"
1783 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001784 assert main.CLIs, "main.CLIs not defined"
1785 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001786 main.case( "Stop minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -07001787
1788 main.step( "Checking ONOS Logs for errors" )
1789 for node in main.nodes:
1790 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1791 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1792
Jon Hall3b489db2015-10-05 14:38:37 -07001793 n = len( main.nodes ) # Number of nodes
1794 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
1795 main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
1796 if n > 3:
1797 main.kill.append( p - 1 )
1798 # NOTE: This only works for cluster sizes of 3,5, or 7.
1799
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001800 main.step( "Stopping " + str( len( main.kill ) ) + " ONOS nodes" )
Jon Hall3b489db2015-10-05 14:38:37 -07001801 killResults = main.TRUE
1802 for i in main.kill:
1803 killResults = killResults and\
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001804 main.ONOSbench.onosStop( main.nodes[i].ip_address )
1805 main.activeNodes.remove( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001806 utilities.assert_equals( expect=main.TRUE, actual=killResults,
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001807 onpass="ONOS nodes stopped successfully",
1808 onfail="ONOS nodes NOT successfully stopped" )
1809
1810 def CASE62( self, main ):
1811 """
1812 The bring up stopped nodes
1813 """
1814 import time
1815 assert main.numCtrls, "main.numCtrls not defined"
1816 assert main, "main not defined"
1817 assert utilities.assert_equals, "utilities.assert_equals not defined"
1818 assert main.CLIs, "main.CLIs not defined"
1819 assert main.nodes, "main.nodes not defined"
1820 assert main.kill, "main.kill not defined"
1821 main.case( "Restart minority of ONOS nodes" )
1822
1823 main.step( "Restarting " + str( len( main.kill ) ) + " ONOS nodes" )
1824 startResults = main.TRUE
1825 restartTime = time.time()
1826 for i in main.kill:
1827 startResults = startResults and\
1828 main.ONOSbench.onosStart( main.nodes[i].ip_address )
1829 utilities.assert_equals( expect=main.TRUE, actual=startResults,
1830 onpass="ONOS nodes started successfully",
1831 onfail="ONOS nodes NOT successfully started" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001832
1833 main.step( "Checking if ONOS is up yet" )
1834 count = 0
1835 onosIsupResult = main.FALSE
1836 while onosIsupResult == main.FALSE and count < 10:
Jon Hall3b489db2015-10-05 14:38:37 -07001837 onosIsupResult = main.TRUE
1838 for i in main.kill:
1839 onosIsupResult = onosIsupResult and\
1840 main.ONOSbench.isup( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001841 count = count + 1
Jon Hall5cf14d52015-07-16 12:15:19 -07001842 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1843 onpass="ONOS restarted successfully",
1844 onfail="ONOS restart NOT successful" )
1845
Jon Halle1a3b752015-07-22 13:02:46 -07001846 main.step( "Restarting ONOS main.CLIs" )
Jon Hall3b489db2015-10-05 14:38:37 -07001847 cliResults = main.TRUE
1848 for i in main.kill:
1849 cliResults = cliResults and\
1850 main.CLIs[i].startOnosCli( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001851 main.activeNodes.append( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001852 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1853 onpass="ONOS cli restarted",
1854 onfail="ONOS cli did not restart" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001855 main.activeNodes.sort()
1856 try:
1857 assert list( set( main.activeNodes ) ) == main.activeNodes,\
1858 "List of active nodes has duplicates, this likely indicates something was run out of order"
1859 except AssertionError:
1860 main.log.exception( "" )
1861 main.cleanup()
1862 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -07001863
1864 # Grab the time of restart so we chan check how long the gossip
1865 # protocol has had time to work
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001866 main.restartTime = time.time() - restartTime
Jon Hall5cf14d52015-07-16 12:15:19 -07001867 main.log.debug( "Restart time: " + str( main.restartTime ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001868 # TODO: MAke this configurable. Also, we are breaking the above timer
1869 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001870 node = main.activeNodes[0]
1871 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1872 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1873 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001874
Jon Halla440e872016-03-31 15:15:50 -07001875 main.step( "Rerun for election on the node(s) that were killed" )
1876 runResults = main.TRUE
1877 for i in main.kill:
1878 runResults = runResults and\
1879 main.CLIs[i].electionTestRun()
1880 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1881 onpass="ONOS nodes reran for election topic",
1882 onfail="Errror rerunning for election" )
1883
Jon Hall5cf14d52015-07-16 12:15:19 -07001884 def CASE7( self, main ):
1885 """
1886 Check state after ONOS failure
1887 """
1888 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001889 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001890 assert main, "main not defined"
1891 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001892 assert main.CLIs, "main.CLIs not defined"
1893 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001894 try:
1895 main.kill
1896 except AttributeError:
1897 main.kill = []
1898
Jon Hall5cf14d52015-07-16 12:15:19 -07001899 main.case( "Running ONOS Constant State Tests" )
1900
1901 main.step( "Check that each switch has a master" )
1902 # Assert that each device has a master
1903 rolesNotNull = main.TRUE
1904 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001905 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001906 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001907 name="rolesNotNull-" + str( i ),
1908 args=[ ] )
1909 threads.append( t )
1910 t.start()
1911
1912 for t in threads:
1913 t.join()
1914 rolesNotNull = rolesNotNull and t.result
1915 utilities.assert_equals(
1916 expect=main.TRUE,
1917 actual=rolesNotNull,
1918 onpass="Each device has a master",
1919 onfail="Some devices don't have a master assigned" )
1920
1921 main.step( "Read device roles from ONOS" )
1922 ONOSMastership = []
Jon Halla440e872016-03-31 15:15:50 -07001923 mastershipCheck = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001924 consistentMastership = True
1925 rolesResults = True
1926 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001927 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001928 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001929 name="roles-" + str( i ),
1930 args=[] )
1931 threads.append( t )
1932 t.start()
1933
1934 for t in threads:
1935 t.join()
1936 ONOSMastership.append( t.result )
1937
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001938 for i in range( len( ONOSMastership ) ):
1939 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001940 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001941 main.log.error( "Error in getting ONOS" + node + " roles" )
1942 main.log.warn( "ONOS" + node + " mastership response: " +
1943 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001944 rolesResults = False
1945 utilities.assert_equals(
1946 expect=True,
1947 actual=rolesResults,
1948 onpass="No error in reading roles output",
1949 onfail="Error in reading roles from ONOS" )
1950
1951 main.step( "Check for consistency in roles from each controller" )
1952 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1953 main.log.info(
1954 "Switch roles are consistent across all ONOS nodes" )
1955 else:
1956 consistentMastership = False
1957 utilities.assert_equals(
1958 expect=True,
1959 actual=consistentMastership,
1960 onpass="Switch roles are consistent across all ONOS nodes",
1961 onfail="ONOS nodes have different views of switch roles" )
1962
1963 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001964 for i in range( len( ONOSMastership ) ):
1965 node = str( main.activeNodes[i] + 1 )
1966 main.log.warn( "ONOS" + node + " roles: ",
1967 json.dumps( json.loads( ONOSMastership[ i ] ),
1968 sort_keys=True,
1969 indent=4,
1970 separators=( ',', ': ' ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001971
1972 # NOTE: we expect mastership to change on controller failure
Jon Hall5cf14d52015-07-16 12:15:19 -07001973
1974 main.step( "Get the intents and compare across all nodes" )
1975 ONOSIntents = []
1976 intentCheck = main.FALSE
1977 consistentIntents = True
1978 intentsResults = True
1979 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001980 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001981 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001982 name="intents-" + str( i ),
1983 args=[],
1984 kwargs={ 'jsonFormat': True } )
1985 threads.append( t )
1986 t.start()
1987
1988 for t in threads:
1989 t.join()
1990 ONOSIntents.append( t.result )
1991
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001992 for i in range( len( ONOSIntents) ):
1993 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001994 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001995 main.log.error( "Error in getting ONOS" + node + " intents" )
1996 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001997 repr( ONOSIntents[ i ] ) )
1998 intentsResults = False
1999 utilities.assert_equals(
2000 expect=True,
2001 actual=intentsResults,
2002 onpass="No error in reading intents output",
2003 onfail="Error in reading intents from ONOS" )
2004
2005 main.step( "Check for consistency in Intents from each controller" )
2006 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2007 main.log.info( "Intents are consistent across all ONOS " +
2008 "nodes" )
2009 else:
2010 consistentIntents = False
2011
2012 # Try to make it easy to figure out what is happening
2013 #
2014 # Intent ONOS1 ONOS2 ...
2015 # 0x01 INSTALLED INSTALLING
2016 # ... ... ...
2017 # ... ... ...
2018 title = " ID"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002019 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002020 title += " " * 10 + "ONOS" + str( n + 1 )
2021 main.log.warn( title )
2022 # get all intent keys in the cluster
2023 keys = []
2024 for nodeStr in ONOSIntents:
2025 node = json.loads( nodeStr )
2026 for intent in node:
2027 keys.append( intent.get( 'id' ) )
2028 keys = set( keys )
2029 for key in keys:
2030 row = "%-13s" % key
2031 for nodeStr in ONOSIntents:
2032 node = json.loads( nodeStr )
2033 for intent in node:
2034 if intent.get( 'id' ) == key:
2035 row += "%-15s" % intent.get( 'state' )
2036 main.log.warn( row )
2037 # End table view
2038
2039 utilities.assert_equals(
2040 expect=True,
2041 actual=consistentIntents,
2042 onpass="Intents are consistent across all ONOS nodes",
2043 onfail="ONOS nodes have different views of intents" )
2044 intentStates = []
2045 for node in ONOSIntents: # Iter through ONOS nodes
2046 nodeStates = []
2047 # Iter through intents of a node
2048 try:
2049 for intent in json.loads( node ):
2050 nodeStates.append( intent[ 'state' ] )
2051 except ( ValueError, TypeError ):
2052 main.log.exception( "Error in parsing intents" )
2053 main.log.error( repr( node ) )
2054 intentStates.append( nodeStates )
2055 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2056 main.log.info( dict( out ) )
2057
2058 if intentsResults and not consistentIntents:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002059 for i in range( len( main.activeNodes ) ):
2060 node = str( main.activeNodes[i] + 1 )
2061 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002062 main.log.warn( json.dumps(
2063 json.loads( ONOSIntents[ i ] ),
2064 sort_keys=True,
2065 indent=4,
2066 separators=( ',', ': ' ) ) )
2067 elif intentsResults and consistentIntents:
2068 intentCheck = main.TRUE
2069
2070 # NOTE: Store has no durability, so intents are lost across system
2071 # restarts
2072 main.step( "Compare current intents with intents before the failure" )
2073 # NOTE: this requires case 5 to pass for intentState to be set.
2074 # maybe we should stop the test if that fails?
2075 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002076 try:
2077 intentState
2078 except NameError:
2079 main.log.warn( "No previous intent state was saved" )
2080 else:
2081 if intentState and intentState == ONOSIntents[ 0 ]:
2082 sameIntents = main.TRUE
2083 main.log.info( "Intents are consistent with before failure" )
2084 # TODO: possibly the states have changed? we may need to figure out
2085 # what the acceptable states are
2086 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2087 sameIntents = main.TRUE
2088 try:
2089 before = json.loads( intentState )
2090 after = json.loads( ONOSIntents[ 0 ] )
2091 for intent in before:
2092 if intent not in after:
2093 sameIntents = main.FALSE
2094 main.log.debug( "Intent is not currently in ONOS " +
2095 "(at least in the same form):" )
2096 main.log.debug( json.dumps( intent ) )
2097 except ( ValueError, TypeError ):
2098 main.log.exception( "Exception printing intents" )
2099 main.log.debug( repr( ONOSIntents[0] ) )
2100 main.log.debug( repr( intentState ) )
2101 if sameIntents == main.FALSE:
2102 try:
2103 main.log.debug( "ONOS intents before: " )
2104 main.log.debug( json.dumps( json.loads( intentState ),
2105 sort_keys=True, indent=4,
2106 separators=( ',', ': ' ) ) )
2107 main.log.debug( "Current ONOS intents: " )
2108 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2109 sort_keys=True, indent=4,
2110 separators=( ',', ': ' ) ) )
2111 except ( ValueError, TypeError ):
2112 main.log.exception( "Exception printing intents" )
2113 main.log.debug( repr( ONOSIntents[0] ) )
2114 main.log.debug( repr( intentState ) )
2115 utilities.assert_equals(
2116 expect=main.TRUE,
2117 actual=sameIntents,
2118 onpass="Intents are consistent with before failure",
2119 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002120 intentCheck = intentCheck and sameIntents
2121
2122 main.step( "Get the OF Table entries and compare to before " +
2123 "component failure" )
2124 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002125 for i in range( 28 ):
2126 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002127 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002128 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2129 FlowTables = FlowTables and curSwitch
2130 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002131 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002132 utilities.assert_equals(
2133 expect=main.TRUE,
2134 actual=FlowTables,
2135 onpass="No changes were found in the flow tables",
2136 onfail="Changes were found in the flow tables" )
2137
2138 main.Mininet2.pingLongKill()
2139 '''
2140 main.step( "Check the continuous pings to ensure that no packets " +
2141 "were dropped during component failure" )
2142 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2143 main.params[ 'TESTONIP' ] )
2144 LossInPings = main.FALSE
2145 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2146 for i in range( 8, 18 ):
2147 main.log.info(
2148 "Checking for a loss in pings along flow from s" +
2149 str( i ) )
2150 LossInPings = main.Mininet2.checkForLoss(
2151 "/tmp/ping.h" +
2152 str( i ) ) or LossInPings
2153 if LossInPings == main.TRUE:
2154 main.log.info( "Loss in ping detected" )
2155 elif LossInPings == main.ERROR:
2156 main.log.info( "There are multiple mininet process running" )
2157 elif LossInPings == main.FALSE:
2158 main.log.info( "No Loss in the pings" )
2159 main.log.info( "No loss of dataplane connectivity" )
2160 utilities.assert_equals(
2161 expect=main.FALSE,
2162 actual=LossInPings,
2163 onpass="No Loss of connectivity",
2164 onfail="Loss of dataplane connectivity detected" )
2165 '''
2166
2167 main.step( "Leadership Election is still functional" )
2168 # Test of LeadershipElection
2169 leaderList = []
Jon Hall5cf14d52015-07-16 12:15:19 -07002170
Jon Hall3b489db2015-10-05 14:38:37 -07002171 restarted = []
2172 for i in main.kill:
2173 restarted.append( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002174 leaderResult = main.TRUE
Jon Hall3b489db2015-10-05 14:38:37 -07002175
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002176 for i in main.activeNodes:
2177 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002178 leaderN = cli.electionTestLeader()
2179 leaderList.append( leaderN )
2180 if leaderN == main.FALSE:
2181 # error in response
2182 main.log.error( "Something is wrong with " +
2183 "electionTestLeader function, check the" +
2184 " error logs" )
2185 leaderResult = main.FALSE
2186 elif leaderN is None:
2187 main.log.error( cli.name +
2188 " shows no leader for the election-app was" +
2189 " elected after the old one died" )
2190 leaderResult = main.FALSE
2191 elif leaderN in restarted:
2192 main.log.error( cli.name + " shows " + str( leaderN ) +
2193 " as leader for the election-app, but it " +
2194 "was restarted" )
2195 leaderResult = main.FALSE
2196 if len( set( leaderList ) ) != 1:
2197 leaderResult = main.FALSE
2198 main.log.error(
2199 "Inconsistent view of leader for the election test app" )
2200 # TODO: print the list
2201 utilities.assert_equals(
2202 expect=main.TRUE,
2203 actual=leaderResult,
2204 onpass="Leadership election passed",
2205 onfail="Something went wrong with Leadership election" )
2206
2207 def CASE8( self, main ):
2208 """
2209 Compare topo
2210 """
2211 import json
2212 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002213 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002214 assert main, "main not defined"
2215 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002216 assert main.CLIs, "main.CLIs not defined"
2217 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002218
2219 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002220 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002221 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002222 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002223 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002224 elapsed = 0
2225 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002226 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002227 startTime = time.time()
2228 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002229 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002230 devicesResults = main.TRUE
2231 linksResults = main.TRUE
2232 hostsResults = main.TRUE
2233 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002234 count += 1
2235 cliStart = time.time()
2236 devices = []
2237 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002238 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002239 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002240 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002241 args=[ main.CLIs[i].devices, [ None ] ],
2242 kwargs= { 'sleep': 5, 'attempts': 5,
2243 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002244 threads.append( t )
2245 t.start()
2246
2247 for t in threads:
2248 t.join()
2249 devices.append( t.result )
2250 hosts = []
2251 ipResult = main.TRUE
2252 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002253 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002254 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002255 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002256 args=[ main.CLIs[i].hosts, [ None ] ],
2257 kwargs= { 'sleep': 5, 'attempts': 5,
2258 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002259 threads.append( t )
2260 t.start()
2261
2262 for t in threads:
2263 t.join()
2264 try:
2265 hosts.append( json.loads( t.result ) )
2266 except ( ValueError, TypeError ):
2267 main.log.exception( "Error parsing hosts results" )
2268 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002269 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002270 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002271 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002272 if hosts[ controller ]:
2273 for host in hosts[ controller ]:
2274 if host is None or host.get( 'ipAddresses', [] ) == []:
2275 main.log.error(
2276 "Error with host ipAddresses on controller" +
2277 controllerStr + ": " + str( host ) )
2278 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002279 ports = []
2280 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002281 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002282 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002283 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002284 args=[ main.CLIs[i].ports, [ None ] ],
2285 kwargs= { 'sleep': 5, 'attempts': 5,
2286 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002287 threads.append( t )
2288 t.start()
2289
2290 for t in threads:
2291 t.join()
2292 ports.append( t.result )
2293 links = []
2294 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002295 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002296 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002297 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002298 args=[ main.CLIs[i].links, [ None ] ],
2299 kwargs= { 'sleep': 5, 'attempts': 5,
2300 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002301 threads.append( t )
2302 t.start()
2303
2304 for t in threads:
2305 t.join()
2306 links.append( t.result )
2307 clusters = []
2308 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002309 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002310 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002311 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002312 args=[ main.CLIs[i].clusters, [ None ] ],
2313 kwargs= { 'sleep': 5, 'attempts': 5,
2314 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002315 threads.append( t )
2316 t.start()
2317
2318 for t in threads:
2319 t.join()
2320 clusters.append( t.result )
2321
2322 elapsed = time.time() - startTime
2323 cliTime = time.time() - cliStart
2324 print "Elapsed time: " + str( elapsed )
2325 print "CLI time: " + str( cliTime )
2326
Jon Hall6e709752016-02-01 13:38:46 -08002327 if all( e is None for e in devices ) and\
2328 all( e is None for e in hosts ) and\
2329 all( e is None for e in ports ) and\
2330 all( e is None for e in links ) and\
2331 all( e is None for e in clusters ):
2332 topoFailMsg = "Could not get topology from ONOS"
2333 main.log.error( topoFailMsg )
2334 continue # Try again, No use trying to compare
2335
Jon Hall5cf14d52015-07-16 12:15:19 -07002336 mnSwitches = main.Mininet1.getSwitches()
2337 mnLinks = main.Mininet1.getLinks()
2338 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002339 for controller in range( len( main.activeNodes ) ):
2340 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002341 if devices[ controller ] and ports[ controller ] and\
2342 "Error" not in devices[ controller ] and\
2343 "Error" not in ports[ controller ]:
2344
Jon Hallc6793552016-01-19 14:18:37 -08002345 try:
2346 currentDevicesResult = main.Mininet1.compareSwitches(
2347 mnSwitches,
2348 json.loads( devices[ controller ] ),
2349 json.loads( ports[ controller ] ) )
2350 except ( TypeError, ValueError ) as e:
2351 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2352 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002353 else:
2354 currentDevicesResult = main.FALSE
2355 utilities.assert_equals( expect=main.TRUE,
2356 actual=currentDevicesResult,
2357 onpass="ONOS" + controllerStr +
2358 " Switches view is correct",
2359 onfail="ONOS" + controllerStr +
2360 " Switches view is incorrect" )
2361
2362 if links[ controller ] and "Error" not in links[ controller ]:
2363 currentLinksResult = main.Mininet1.compareLinks(
2364 mnSwitches, mnLinks,
2365 json.loads( links[ controller ] ) )
2366 else:
2367 currentLinksResult = main.FALSE
2368 utilities.assert_equals( expect=main.TRUE,
2369 actual=currentLinksResult,
2370 onpass="ONOS" + controllerStr +
2371 " links view is correct",
2372 onfail="ONOS" + controllerStr +
2373 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002374 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002375 currentHostsResult = main.Mininet1.compareHosts(
2376 mnHosts,
2377 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002378 elif hosts[ controller ] == []:
2379 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002380 else:
2381 currentHostsResult = main.FALSE
2382 utilities.assert_equals( expect=main.TRUE,
2383 actual=currentHostsResult,
2384 onpass="ONOS" + controllerStr +
2385 " hosts exist in Mininet",
2386 onfail="ONOS" + controllerStr +
2387 " hosts don't match Mininet" )
2388 # CHECKING HOST ATTACHMENT POINTS
2389 hostAttachment = True
2390 zeroHosts = False
2391 # FIXME: topo-HA/obelisk specific mappings:
2392 # key is mac and value is dpid
2393 mappings = {}
2394 for i in range( 1, 29 ): # hosts 1 through 28
2395 # set up correct variables:
2396 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2397 if i == 1:
2398 deviceId = "1000".zfill(16)
2399 elif i == 2:
2400 deviceId = "2000".zfill(16)
2401 elif i == 3:
2402 deviceId = "3000".zfill(16)
2403 elif i == 4:
2404 deviceId = "3004".zfill(16)
2405 elif i == 5:
2406 deviceId = "5000".zfill(16)
2407 elif i == 6:
2408 deviceId = "6000".zfill(16)
2409 elif i == 7:
2410 deviceId = "6007".zfill(16)
2411 elif i >= 8 and i <= 17:
2412 dpid = '3' + str( i ).zfill( 3 )
2413 deviceId = dpid.zfill(16)
2414 elif i >= 18 and i <= 27:
2415 dpid = '6' + str( i ).zfill( 3 )
2416 deviceId = dpid.zfill(16)
2417 elif i == 28:
2418 deviceId = "2800".zfill(16)
2419 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002420 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002421 if hosts[ controller ] == []:
2422 main.log.warn( "There are no hosts discovered" )
2423 zeroHosts = True
2424 else:
2425 for host in hosts[ controller ]:
2426 mac = None
2427 location = None
2428 device = None
2429 port = None
2430 try:
2431 mac = host.get( 'mac' )
2432 assert mac, "mac field could not be found for this host object"
2433
2434 location = host.get( 'location' )
2435 assert location, "location field could not be found for this host object"
2436
2437 # Trim the protocol identifier off deviceId
2438 device = str( location.get( 'elementId' ) ).split(':')[1]
2439 assert device, "elementId field could not be found for this host location object"
2440
2441 port = location.get( 'port' )
2442 assert port, "port field could not be found for this host location object"
2443
2444 # Now check if this matches where they should be
2445 if mac and device and port:
2446 if str( port ) != "1":
2447 main.log.error( "The attachment port is incorrect for " +
2448 "host " + str( mac ) +
2449 ". Expected: 1 Actual: " + str( port) )
2450 hostAttachment = False
2451 if device != mappings[ str( mac ) ]:
2452 main.log.error( "The attachment device is incorrect for " +
2453 "host " + str( mac ) +
2454 ". Expected: " + mappings[ str( mac ) ] +
2455 " Actual: " + device )
2456 hostAttachment = False
2457 else:
2458 hostAttachment = False
2459 except AssertionError:
2460 main.log.exception( "Json object not as expected" )
2461 main.log.error( repr( host ) )
2462 hostAttachment = False
2463 else:
2464 main.log.error( "No hosts json output or \"Error\"" +
2465 " in output. hosts = " +
2466 repr( hosts[ controller ] ) )
2467 if zeroHosts is False:
2468 hostAttachment = True
2469
2470 # END CHECKING HOST ATTACHMENT POINTS
2471 devicesResults = devicesResults and currentDevicesResult
2472 linksResults = linksResults and currentLinksResult
2473 hostsResults = hostsResults and currentHostsResult
2474 hostAttachmentResults = hostAttachmentResults and\
2475 hostAttachment
Jon Halla440e872016-03-31 15:15:50 -07002476 topoResult = ( devicesResults and linksResults
2477 and hostsResults and ipResult and
2478 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002479 utilities.assert_equals( expect=True,
2480 actual=topoResult,
2481 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002482 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002483 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002484
2485 # Compare json objects for hosts and dataplane clusters
2486
2487 # hosts
2488 main.step( "Hosts view is consistent across all ONOS nodes" )
2489 consistentHostsResult = main.TRUE
2490 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002491 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002492 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002493 if hosts[ controller ] == hosts[ 0 ]:
2494 continue
2495 else: # hosts not consistent
2496 main.log.error( "hosts from ONOS" + controllerStr +
2497 " is inconsistent with ONOS1" )
2498 main.log.warn( repr( hosts[ controller ] ) )
2499 consistentHostsResult = main.FALSE
2500
2501 else:
2502 main.log.error( "Error in getting ONOS hosts from ONOS" +
2503 controllerStr )
2504 consistentHostsResult = main.FALSE
2505 main.log.warn( "ONOS" + controllerStr +
2506 " hosts response: " +
2507 repr( hosts[ controller ] ) )
2508 utilities.assert_equals(
2509 expect=main.TRUE,
2510 actual=consistentHostsResult,
2511 onpass="Hosts view is consistent across all ONOS nodes",
2512 onfail="ONOS nodes have different views of hosts" )
2513
2514 main.step( "Hosts information is correct" )
2515 hostsResults = hostsResults and ipResult
2516 utilities.assert_equals(
2517 expect=main.TRUE,
2518 actual=hostsResults,
2519 onpass="Host information is correct",
2520 onfail="Host information is incorrect" )
2521
2522 main.step( "Host attachment points to the network" )
2523 utilities.assert_equals(
2524 expect=True,
2525 actual=hostAttachmentResults,
2526 onpass="Hosts are correctly attached to the network",
2527 onfail="ONOS did not correctly attach hosts to the network" )
2528
2529 # Strongly connected clusters of devices
2530 main.step( "Clusters view is consistent across all ONOS nodes" )
2531 consistentClustersResult = main.TRUE
2532 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002533 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002534 if "Error" not in clusters[ controller ]:
2535 if clusters[ controller ] == clusters[ 0 ]:
2536 continue
2537 else: # clusters not consistent
2538 main.log.error( "clusters from ONOS" +
2539 controllerStr +
2540 " is inconsistent with ONOS1" )
2541 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002542 else:
2543 main.log.error( "Error in getting dataplane clusters " +
2544 "from ONOS" + controllerStr )
2545 consistentClustersResult = main.FALSE
2546 main.log.warn( "ONOS" + controllerStr +
2547 " clusters response: " +
2548 repr( clusters[ controller ] ) )
2549 utilities.assert_equals(
2550 expect=main.TRUE,
2551 actual=consistentClustersResult,
2552 onpass="Clusters view is consistent across all ONOS nodes",
2553 onfail="ONOS nodes have different views of clusters" )
2554
2555 main.step( "There is only one SCC" )
2556 # there should always only be one cluster
2557 try:
2558 numClusters = len( json.loads( clusters[ 0 ] ) )
2559 except ( ValueError, TypeError ):
2560 main.log.exception( "Error parsing clusters[0]: " +
2561 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002562 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002563 clusterResults = main.FALSE
2564 if numClusters == 1:
2565 clusterResults = main.TRUE
2566 utilities.assert_equals(
2567 expect=1,
2568 actual=numClusters,
2569 onpass="ONOS shows 1 SCC",
2570 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2571
2572 topoResult = ( devicesResults and linksResults
2573 and hostsResults and consistentHostsResult
2574 and consistentClustersResult and clusterResults
2575 and ipResult and hostAttachmentResults )
2576
2577 topoResult = topoResult and int( count <= 2 )
2578 note = "note it takes about " + str( int( cliTime ) ) + \
2579 " seconds for the test to make all the cli calls to fetch " +\
2580 "the topology from each ONOS instance"
2581 main.log.info(
2582 "Very crass estimate for topology discovery/convergence( " +
2583 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2584 str( count ) + " tries" )
2585
2586 main.step( "Device information is correct" )
2587 utilities.assert_equals(
2588 expect=main.TRUE,
2589 actual=devicesResults,
2590 onpass="Device information is correct",
2591 onfail="Device information is incorrect" )
2592
2593 main.step( "Links are correct" )
2594 utilities.assert_equals(
2595 expect=main.TRUE,
2596 actual=linksResults,
2597 onpass="Link are correct",
2598 onfail="Links are incorrect" )
2599
Jon Halla440e872016-03-31 15:15:50 -07002600 main.step( "Hosts are correct" )
2601 utilities.assert_equals(
2602 expect=main.TRUE,
2603 actual=hostsResults,
2604 onpass="Hosts are correct",
2605 onfail="Hosts are incorrect" )
2606
Jon Hall5cf14d52015-07-16 12:15:19 -07002607 # FIXME: move this to an ONOS state case
2608 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002609 nodeResults = utilities.retry( main.HA.nodesCheck,
2610 False,
2611 args=[main.activeNodes],
2612 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002613
Jon Hall41d39f12016-04-11 22:54:35 -07002614 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002615 onpass="Nodes check successful",
2616 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002617 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002618 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002619 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002620 main.CLIs[i].name,
2621 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002622
2623 def CASE9( self, main ):
2624 """
2625 Link s3-s28 down
2626 """
2627 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002628 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002629 assert main, "main not defined"
2630 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002631 assert main.CLIs, "main.CLIs not defined"
2632 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002633 # NOTE: You should probably run a topology check after this
2634
2635 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2636
2637 description = "Turn off a link to ensure that Link Discovery " +\
2638 "is working properly"
2639 main.case( description )
2640
2641 main.step( "Kill Link between s3 and s28" )
2642 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2643 main.log.info( "Waiting " + str( linkSleep ) +
2644 " seconds for link down to be discovered" )
2645 time.sleep( linkSleep )
2646 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2647 onpass="Link down successful",
2648 onfail="Failed to bring link down" )
2649 # TODO do some sort of check here
2650
2651 def CASE10( self, main ):
2652 """
2653 Link s3-s28 up
2654 """
2655 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002656 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002657 assert main, "main not defined"
2658 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002659 assert main.CLIs, "main.CLIs not defined"
2660 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002661 # NOTE: You should probably run a topology check after this
2662
2663 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2664
2665 description = "Restore a link to ensure that Link Discovery is " + \
2666 "working properly"
2667 main.case( description )
2668
2669 main.step( "Bring link between s3 and s28 back up" )
2670 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2671 main.log.info( "Waiting " + str( linkSleep ) +
2672 " seconds for link up to be discovered" )
2673 time.sleep( linkSleep )
2674 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2675 onpass="Link up successful",
2676 onfail="Failed to bring link up" )
2677 # TODO do some sort of check here
2678
2679 def CASE11( self, main ):
2680 """
2681 Switch Down
2682 """
2683 # NOTE: You should probably run a topology check after this
2684 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002685 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002686 assert main, "main not defined"
2687 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002688 assert main.CLIs, "main.CLIs not defined"
2689 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002690
2691 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2692
2693 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002694 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002695 main.case( description )
2696 switch = main.params[ 'kill' ][ 'switch' ]
2697 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2698
2699 # TODO: Make this switch parameterizable
2700 main.step( "Kill " + switch )
2701 main.log.info( "Deleting " + switch )
2702 main.Mininet1.delSwitch( switch )
2703 main.log.info( "Waiting " + str( switchSleep ) +
2704 " seconds for switch down to be discovered" )
2705 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002706 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002707 # Peek at the deleted switch
2708 main.log.warn( str( device ) )
2709 result = main.FALSE
2710 if device and device[ 'available' ] is False:
2711 result = main.TRUE
2712 utilities.assert_equals( expect=main.TRUE, actual=result,
2713 onpass="Kill switch successful",
2714 onfail="Failed to kill switch?" )
2715
2716 def CASE12( self, main ):
2717 """
2718 Switch Up
2719 """
2720 # NOTE: You should probably run a topology check after this
2721 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002722 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002723 assert main, "main not defined"
2724 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002725 assert main.CLIs, "main.CLIs not defined"
2726 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002727 assert ONOS1Port, "ONOS1Port not defined"
2728 assert ONOS2Port, "ONOS2Port not defined"
2729 assert ONOS3Port, "ONOS3Port not defined"
2730 assert ONOS4Port, "ONOS4Port not defined"
2731 assert ONOS5Port, "ONOS5Port not defined"
2732 assert ONOS6Port, "ONOS6Port not defined"
2733 assert ONOS7Port, "ONOS7Port not defined"
2734
2735 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2736 switch = main.params[ 'kill' ][ 'switch' ]
2737 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2738 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002739 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002740 description = "Adding a switch to ensure it is discovered correctly"
2741 main.case( description )
2742
2743 main.step( "Add back " + switch )
2744 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2745 for peer in links:
2746 main.Mininet1.addLink( switch, peer )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002747 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002748 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2749 main.log.info( "Waiting " + str( switchSleep ) +
2750 " seconds for switch up to be discovered" )
2751 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002752 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002753 # Peek at the deleted switch
2754 main.log.warn( str( device ) )
2755 result = main.FALSE
2756 if device and device[ 'available' ]:
2757 result = main.TRUE
2758 utilities.assert_equals( expect=main.TRUE, actual=result,
2759 onpass="add switch successful",
2760 onfail="Failed to add switch?" )
2761
2762 def CASE13( self, main ):
2763 """
2764 Clean up
2765 """
2766 import os
2767 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002768 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002769 assert main, "main not defined"
2770 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002771 assert main.CLIs, "main.CLIs not defined"
2772 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002773
2774 # printing colors to terminal
2775 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2776 'blue': '\033[94m', 'green': '\033[92m',
2777 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2778 main.case( "Test Cleanup" )
2779 main.step( "Killing tcpdumps" )
2780 main.Mininet2.stopTcpdump()
2781
2782 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002783 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002784 main.step( "Copying MN pcap and ONOS log files to test station" )
2785 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2786 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002787 # NOTE: MN Pcap file is being saved to logdir.
2788 # We scp this file as MN and TestON aren't necessarily the same vm
2789
2790 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002791 # TODO: Load these from params
2792 # NOTE: must end in /
2793 logFolder = "/opt/onos/log/"
2794 logFiles = [ "karaf.log", "karaf.log.1" ]
2795 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002796 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002797 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002798 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002799 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2800 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002801 # std*.log's
2802 # NOTE: must end in /
2803 logFolder = "/opt/onos/var/"
2804 logFiles = [ "stderr.log", "stdout.log" ]
2805 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002806 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002807 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002808 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002809 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2810 logFolder + f, dstName )
2811 else:
2812 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002813
2814 main.step( "Stopping Mininet" )
2815 mnResult = main.Mininet1.stopNet()
2816 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2817 onpass="Mininet stopped",
2818 onfail="MN cleanup NOT successful" )
2819
2820 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002821 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002822 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2823 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002824
2825 try:
2826 timerLog = open( main.logdir + "/Timers.csv", 'w')
2827 # Overwrite with empty line and close
2828 labels = "Gossip Intents, Restart"
2829 data = str( gossipTime ) + ", " + str( main.restartTime )
2830 timerLog.write( labels + "\n" + data )
2831 timerLog.close()
2832 except NameError, e:
2833 main.log.exception(e)
2834
2835 def CASE14( self, main ):
2836 """
2837 start election app on all onos nodes
2838 """
Jon Halle1a3b752015-07-22 13:02:46 -07002839 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002840 assert main, "main not defined"
2841 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002842 assert main.CLIs, "main.CLIs not defined"
2843 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002844
2845 main.case("Start Leadership Election app")
2846 main.step( "Install leadership election app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002847 onosCli = main.CLIs[ main.activeNodes[0] ]
2848 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002849 utilities.assert_equals(
2850 expect=main.TRUE,
2851 actual=appResult,
2852 onpass="Election app installed",
2853 onfail="Something went wrong with installing Leadership election" )
2854
2855 main.step( "Run for election on each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002856 for i in main.activeNodes:
2857 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002858 time.sleep(5)
2859 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2860 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002861 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002862 expect=True,
2863 actual=sameResult,
2864 onpass="All nodes see the same leaderboards",
2865 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002866
Jon Hall25463a82016-04-13 14:03:52 -07002867 if sameResult:
2868 leader = leaders[ 0 ][ 0 ]
2869 if main.nodes[main.activeNodes[0]].ip_address in leader:
2870 correctLeader = True
2871 else:
2872 correctLeader = False
2873 main.step( "First node was elected leader" )
2874 utilities.assert_equals(
2875 expect=True,
2876 actual=correctLeader,
2877 onpass="Correct leader was elected",
2878 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002879
2880 def CASE15( self, main ):
2881 """
2882 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002883 15.1 Run election on each node
2884 15.2 Check that each node has the same leaders and candidates
2885 15.3 Find current leader and withdraw
2886 15.4 Check that a new node was elected leader
2887 15.5 Check that that new leader was the candidate of old leader
2888 15.6 Run for election on old leader
2889 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2890 15.8 Make sure that the old leader was added to the candidate list
2891
2892 old and new variable prefixes refer to data from before vs after
2893 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002894 """
2895 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002896 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002897 assert main, "main not defined"
2898 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002899 assert main.CLIs, "main.CLIs not defined"
2900 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002901
Jon Hall5cf14d52015-07-16 12:15:19 -07002902 description = "Check that Leadership Election is still functional"
2903 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002904 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002905
Jon Halla440e872016-03-31 15:15:50 -07002906 oldLeaders = [] # list of lists of each nodes' candidates before
2907 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002908 oldLeader = '' # the old leader from oldLeaders, None if not same
2909 newLeader = '' # the new leaders fron newLoeaders, None if not same
2910 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2911 expectNoLeader = False # True when there is only one leader
2912 if main.numCtrls == 1:
2913 expectNoLeader = True
2914
2915 main.step( "Run for election on each node" )
2916 electionResult = main.TRUE
2917
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002918 for i in main.activeNodes: # run test election on each node
2919 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002920 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002921 utilities.assert_equals(
2922 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002923 actual=electionResult,
2924 onpass="All nodes successfully ran for leadership",
2925 onfail="At least one node failed to run for leadership" )
2926
acsmars3a72bde2015-09-02 14:16:22 -07002927 if electionResult == main.FALSE:
2928 main.log.error(
2929 "Skipping Test Case because Election Test App isn't loaded" )
2930 main.skipCase()
2931
acsmars71adceb2015-08-31 15:09:26 -07002932 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002933 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002934 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002935 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002936 if sameResult:
2937 oldLeader = oldLeaders[ 0 ][ 0 ]
2938 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07002939 else:
Jon Halla440e872016-03-31 15:15:50 -07002940 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002941 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002942 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002943 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002944 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002945 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002946
2947 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002948 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002949 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002950 if oldLeader is None:
2951 main.log.error( "Leadership isn't consistent." )
2952 withdrawResult = main.FALSE
2953 # Get the CLI of the oldLeader
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002954 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002955 if oldLeader == main.nodes[ i ].ip_address:
2956 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002957 break
2958 else: # FOR/ELSE statement
2959 main.log.error( "Leader election, could not find current leader" )
2960 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002961 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002962 utilities.assert_equals(
2963 expect=main.TRUE,
2964 actual=withdrawResult,
2965 onpass="Node was withdrawn from election",
2966 onfail="Node was not withdrawn from election" )
2967
acsmars71adceb2015-08-31 15:09:26 -07002968 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002969 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07002970 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002971 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002972 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002973 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002974 if newLeaders[ 0 ][ 0 ] == 'none':
2975 main.log.error( "No leader was elected on at least 1 node" )
2976 if not expectNoLeader:
2977 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002978 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002979
2980 # Check that the new leader is not the older leader, which was withdrawn
2981 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002982 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08002983 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07002984 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002985 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002986 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002987 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002988 onpass="Leadership election passed",
2989 onfail="Something went wrong with Leadership election" )
2990
Jon Halla440e872016-03-31 15:15:50 -07002991 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08002992 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07002993 correctCandidateResult = main.TRUE
2994 if expectNoLeader:
2995 if newLeader == 'none':
2996 main.log.info( "No leader expected. None found. Pass" )
2997 correctCandidateResult = main.TRUE
2998 else:
2999 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3000 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003001 elif len( oldLeaders[0] ) >= 3:
3002 if newLeader == oldLeaders[ 0 ][ 2 ]:
3003 # correct leader was elected
3004 correctCandidateResult = main.TRUE
3005 else:
3006 correctCandidateResult = main.FALSE
3007 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3008 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003009 else:
3010 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003011 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003012 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07003013 utilities.assert_equals(
3014 expect=main.TRUE,
3015 actual=correctCandidateResult,
3016 onpass="Correct Candidate Elected",
3017 onfail="Incorrect Candidate Elected" )
3018
Jon Hall5cf14d52015-07-16 12:15:19 -07003019 main.step( "Run for election on old leader( just so everyone " +
3020 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07003021 if oldLeaderCLI is not None:
3022 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003023 else:
acsmars71adceb2015-08-31 15:09:26 -07003024 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003025 runResult = main.FALSE
3026 utilities.assert_equals(
3027 expect=main.TRUE,
3028 actual=runResult,
3029 onpass="App re-ran for election",
3030 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003031
acsmars71adceb2015-08-31 15:09:26 -07003032 main.step(
3033 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003034 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003035 # Get new leaders and candidates
3036 reRunLeaders = []
3037 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003038 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003039
3040 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07003041 if not reRunLeaders[0]:
3042 positionResult = main.FALSE
3043 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07003044 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3045 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07003046 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003047 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003048 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003049 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003050 onpass="Old leader successfully re-ran for election",
3051 onfail="Something went wrong with Leadership election after " +
3052 "the old leader re-ran for election" )
3053
3054 def CASE16( self, main ):
3055 """
3056 Install Distributed Primitives app
3057 """
3058 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003059 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003060 assert main, "main not defined"
3061 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003062 assert main.CLIs, "main.CLIs not defined"
3063 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003064
3065 # Variables for the distributed primitives tests
3066 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003067 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003068 global onosSet
3069 global onosSetName
3070 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003071 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003072 onosSet = set([])
3073 onosSetName = "TestON-set"
3074
3075 description = "Install Primitives app"
3076 main.case( description )
3077 main.step( "Install Primitives app" )
3078 appName = "org.onosproject.distributedprimitives"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003079 node = main.activeNodes[0]
3080 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003081 utilities.assert_equals( expect=main.TRUE,
3082 actual=appResults,
3083 onpass="Primitives app activated",
3084 onfail="Primitives app not activated" )
3085 time.sleep( 5 ) # To allow all nodes to activate
3086
3087 def CASE17( self, main ):
3088 """
3089 Check for basic functionality with distributed primitives
3090 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003091 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003092 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003093 assert main, "main not defined"
3094 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003095 assert main.CLIs, "main.CLIs not defined"
3096 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003097 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003098 assert onosSetName, "onosSetName not defined"
3099 # NOTE: assert fails if value is 0/None/Empty/False
3100 try:
3101 pCounterValue
3102 except NameError:
3103 main.log.error( "pCounterValue not defined, setting to 0" )
3104 pCounterValue = 0
3105 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003106 onosSet
3107 except NameError:
3108 main.log.error( "onosSet not defined, setting to empty Set" )
3109 onosSet = set([])
3110 # Variables for the distributed primitives tests. These are local only
3111 addValue = "a"
3112 addAllValue = "a b c d e f"
3113 retainValue = "c d e f"
3114
3115 description = "Check for basic functionality with distributed " +\
3116 "primitives"
3117 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003118 main.caseExplanation = "Test the methods of the distributed " +\
3119 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003120 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003121 # Partitioned counters
3122 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003123 pCounters = []
3124 threads = []
3125 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003126 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003127 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3128 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003129 args=[ pCounterName ] )
3130 pCounterValue += 1
3131 addedPValues.append( pCounterValue )
3132 threads.append( t )
3133 t.start()
3134
3135 for t in threads:
3136 t.join()
3137 pCounters.append( t.result )
3138 # Check that counter incremented numController times
3139 pCounterResults = True
3140 for i in addedPValues:
3141 tmpResult = i in pCounters
3142 pCounterResults = pCounterResults and tmpResult
3143 if not tmpResult:
3144 main.log.error( str( i ) + " is not in partitioned "
3145 "counter incremented results" )
3146 utilities.assert_equals( expect=True,
3147 actual=pCounterResults,
3148 onpass="Default counter incremented",
3149 onfail="Error incrementing default" +
3150 " counter" )
3151
Jon Halle1a3b752015-07-22 13:02:46 -07003152 main.step( "Get then Increment a default counter on each node" )
3153 pCounters = []
3154 threads = []
3155 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003156 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003157 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3158 name="counterGetAndAdd-" + str( i ),
3159 args=[ pCounterName ] )
3160 addedPValues.append( pCounterValue )
3161 pCounterValue += 1
3162 threads.append( t )
3163 t.start()
3164
3165 for t in threads:
3166 t.join()
3167 pCounters.append( t.result )
3168 # Check that counter incremented numController times
3169 pCounterResults = True
3170 for i in addedPValues:
3171 tmpResult = i in pCounters
3172 pCounterResults = pCounterResults and tmpResult
3173 if not tmpResult:
3174 main.log.error( str( i ) + " is not in partitioned "
3175 "counter incremented results" )
3176 utilities.assert_equals( expect=True,
3177 actual=pCounterResults,
3178 onpass="Default counter incremented",
3179 onfail="Error incrementing default" +
3180 " counter" )
3181
3182 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003183 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003184 utilities.assert_equals( expect=main.TRUE,
3185 actual=incrementCheck,
3186 onpass="Added counters are correct",
3187 onfail="Added counters are incorrect" )
3188
3189 main.step( "Add -8 to then get a default counter on each node" )
3190 pCounters = []
3191 threads = []
3192 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003193 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003194 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3195 name="counterIncrement-" + str( i ),
3196 args=[ pCounterName ],
3197 kwargs={ "delta": -8 } )
3198 pCounterValue += -8
3199 addedPValues.append( pCounterValue )
3200 threads.append( t )
3201 t.start()
3202
3203 for t in threads:
3204 t.join()
3205 pCounters.append( t.result )
3206 # Check that counter incremented numController times
3207 pCounterResults = True
3208 for i in addedPValues:
3209 tmpResult = i in pCounters
3210 pCounterResults = pCounterResults and tmpResult
3211 if not tmpResult:
3212 main.log.error( str( i ) + " is not in partitioned "
3213 "counter incremented results" )
3214 utilities.assert_equals( expect=True,
3215 actual=pCounterResults,
3216 onpass="Default counter incremented",
3217 onfail="Error incrementing default" +
3218 " counter" )
3219
3220 main.step( "Add 5 to then get a default counter on each node" )
3221 pCounters = []
3222 threads = []
3223 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003224 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003225 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3226 name="counterIncrement-" + str( i ),
3227 args=[ pCounterName ],
3228 kwargs={ "delta": 5 } )
3229 pCounterValue += 5
3230 addedPValues.append( pCounterValue )
3231 threads.append( t )
3232 t.start()
3233
3234 for t in threads:
3235 t.join()
3236 pCounters.append( t.result )
3237 # Check that counter incremented numController times
3238 pCounterResults = True
3239 for i in addedPValues:
3240 tmpResult = i in pCounters
3241 pCounterResults = pCounterResults and tmpResult
3242 if not tmpResult:
3243 main.log.error( str( i ) + " is not in partitioned "
3244 "counter incremented results" )
3245 utilities.assert_equals( expect=True,
3246 actual=pCounterResults,
3247 onpass="Default counter incremented",
3248 onfail="Error incrementing default" +
3249 " counter" )
3250
3251 main.step( "Get then add 5 to a default counter on each node" )
3252 pCounters = []
3253 threads = []
3254 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003255 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003256 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3257 name="counterIncrement-" + str( i ),
3258 args=[ pCounterName ],
3259 kwargs={ "delta": 5 } )
3260 addedPValues.append( pCounterValue )
3261 pCounterValue += 5
3262 threads.append( t )
3263 t.start()
3264
3265 for t in threads:
3266 t.join()
3267 pCounters.append( t.result )
3268 # Check that counter incremented numController times
3269 pCounterResults = True
3270 for i in addedPValues:
3271 tmpResult = i in pCounters
3272 pCounterResults = pCounterResults and tmpResult
3273 if not tmpResult:
3274 main.log.error( str( i ) + " is not in partitioned "
3275 "counter incremented results" )
3276 utilities.assert_equals( expect=True,
3277 actual=pCounterResults,
3278 onpass="Default counter incremented",
3279 onfail="Error incrementing default" +
3280 " counter" )
3281
3282 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003283 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003284 utilities.assert_equals( expect=main.TRUE,
3285 actual=incrementCheck,
3286 onpass="Added counters are correct",
3287 onfail="Added counters are incorrect" )
3288
Jon Hall5cf14d52015-07-16 12:15:19 -07003289 # DISTRIBUTED SETS
3290 main.step( "Distributed Set get" )
3291 size = len( onosSet )
3292 getResponses = []
3293 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003294 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003295 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003296 name="setTestGet-" + str( i ),
3297 args=[ onosSetName ] )
3298 threads.append( t )
3299 t.start()
3300 for t in threads:
3301 t.join()
3302 getResponses.append( t.result )
3303
3304 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003305 for i in range( len( main.activeNodes ) ):
3306 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003307 if isinstance( getResponses[ i ], list):
3308 current = set( getResponses[ i ] )
3309 if len( current ) == len( getResponses[ i ] ):
3310 # no repeats
3311 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003312 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003313 " has incorrect view" +
3314 " of set " + onosSetName + ":\n" +
3315 str( getResponses[ i ] ) )
3316 main.log.debug( "Expected: " + str( onosSet ) )
3317 main.log.debug( "Actual: " + str( current ) )
3318 getResults = main.FALSE
3319 else:
3320 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003321 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003322 " has repeat elements in" +
3323 " set " + onosSetName + ":\n" +
3324 str( getResponses[ i ] ) )
3325 getResults = main.FALSE
3326 elif getResponses[ i ] == main.ERROR:
3327 getResults = main.FALSE
3328 utilities.assert_equals( expect=main.TRUE,
3329 actual=getResults,
3330 onpass="Set elements are correct",
3331 onfail="Set elements are incorrect" )
3332
3333 main.step( "Distributed Set size" )
3334 sizeResponses = []
3335 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003336 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003337 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003338 name="setTestSize-" + str( i ),
3339 args=[ onosSetName ] )
3340 threads.append( t )
3341 t.start()
3342 for t in threads:
3343 t.join()
3344 sizeResponses.append( t.result )
3345
3346 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003347 for i in range( len( main.activeNodes ) ):
3348 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003349 if size != sizeResponses[ i ]:
3350 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003351 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003352 " expected a size of " + str( size ) +
3353 " for set " + onosSetName +
3354 " but got " + str( sizeResponses[ i ] ) )
3355 utilities.assert_equals( expect=main.TRUE,
3356 actual=sizeResults,
3357 onpass="Set sizes are correct",
3358 onfail="Set sizes are incorrect" )
3359
3360 main.step( "Distributed Set add()" )
3361 onosSet.add( addValue )
3362 addResponses = []
3363 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003364 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003365 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003366 name="setTestAdd-" + str( i ),
3367 args=[ onosSetName, addValue ] )
3368 threads.append( t )
3369 t.start()
3370 for t in threads:
3371 t.join()
3372 addResponses.append( t.result )
3373
3374 # main.TRUE = successfully changed the set
3375 # main.FALSE = action resulted in no change in set
3376 # main.ERROR - Some error in executing the function
3377 addResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003378 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003379 if addResponses[ i ] == main.TRUE:
3380 # All is well
3381 pass
3382 elif addResponses[ i ] == main.FALSE:
3383 # Already in set, probably fine
3384 pass
3385 elif addResponses[ i ] == main.ERROR:
3386 # Error in execution
3387 addResults = main.FALSE
3388 else:
3389 # unexpected result
3390 addResults = main.FALSE
3391 if addResults != main.TRUE:
3392 main.log.error( "Error executing set add" )
3393
3394 # Check if set is still correct
3395 size = len( onosSet )
3396 getResponses = []
3397 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003398 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003399 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003400 name="setTestGet-" + str( i ),
3401 args=[ onosSetName ] )
3402 threads.append( t )
3403 t.start()
3404 for t in threads:
3405 t.join()
3406 getResponses.append( t.result )
3407 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003408 for i in range( len( main.activeNodes ) ):
3409 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003410 if isinstance( getResponses[ i ], list):
3411 current = set( getResponses[ i ] )
3412 if len( current ) == len( getResponses[ i ] ):
3413 # no repeats
3414 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003415 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003416 " of set " + onosSetName + ":\n" +
3417 str( getResponses[ i ] ) )
3418 main.log.debug( "Expected: " + str( onosSet ) )
3419 main.log.debug( "Actual: " + str( current ) )
3420 getResults = main.FALSE
3421 else:
3422 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003423 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003424 " set " + onosSetName + ":\n" +
3425 str( getResponses[ i ] ) )
3426 getResults = main.FALSE
3427 elif getResponses[ i ] == main.ERROR:
3428 getResults = main.FALSE
3429 sizeResponses = []
3430 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003431 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003432 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003433 name="setTestSize-" + str( i ),
3434 args=[ onosSetName ] )
3435 threads.append( t )
3436 t.start()
3437 for t in threads:
3438 t.join()
3439 sizeResponses.append( t.result )
3440 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003441 for i in range( len( main.activeNodes ) ):
3442 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003443 if size != sizeResponses[ i ]:
3444 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003445 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003446 " expected a size of " + str( size ) +
3447 " for set " + onosSetName +
3448 " but got " + str( sizeResponses[ i ] ) )
3449 addResults = addResults and getResults and sizeResults
3450 utilities.assert_equals( expect=main.TRUE,
3451 actual=addResults,
3452 onpass="Set add correct",
3453 onfail="Set add was incorrect" )
3454
3455 main.step( "Distributed Set addAll()" )
3456 onosSet.update( addAllValue.split() )
3457 addResponses = []
3458 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003459 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003460 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003461 name="setTestAddAll-" + str( i ),
3462 args=[ onosSetName, addAllValue ] )
3463 threads.append( t )
3464 t.start()
3465 for t in threads:
3466 t.join()
3467 addResponses.append( t.result )
3468
3469 # main.TRUE = successfully changed the set
3470 # main.FALSE = action resulted in no change in set
3471 # main.ERROR - Some error in executing the function
3472 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003473 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003474 if addResponses[ i ] == main.TRUE:
3475 # All is well
3476 pass
3477 elif addResponses[ i ] == main.FALSE:
3478 # Already in set, probably fine
3479 pass
3480 elif addResponses[ i ] == main.ERROR:
3481 # Error in execution
3482 addAllResults = main.FALSE
3483 else:
3484 # unexpected result
3485 addAllResults = main.FALSE
3486 if addAllResults != main.TRUE:
3487 main.log.error( "Error executing set addAll" )
3488
3489 # Check if set is still correct
3490 size = len( onosSet )
3491 getResponses = []
3492 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003493 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003494 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003495 name="setTestGet-" + str( i ),
3496 args=[ onosSetName ] )
3497 threads.append( t )
3498 t.start()
3499 for t in threads:
3500 t.join()
3501 getResponses.append( t.result )
3502 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003503 for i in range( len( main.activeNodes ) ):
3504 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003505 if isinstance( getResponses[ i ], list):
3506 current = set( getResponses[ i ] )
3507 if len( current ) == len( getResponses[ i ] ):
3508 # no repeats
3509 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003510 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003511 " has incorrect view" +
3512 " of set " + onosSetName + ":\n" +
3513 str( getResponses[ i ] ) )
3514 main.log.debug( "Expected: " + str( onosSet ) )
3515 main.log.debug( "Actual: " + str( current ) )
3516 getResults = main.FALSE
3517 else:
3518 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003519 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003520 " has repeat elements in" +
3521 " set " + onosSetName + ":\n" +
3522 str( getResponses[ i ] ) )
3523 getResults = main.FALSE
3524 elif getResponses[ i ] == main.ERROR:
3525 getResults = main.FALSE
3526 sizeResponses = []
3527 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003528 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003529 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003530 name="setTestSize-" + str( i ),
3531 args=[ onosSetName ] )
3532 threads.append( t )
3533 t.start()
3534 for t in threads:
3535 t.join()
3536 sizeResponses.append( t.result )
3537 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003538 for i in range( len( main.activeNodes ) ):
3539 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003540 if size != sizeResponses[ i ]:
3541 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003542 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003543 " expected a size of " + str( size ) +
3544 " for set " + onosSetName +
3545 " but got " + str( sizeResponses[ i ] ) )
3546 addAllResults = addAllResults and getResults and sizeResults
3547 utilities.assert_equals( expect=main.TRUE,
3548 actual=addAllResults,
3549 onpass="Set addAll correct",
3550 onfail="Set addAll was incorrect" )
3551
3552 main.step( "Distributed Set contains()" )
3553 containsResponses = []
3554 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003555 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003556 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003557 name="setContains-" + str( i ),
3558 args=[ onosSetName ],
3559 kwargs={ "values": addValue } )
3560 threads.append( t )
3561 t.start()
3562 for t in threads:
3563 t.join()
3564 # NOTE: This is the tuple
3565 containsResponses.append( t.result )
3566
3567 containsResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003568 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003569 if containsResponses[ i ] == main.ERROR:
3570 containsResults = main.FALSE
3571 else:
3572 containsResults = containsResults and\
3573 containsResponses[ i ][ 1 ]
3574 utilities.assert_equals( expect=main.TRUE,
3575 actual=containsResults,
3576 onpass="Set contains is functional",
3577 onfail="Set contains failed" )
3578
3579 main.step( "Distributed Set containsAll()" )
3580 containsAllResponses = []
3581 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003582 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003583 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003584 name="setContainsAll-" + str( i ),
3585 args=[ onosSetName ],
3586 kwargs={ "values": addAllValue } )
3587 threads.append( t )
3588 t.start()
3589 for t in threads:
3590 t.join()
3591 # NOTE: This is the tuple
3592 containsAllResponses.append( t.result )
3593
3594 containsAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003595 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003596 if containsResponses[ i ] == main.ERROR:
3597 containsResults = main.FALSE
3598 else:
3599 containsResults = containsResults and\
3600 containsResponses[ i ][ 1 ]
3601 utilities.assert_equals( expect=main.TRUE,
3602 actual=containsAllResults,
3603 onpass="Set containsAll is functional",
3604 onfail="Set containsAll failed" )
3605
3606 main.step( "Distributed Set remove()" )
3607 onosSet.remove( addValue )
3608 removeResponses = []
3609 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003610 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003611 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003612 name="setTestRemove-" + str( i ),
3613 args=[ onosSetName, addValue ] )
3614 threads.append( t )
3615 t.start()
3616 for t in threads:
3617 t.join()
3618 removeResponses.append( t.result )
3619
3620 # main.TRUE = successfully changed the set
3621 # main.FALSE = action resulted in no change in set
3622 # main.ERROR - Some error in executing the function
3623 removeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003624 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003625 if removeResponses[ i ] == main.TRUE:
3626 # All is well
3627 pass
3628 elif removeResponses[ i ] == main.FALSE:
3629 # not in set, probably fine
3630 pass
3631 elif removeResponses[ i ] == main.ERROR:
3632 # Error in execution
3633 removeResults = main.FALSE
3634 else:
3635 # unexpected result
3636 removeResults = main.FALSE
3637 if removeResults != main.TRUE:
3638 main.log.error( "Error executing set remove" )
3639
3640 # Check if set is still correct
3641 size = len( onosSet )
3642 getResponses = []
3643 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003644 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003645 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003646 name="setTestGet-" + str( i ),
3647 args=[ onosSetName ] )
3648 threads.append( t )
3649 t.start()
3650 for t in threads:
3651 t.join()
3652 getResponses.append( t.result )
3653 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003654 for i in range( len( main.activeNodes ) ):
3655 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003656 if isinstance( getResponses[ i ], list):
3657 current = set( getResponses[ i ] )
3658 if len( current ) == len( getResponses[ i ] ):
3659 # no repeats
3660 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003661 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003662 " has incorrect view" +
3663 " of set " + onosSetName + ":\n" +
3664 str( getResponses[ i ] ) )
3665 main.log.debug( "Expected: " + str( onosSet ) )
3666 main.log.debug( "Actual: " + str( current ) )
3667 getResults = main.FALSE
3668 else:
3669 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003670 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003671 " has repeat elements in" +
3672 " set " + onosSetName + ":\n" +
3673 str( getResponses[ i ] ) )
3674 getResults = main.FALSE
3675 elif getResponses[ i ] == main.ERROR:
3676 getResults = main.FALSE
3677 sizeResponses = []
3678 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003679 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003680 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003681 name="setTestSize-" + str( i ),
3682 args=[ onosSetName ] )
3683 threads.append( t )
3684 t.start()
3685 for t in threads:
3686 t.join()
3687 sizeResponses.append( t.result )
3688 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003689 for i in range( len( main.activeNodes ) ):
3690 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003691 if size != sizeResponses[ i ]:
3692 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003693 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003694 " expected a size of " + str( size ) +
3695 " for set " + onosSetName +
3696 " but got " + str( sizeResponses[ i ] ) )
3697 removeResults = removeResults and getResults and sizeResults
3698 utilities.assert_equals( expect=main.TRUE,
3699 actual=removeResults,
3700 onpass="Set remove correct",
3701 onfail="Set remove was incorrect" )
3702
3703 main.step( "Distributed Set removeAll()" )
3704 onosSet.difference_update( addAllValue.split() )
3705 removeAllResponses = []
3706 threads = []
3707 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003708 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003709 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003710 name="setTestRemoveAll-" + str( i ),
3711 args=[ onosSetName, addAllValue ] )
3712 threads.append( t )
3713 t.start()
3714 for t in threads:
3715 t.join()
3716 removeAllResponses.append( t.result )
3717 except Exception, e:
3718 main.log.exception(e)
3719
3720 # main.TRUE = successfully changed the set
3721 # main.FALSE = action resulted in no change in set
3722 # main.ERROR - Some error in executing the function
3723 removeAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003724 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003725 if removeAllResponses[ i ] == main.TRUE:
3726 # All is well
3727 pass
3728 elif removeAllResponses[ i ] == main.FALSE:
3729 # not in set, probably fine
3730 pass
3731 elif removeAllResponses[ i ] == main.ERROR:
3732 # Error in execution
3733 removeAllResults = main.FALSE
3734 else:
3735 # unexpected result
3736 removeAllResults = main.FALSE
3737 if removeAllResults != main.TRUE:
3738 main.log.error( "Error executing set removeAll" )
3739
3740 # Check if set is still correct
3741 size = len( onosSet )
3742 getResponses = []
3743 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003744 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003745 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003746 name="setTestGet-" + str( i ),
3747 args=[ onosSetName ] )
3748 threads.append( t )
3749 t.start()
3750 for t in threads:
3751 t.join()
3752 getResponses.append( t.result )
3753 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003754 for i in range( len( main.activeNodes ) ):
3755 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003756 if isinstance( getResponses[ i ], list):
3757 current = set( getResponses[ i ] )
3758 if len( current ) == len( getResponses[ i ] ):
3759 # no repeats
3760 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003761 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003762 " has incorrect view" +
3763 " of set " + onosSetName + ":\n" +
3764 str( getResponses[ i ] ) )
3765 main.log.debug( "Expected: " + str( onosSet ) )
3766 main.log.debug( "Actual: " + str( current ) )
3767 getResults = main.FALSE
3768 else:
3769 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003770 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003771 " has repeat elements in" +
3772 " set " + onosSetName + ":\n" +
3773 str( getResponses[ i ] ) )
3774 getResults = main.FALSE
3775 elif getResponses[ i ] == main.ERROR:
3776 getResults = main.FALSE
3777 sizeResponses = []
3778 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003779 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003780 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003781 name="setTestSize-" + str( i ),
3782 args=[ onosSetName ] )
3783 threads.append( t )
3784 t.start()
3785 for t in threads:
3786 t.join()
3787 sizeResponses.append( t.result )
3788 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003789 for i in range( len( main.activeNodes ) ):
3790 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003791 if size != sizeResponses[ i ]:
3792 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003793 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003794 " expected a size of " + str( size ) +
3795 " for set " + onosSetName +
3796 " but got " + str( sizeResponses[ i ] ) )
3797 removeAllResults = removeAllResults and getResults and sizeResults
3798 utilities.assert_equals( expect=main.TRUE,
3799 actual=removeAllResults,
3800 onpass="Set removeAll correct",
3801 onfail="Set removeAll was incorrect" )
3802
3803 main.step( "Distributed Set addAll()" )
3804 onosSet.update( addAllValue.split() )
3805 addResponses = []
3806 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003807 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003808 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003809 name="setTestAddAll-" + str( i ),
3810 args=[ onosSetName, addAllValue ] )
3811 threads.append( t )
3812 t.start()
3813 for t in threads:
3814 t.join()
3815 addResponses.append( t.result )
3816
3817 # main.TRUE = successfully changed the set
3818 # main.FALSE = action resulted in no change in set
3819 # main.ERROR - Some error in executing the function
3820 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003821 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003822 if addResponses[ i ] == main.TRUE:
3823 # All is well
3824 pass
3825 elif addResponses[ i ] == main.FALSE:
3826 # Already in set, probably fine
3827 pass
3828 elif addResponses[ i ] == main.ERROR:
3829 # Error in execution
3830 addAllResults = main.FALSE
3831 else:
3832 # unexpected result
3833 addAllResults = main.FALSE
3834 if addAllResults != main.TRUE:
3835 main.log.error( "Error executing set addAll" )
3836
3837 # Check if set is still correct
3838 size = len( onosSet )
3839 getResponses = []
3840 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003841 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003842 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003843 name="setTestGet-" + str( i ),
3844 args=[ onosSetName ] )
3845 threads.append( t )
3846 t.start()
3847 for t in threads:
3848 t.join()
3849 getResponses.append( t.result )
3850 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003851 for i in range( len( main.activeNodes ) ):
3852 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003853 if isinstance( getResponses[ i ], list):
3854 current = set( getResponses[ i ] )
3855 if len( current ) == len( getResponses[ i ] ):
3856 # no repeats
3857 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003858 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003859 " has incorrect view" +
3860 " of set " + onosSetName + ":\n" +
3861 str( getResponses[ i ] ) )
3862 main.log.debug( "Expected: " + str( onosSet ) )
3863 main.log.debug( "Actual: " + str( current ) )
3864 getResults = main.FALSE
3865 else:
3866 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003867 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003868 " has repeat elements in" +
3869 " set " + onosSetName + ":\n" +
3870 str( getResponses[ i ] ) )
3871 getResults = main.FALSE
3872 elif getResponses[ i ] == main.ERROR:
3873 getResults = main.FALSE
3874 sizeResponses = []
3875 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003876 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003877 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003878 name="setTestSize-" + str( i ),
3879 args=[ onosSetName ] )
3880 threads.append( t )
3881 t.start()
3882 for t in threads:
3883 t.join()
3884 sizeResponses.append( t.result )
3885 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003886 for i in range( len( main.activeNodes ) ):
3887 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003888 if size != sizeResponses[ i ]:
3889 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003890 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003891 " expected a size of " + str( size ) +
3892 " for set " + onosSetName +
3893 " but got " + str( sizeResponses[ i ] ) )
3894 addAllResults = addAllResults and getResults and sizeResults
3895 utilities.assert_equals( expect=main.TRUE,
3896 actual=addAllResults,
3897 onpass="Set addAll correct",
3898 onfail="Set addAll was incorrect" )
3899
3900 main.step( "Distributed Set clear()" )
3901 onosSet.clear()
3902 clearResponses = []
3903 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003904 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003905 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003906 name="setTestClear-" + str( i ),
3907 args=[ onosSetName, " "], # Values doesn't matter
3908 kwargs={ "clear": True } )
3909 threads.append( t )
3910 t.start()
3911 for t in threads:
3912 t.join()
3913 clearResponses.append( t.result )
3914
3915 # main.TRUE = successfully changed the set
3916 # main.FALSE = action resulted in no change in set
3917 # main.ERROR - Some error in executing the function
3918 clearResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003919 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003920 if clearResponses[ i ] == main.TRUE:
3921 # All is well
3922 pass
3923 elif clearResponses[ i ] == main.FALSE:
3924 # Nothing set, probably fine
3925 pass
3926 elif clearResponses[ i ] == main.ERROR:
3927 # Error in execution
3928 clearResults = main.FALSE
3929 else:
3930 # unexpected result
3931 clearResults = main.FALSE
3932 if clearResults != main.TRUE:
3933 main.log.error( "Error executing set clear" )
3934
3935 # Check if set is still correct
3936 size = len( onosSet )
3937 getResponses = []
3938 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003939 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003940 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003941 name="setTestGet-" + str( i ),
3942 args=[ onosSetName ] )
3943 threads.append( t )
3944 t.start()
3945 for t in threads:
3946 t.join()
3947 getResponses.append( t.result )
3948 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003949 for i in range( len( main.activeNodes ) ):
3950 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003951 if isinstance( getResponses[ i ], list):
3952 current = set( getResponses[ i ] )
3953 if len( current ) == len( getResponses[ i ] ):
3954 # no repeats
3955 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003956 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003957 " has incorrect view" +
3958 " of set " + onosSetName + ":\n" +
3959 str( getResponses[ i ] ) )
3960 main.log.debug( "Expected: " + str( onosSet ) )
3961 main.log.debug( "Actual: " + str( current ) )
3962 getResults = main.FALSE
3963 else:
3964 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003965 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003966 " has repeat elements in" +
3967 " set " + onosSetName + ":\n" +
3968 str( getResponses[ i ] ) )
3969 getResults = main.FALSE
3970 elif getResponses[ i ] == main.ERROR:
3971 getResults = main.FALSE
3972 sizeResponses = []
3973 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003974 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003975 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003976 name="setTestSize-" + str( i ),
3977 args=[ onosSetName ] )
3978 threads.append( t )
3979 t.start()
3980 for t in threads:
3981 t.join()
3982 sizeResponses.append( t.result )
3983 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003984 for i in range( len( main.activeNodes ) ):
3985 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003986 if size != sizeResponses[ i ]:
3987 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003988 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003989 " expected a size of " + str( size ) +
3990 " for set " + onosSetName +
3991 " but got " + str( sizeResponses[ i ] ) )
3992 clearResults = clearResults and getResults and sizeResults
3993 utilities.assert_equals( expect=main.TRUE,
3994 actual=clearResults,
3995 onpass="Set clear correct",
3996 onfail="Set clear was incorrect" )
3997
3998 main.step( "Distributed Set addAll()" )
3999 onosSet.update( addAllValue.split() )
4000 addResponses = []
4001 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004002 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004003 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004004 name="setTestAddAll-" + str( i ),
4005 args=[ onosSetName, addAllValue ] )
4006 threads.append( t )
4007 t.start()
4008 for t in threads:
4009 t.join()
4010 addResponses.append( t.result )
4011
4012 # main.TRUE = successfully changed the set
4013 # main.FALSE = action resulted in no change in set
4014 # main.ERROR - Some error in executing the function
4015 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004016 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004017 if addResponses[ i ] == main.TRUE:
4018 # All is well
4019 pass
4020 elif addResponses[ i ] == main.FALSE:
4021 # Already in set, probably fine
4022 pass
4023 elif addResponses[ i ] == main.ERROR:
4024 # Error in execution
4025 addAllResults = main.FALSE
4026 else:
4027 # unexpected result
4028 addAllResults = main.FALSE
4029 if addAllResults != main.TRUE:
4030 main.log.error( "Error executing set addAll" )
4031
4032 # Check if set is still correct
4033 size = len( onosSet )
4034 getResponses = []
4035 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004036 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004037 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004038 name="setTestGet-" + str( i ),
4039 args=[ onosSetName ] )
4040 threads.append( t )
4041 t.start()
4042 for t in threads:
4043 t.join()
4044 getResponses.append( t.result )
4045 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004046 for i in range( len( main.activeNodes ) ):
4047 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004048 if isinstance( getResponses[ i ], list):
4049 current = set( getResponses[ i ] )
4050 if len( current ) == len( getResponses[ i ] ):
4051 # no repeats
4052 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004053 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004054 " has incorrect view" +
4055 " of set " + onosSetName + ":\n" +
4056 str( getResponses[ i ] ) )
4057 main.log.debug( "Expected: " + str( onosSet ) )
4058 main.log.debug( "Actual: " + str( current ) )
4059 getResults = main.FALSE
4060 else:
4061 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004062 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004063 " has repeat elements in" +
4064 " set " + onosSetName + ":\n" +
4065 str( getResponses[ i ] ) )
4066 getResults = main.FALSE
4067 elif getResponses[ i ] == main.ERROR:
4068 getResults = main.FALSE
4069 sizeResponses = []
4070 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004071 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004072 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004073 name="setTestSize-" + str( i ),
4074 args=[ onosSetName ] )
4075 threads.append( t )
4076 t.start()
4077 for t in threads:
4078 t.join()
4079 sizeResponses.append( t.result )
4080 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004081 for i in range( len( main.activeNodes ) ):
4082 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004083 if size != sizeResponses[ i ]:
4084 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004085 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004086 " expected a size of " + str( size ) +
4087 " for set " + onosSetName +
4088 " but got " + str( sizeResponses[ i ] ) )
4089 addAllResults = addAllResults and getResults and sizeResults
4090 utilities.assert_equals( expect=main.TRUE,
4091 actual=addAllResults,
4092 onpass="Set addAll correct",
4093 onfail="Set addAll was incorrect" )
4094
4095 main.step( "Distributed Set retain()" )
4096 onosSet.intersection_update( retainValue.split() )
4097 retainResponses = []
4098 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004099 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004100 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004101 name="setTestRetain-" + str( i ),
4102 args=[ onosSetName, retainValue ],
4103 kwargs={ "retain": True } )
4104 threads.append( t )
4105 t.start()
4106 for t in threads:
4107 t.join()
4108 retainResponses.append( t.result )
4109
4110 # main.TRUE = successfully changed the set
4111 # main.FALSE = action resulted in no change in set
4112 # main.ERROR - Some error in executing the function
4113 retainResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004114 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004115 if retainResponses[ i ] == main.TRUE:
4116 # All is well
4117 pass
4118 elif retainResponses[ i ] == main.FALSE:
4119 # Already in set, probably fine
4120 pass
4121 elif retainResponses[ i ] == main.ERROR:
4122 # Error in execution
4123 retainResults = main.FALSE
4124 else:
4125 # unexpected result
4126 retainResults = main.FALSE
4127 if retainResults != main.TRUE:
4128 main.log.error( "Error executing set retain" )
4129
4130 # Check if set is still correct
4131 size = len( onosSet )
4132 getResponses = []
4133 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004134 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004135 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004136 name="setTestGet-" + str( i ),
4137 args=[ onosSetName ] )
4138 threads.append( t )
4139 t.start()
4140 for t in threads:
4141 t.join()
4142 getResponses.append( t.result )
4143 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004144 for i in range( len( main.activeNodes ) ):
4145 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004146 if isinstance( getResponses[ i ], list):
4147 current = set( getResponses[ i ] )
4148 if len( current ) == len( getResponses[ i ] ):
4149 # no repeats
4150 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004151 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004152 " has incorrect view" +
4153 " of set " + onosSetName + ":\n" +
4154 str( getResponses[ i ] ) )
4155 main.log.debug( "Expected: " + str( onosSet ) )
4156 main.log.debug( "Actual: " + str( current ) )
4157 getResults = main.FALSE
4158 else:
4159 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004160 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004161 " has repeat elements in" +
4162 " set " + onosSetName + ":\n" +
4163 str( getResponses[ i ] ) )
4164 getResults = main.FALSE
4165 elif getResponses[ i ] == main.ERROR:
4166 getResults = main.FALSE
4167 sizeResponses = []
4168 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004169 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004170 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004171 name="setTestSize-" + str( i ),
4172 args=[ onosSetName ] )
4173 threads.append( t )
4174 t.start()
4175 for t in threads:
4176 t.join()
4177 sizeResponses.append( t.result )
4178 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004179 for i in range( len( main.activeNodes ) ):
4180 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004181 if size != sizeResponses[ i ]:
4182 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004183 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004184 str( size ) + " for set " + onosSetName +
4185 " but got " + str( sizeResponses[ i ] ) )
4186 retainResults = retainResults and getResults and sizeResults
4187 utilities.assert_equals( expect=main.TRUE,
4188 actual=retainResults,
4189 onpass="Set retain correct",
4190 onfail="Set retain was incorrect" )
4191
Jon Hall2a5002c2015-08-21 16:49:11 -07004192 # Transactional maps
4193 main.step( "Partitioned Transactional maps put" )
4194 tMapValue = "Testing"
4195 numKeys = 100
4196 putResult = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004197 node = main.activeNodes[0]
4198 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004199 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004200 for i in putResponses:
4201 if putResponses[ i ][ 'value' ] != tMapValue:
4202 putResult = False
4203 else:
4204 putResult = False
4205 if not putResult:
4206 main.log.debug( "Put response values: " + str( putResponses ) )
4207 utilities.assert_equals( expect=True,
4208 actual=putResult,
4209 onpass="Partitioned Transactional Map put successful",
4210 onfail="Partitioned Transactional Map put values are incorrect" )
4211
4212 main.step( "Partitioned Transactional maps get" )
Jon Hall9bfadd22016-05-11 14:48:07 -07004213 # FIXME: is this sleep needed?
4214 time.sleep( 5 )
4215
Jon Hall2a5002c2015-08-21 16:49:11 -07004216 getCheck = True
4217 for n in range( 1, numKeys + 1 ):
4218 getResponses = []
4219 threads = []
4220 valueCheck = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004221 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004222 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4223 name="TMap-get-" + str( i ),
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004224 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004225 threads.append( t )
4226 t.start()
4227 for t in threads:
4228 t.join()
4229 getResponses.append( t.result )
4230 for node in getResponses:
4231 if node != tMapValue:
4232 valueCheck = False
4233 if not valueCheck:
4234 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4235 main.log.warn( getResponses )
4236 getCheck = getCheck and valueCheck
4237 utilities.assert_equals( expect=True,
4238 actual=getCheck,
4239 onpass="Partitioned Transactional Map get values were correct",
4240 onfail="Partitioned Transactional Map values incorrect" )