blob: 0f33ec0edc8b647380acc001545848a4be2e5101 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
Jon Hallb3ed8ed2015-10-28 16:43:55 -070012CASE61: The Failure inducing case.
13CASE62: The Failure recovery case.
Jon Hall5cf14d52015-07-16 12:15:19 -070014CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
Jon Hallb3ed8ed2015-10-28 16:43:55 -070028class HAstopNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -070029
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall3b489db2015-10-05 14:38:37 -070052 import pexpect
Jon Hallf3d16e72015-12-16 17:45:08 -080053 import time
Jon Halla440e872016-03-31 15:15:50 -070054 import json
Jon Hallb3ed8ed2015-10-28 16:43:55 -070055 main.log.info( "ONOS HA test: Stop a minority of ONOS nodes - " +
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "initialization" )
57 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070058 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070059 "installing ONOS, starting Mininet and ONOS" +\
60 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070061
62 # load some variables from the params file
63 PULLCODE = False
64 if main.params[ 'Git' ] == 'True':
65 PULLCODE = True
66 gitBranch = main.params[ 'branch' ]
67 cellName = main.params[ 'ENV' ][ 'cellName' ]
68
Jon Halle1a3b752015-07-22 13:02:46 -070069 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070071 if main.ONOSbench.maxNodes < main.numCtrls:
72 main.numCtrls = int( main.ONOSbench.maxNodes )
73 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070074 global ONOS1Port
75 global ONOS2Port
76 global ONOS3Port
77 global ONOS4Port
78 global ONOS5Port
79 global ONOS6Port
80 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070081 # These are for csv plotting in jenkins
82 global labels
83 global data
84 labels = []
85 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070086
87 # FIXME: just get controller port from params?
88 # TODO: do we really need all these?
89 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
90 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
91 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
92 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
93 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
94 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
95 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
96
Jon Halle1a3b752015-07-22 13:02:46 -070097 try:
Jon Hall53c5e662016-04-13 16:06:56 -070098 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070099 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -0700100 except Exception as e:
101 main.log.exception( e )
102 main.cleanup()
103 main.exit()
104
105 main.CLIs = []
106 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700107 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700108 for i in range( 1, main.numCtrls + 1 ):
109 try:
110 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
111 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
112 ipList.append( main.nodes[ -1 ].ip_address )
113 except AttributeError:
114 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700115
116 main.step( "Create cell file" )
117 cellAppString = main.params[ 'ENV' ][ 'appString' ]
118 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
119 main.Mininet1.ip_address,
120 cellAppString, ipList )
121 main.step( "Applying cell variable to environment" )
122 cellResult = main.ONOSbench.setCell( cellName )
123 verifyResult = main.ONOSbench.verifyCell()
124
125 # FIXME:this is short term fix
126 main.log.info( "Removing raft logs" )
127 main.ONOSbench.onosRemoveRaftLogs()
128
129 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700130 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700131 main.ONOSbench.onosUninstall( node.ip_address )
132
133 # Make sure ONOS is DEAD
134 main.log.info( "Killing any ONOS processes" )
135 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700136 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700137 killed = main.ONOSbench.onosKill( node.ip_address )
138 killResults = killResults and killed
139
140 cleanInstallResult = main.TRUE
141 gitPullResult = main.TRUE
142
143 main.step( "Starting Mininet" )
144 # scp topo file to mininet
145 # TODO: move to params?
146 topoName = "obelisk.py"
147 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700148 main.ONOSbench.scp( main.Mininet1,
149 filePath + topoName,
150 main.Mininet1.home,
151 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700152 mnResult = main.Mininet1.startNet( )
153 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
154 onpass="Mininet Started",
155 onfail="Error starting Mininet" )
156
157 main.step( "Git checkout and pull " + gitBranch )
158 if PULLCODE:
159 main.ONOSbench.gitCheckout( gitBranch )
160 gitPullResult = main.ONOSbench.gitPull()
161 # values of 1 or 3 are good
162 utilities.assert_lesser( expect=0, actual=gitPullResult,
163 onpass="Git pull successful",
164 onfail="Git pull failed" )
165 main.ONOSbench.getVersion( report=True )
166
167 main.step( "Using mvn clean install" )
168 cleanInstallResult = main.TRUE
169 if PULLCODE and gitPullResult == main.TRUE:
170 cleanInstallResult = main.ONOSbench.cleanInstall()
171 else:
172 main.log.warn( "Did not pull new code so skipping mvn " +
173 "clean install" )
174 utilities.assert_equals( expect=main.TRUE,
175 actual=cleanInstallResult,
176 onpass="MCI successful",
177 onfail="MCI failed" )
178 # GRAPHS
179 # NOTE: important params here:
180 # job = name of Jenkins job
181 # Plot Name = Plot-HA, only can be used if multiple plots
182 # index = The number of the graph under plot name
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700183 job = "HAstopNodes"
Jon Hall5cf14d52015-07-16 12:15:19 -0700184 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700185 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700186 graphs = '<ac:structured-macro ac:name="html">\n'
187 graphs += '<ac:plain-text-body><![CDATA[\n'
188 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800189 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700190 '&width=500&height=300"' +\
191 'noborder="0" width="500" height="300" scrolling="yes" ' +\
192 'seamless="seamless"></iframe>\n'
193 graphs += ']]></ac:plain-text-body>\n'
194 graphs += '</ac:structured-macro>\n'
195 main.log.wiki(graphs)
196
197 main.step( "Creating ONOS package" )
Jon Hall3b489db2015-10-05 14:38:37 -0700198 # copy gen-partions file to ONOS
199 # NOTE: this assumes TestON and ONOS are on the same machine
Jon Hall53c5e662016-04-13 16:06:56 -0700200 srcFile = main.testDir + "/HA/dependencies/onos-gen-partitions"
Jon Hall3b489db2015-10-05 14:38:37 -0700201 dstDir = main.ONOSbench.home + "/tools/test/bin/onos-gen-partitions"
202 cpResult = main.ONOSbench.secureCopy( main.ONOSbench.user_name,
203 main.ONOSbench.ip_address,
204 srcFile,
205 dstDir,
206 pwd=main.ONOSbench.pwd,
207 direction="from" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700208 packageResult = main.ONOSbench.onosPackage()
209 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
210 onpass="ONOS package successful",
211 onfail="ONOS package failed" )
212
213 main.step( "Installing ONOS package" )
214 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700215 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700216 tmpResult = main.ONOSbench.onosInstall( options="-f",
217 node=node.ip_address )
218 onosInstallResult = onosInstallResult and tmpResult
219 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
220 onpass="ONOS install successful",
221 onfail="ONOS install failed" )
Jon Hall3b489db2015-10-05 14:38:37 -0700222 # clean up gen-partitions file
223 try:
224 main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
225 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
226 main.ONOSbench.handle.sendline( "git checkout -- tools/test/bin/onos-gen-partitions" )
227 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
228 main.log.info( " Cleaning custom gen partitions file, response was: \n" +
229 str( main.ONOSbench.handle.before ) )
230 except ( pexpect.TIMEOUT, pexpect.EOF ):
231 main.log.exception( "ONOSbench: pexpect exception found:" +
232 main.ONOSbench.handle.before )
233 main.cleanup()
234 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700235
236 main.step( "Checking if ONOS is up yet" )
237 for i in range( 2 ):
238 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700239 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700240 started = main.ONOSbench.isup( node.ip_address )
241 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800242 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700243 onosIsupResult = onosIsupResult and started
244 if onosIsupResult == main.TRUE:
245 break
246 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
247 onpass="ONOS startup successful",
248 onfail="ONOS startup failed" )
249
250 main.log.step( "Starting ONOS CLI sessions" )
251 cliResults = main.TRUE
252 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700253 for i in range( main.numCtrls ):
254 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700255 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700256 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700257 threads.append( t )
258 t.start()
259
260 for t in threads:
261 t.join()
262 cliResults = cliResults and t.result
263 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
264 onpass="ONOS cli startup successful",
265 onfail="ONOS cli startup failed" )
266
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700267 # Create a list of active nodes for use when some nodes are stopped
268 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
269
Jon Hall5cf14d52015-07-16 12:15:19 -0700270 if main.params[ 'tcpdump' ].lower() == "true":
271 main.step( "Start Packet Capture MN" )
272 main.Mininet2.startTcpdump(
273 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
274 + "-MN.pcap",
275 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
276 port=main.params[ 'MNtcpdump' ][ 'port' ] )
277
Jon Halla440e872016-03-31 15:15:50 -0700278 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700279 nodeResults = utilities.retry( main.HA.nodesCheck,
280 False,
281 args=[main.activeNodes],
282 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700283
Jon Hall41d39f12016-04-11 22:54:35 -0700284 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700285 onpass="Nodes check successful",
286 onfail="Nodes check NOT successful" )
287
288 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700289 for i in main.activeNodes:
290 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700291 main.log.debug( "{} components not ACTIVE: \n{}".format(
292 cli.name,
293 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700294 main.log.error( "Failed to start ONOS, stopping test" )
295 main.cleanup()
296 main.exit()
297
Jon Hall172b7ba2016-04-07 18:12:20 -0700298 main.step( "Activate apps defined in the params file" )
299 # get data from the params
300 apps = main.params.get( 'apps' )
301 if apps:
302 apps = apps.split(',')
303 main.log.warn( apps )
304 activateResult = True
305 for app in apps:
306 main.CLIs[ 0 ].app( app, "Activate" )
307 # TODO: check this worked
308 time.sleep( 10 ) # wait for apps to activate
309 for app in apps:
310 state = main.CLIs[ 0 ].appStatus( app )
311 if state == "ACTIVE":
312 activateResult = activeResult and True
313 else:
314 main.log.error( "{} is in {} state".format( app, state ) )
315 activeResult = False
316 utilities.assert_equals( expect=True,
317 actual=activateResult,
318 onpass="Successfully activated apps",
319 onfail="Failed to activate apps" )
320 else:
321 main.log.warn( "No apps were specified to be loaded after startup" )
322
323 main.step( "Set ONOS configurations" )
324 config = main.params.get( 'ONOS_Configuration' )
325 if config:
326 main.log.debug( config )
327 checkResult = main.TRUE
328 for component in config:
329 for setting in config[component]:
330 value = config[component][setting]
331 check = main.CLIs[ 0 ].setCfg( component, setting, value )
332 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
333 checkResult = check and checkResult
334 utilities.assert_equals( expect=main.TRUE,
335 actual=checkResult,
336 onpass="Successfully set config",
337 onfail="Failed to set config" )
338 else:
339 main.log.warn( "No configurations were specified to be changed after startup" )
340
Jon Hall9d2dcad2016-04-08 10:15:20 -0700341 main.step( "App Ids check" )
342 appCheck = main.TRUE
343 threads = []
344 for i in main.activeNodes:
345 t = main.Thread( target=main.CLIs[i].appToIDCheck,
346 name="appToIDCheck-" + str( i ),
347 args=[] )
348 threads.append( t )
349 t.start()
350
351 for t in threads:
352 t.join()
353 appCheck = appCheck and t.result
354 if appCheck != main.TRUE:
355 node = main.activeNodes[0]
356 main.log.warn( main.CLIs[node].apps() )
357 main.log.warn( main.CLIs[node].appIDs() )
358 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
359 onpass="App Ids seem to be correct",
360 onfail="Something is wrong with app Ids" )
361
Jon Hall5cf14d52015-07-16 12:15:19 -0700362 def CASE2( self, main ):
363 """
364 Assign devices to controllers
365 """
366 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700367 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 assert main, "main not defined"
369 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700370 assert main.CLIs, "main.CLIs not defined"
371 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 assert ONOS1Port, "ONOS1Port not defined"
373 assert ONOS2Port, "ONOS2Port not defined"
374 assert ONOS3Port, "ONOS3Port not defined"
375 assert ONOS4Port, "ONOS4Port not defined"
376 assert ONOS5Port, "ONOS5Port not defined"
377 assert ONOS6Port, "ONOS6Port not defined"
378 assert ONOS7Port, "ONOS7Port not defined"
379
380 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700381 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700382 "and check that an ONOS node becomes the " +\
383 "master of the device."
384 main.step( "Assign switches to controllers" )
385
386 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700387 for i in range( main.numCtrls ):
388 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700389 swList = []
390 for i in range( 1, 29 ):
391 swList.append( "s" + str( i ) )
392 main.Mininet1.assignSwController( sw=swList, ip=ipList )
393
394 mastershipCheck = main.TRUE
395 for i in range( 1, 29 ):
396 response = main.Mininet1.getSwController( "s" + str( i ) )
397 try:
398 main.log.info( str( response ) )
399 except Exception:
400 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700401 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700402 if re.search( "tcp:" + node.ip_address, response ):
403 mastershipCheck = mastershipCheck and main.TRUE
404 else:
405 main.log.error( "Error, node " + node.ip_address + " is " +
406 "not in the list of controllers s" +
407 str( i ) + " is connecting to." )
408 mastershipCheck = main.FALSE
409 utilities.assert_equals(
410 expect=main.TRUE,
411 actual=mastershipCheck,
412 onpass="Switch mastership assigned correctly",
413 onfail="Switches not assigned correctly to controllers" )
414
415 def CASE21( self, main ):
416 """
417 Assign mastership to controllers
418 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700419 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700420 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700421 assert main, "main not defined"
422 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700423 assert main.CLIs, "main.CLIs not defined"
424 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700425 assert ONOS1Port, "ONOS1Port not defined"
426 assert ONOS2Port, "ONOS2Port not defined"
427 assert ONOS3Port, "ONOS3Port not defined"
428 assert ONOS4Port, "ONOS4Port not defined"
429 assert ONOS5Port, "ONOS5Port not defined"
430 assert ONOS6Port, "ONOS6Port not defined"
431 assert ONOS7Port, "ONOS7Port not defined"
432
433 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700434 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700435 "device. Then manually assign" +\
436 " mastership to specific ONOS nodes using" +\
437 " 'device-role'"
438 main.step( "Assign mastership of switches to specific controllers" )
439 # Manually assign mastership to the controller we want
440 roleCall = main.TRUE
441
442 ipList = [ ]
443 deviceList = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700444 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700445 try:
446 # Assign mastership to specific controllers. This assignment was
447 # determined for a 7 node cluser, but will work with any sized
448 # cluster
449 for i in range( 1, 29 ): # switches 1 through 28
450 # set up correct variables:
451 if i == 1:
452 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700453 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700454 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700455 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700456 c = 1 % main.numCtrls
457 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700458 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700460 c = 1 % main.numCtrls
461 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700462 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700464 c = 3 % main.numCtrls
465 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700466 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700467 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700468 c = 2 % main.numCtrls
469 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700470 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700471 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700472 c = 2 % main.numCtrls
473 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700474 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700475 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700476 c = 5 % main.numCtrls
477 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700478 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700479 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700480 c = 4 % main.numCtrls
481 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700482 dpid = '3' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700483 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700484 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700485 c = 6 % main.numCtrls
486 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700487 dpid = '6' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700488 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700489 elif i == 28:
490 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700491 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700492 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700493 else:
494 main.log.error( "You didn't write an else statement for " +
495 "switch s" + str( i ) )
496 roleCall = main.FALSE
497 # Assign switch
498 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
499 # TODO: make this controller dynamic
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700500 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700501 ipList.append( ip )
502 deviceList.append( deviceId )
503 except ( AttributeError, AssertionError ):
504 main.log.exception( "Something is wrong with ONOS device view" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700505 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700506 utilities.assert_equals(
507 expect=main.TRUE,
508 actual=roleCall,
509 onpass="Re-assigned switch mastership to designated controller",
510 onfail="Something wrong with deviceRole calls" )
511
512 main.step( "Check mastership was correctly assigned" )
513 roleCheck = main.TRUE
514 # NOTE: This is due to the fact that device mastership change is not
515 # atomic and is actually a multi step process
516 time.sleep( 5 )
517 for i in range( len( ipList ) ):
518 ip = ipList[i]
519 deviceId = deviceList[i]
520 # Check assignment
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700521 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700522 if ip in master:
523 roleCheck = roleCheck and main.TRUE
524 else:
525 roleCheck = roleCheck and main.FALSE
526 main.log.error( "Error, controller " + ip + " is not" +
527 " master " + "of device " +
528 str( deviceId ) + ". Master is " +
529 repr( master ) + "." )
530 utilities.assert_equals(
531 expect=main.TRUE,
532 actual=roleCheck,
533 onpass="Switches were successfully reassigned to designated " +
534 "controller",
535 onfail="Switches were not successfully reassigned" )
536
537 def CASE3( self, main ):
538 """
539 Assign intents
540 """
541 import time
542 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700543 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700544 assert main, "main not defined"
545 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700546 assert main.CLIs, "main.CLIs not defined"
547 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700548 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700549 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700550 "assign predetermined host-to-host intents." +\
551 " After installation, check that the intent" +\
552 " is distributed to all nodes and the state" +\
553 " is INSTALLED"
554
555 # install onos-app-fwd
556 main.step( "Install reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700557 onosCli = main.CLIs[ main.activeNodes[0] ]
558 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700559 utilities.assert_equals( expect=main.TRUE, actual=installResults,
560 onpass="Install fwd successful",
561 onfail="Install fwd failed" )
562
563 main.step( "Check app ids" )
564 appCheck = main.TRUE
565 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700566 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700567 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700568 name="appToIDCheck-" + str( i ),
569 args=[] )
570 threads.append( t )
571 t.start()
572
573 for t in threads:
574 t.join()
575 appCheck = appCheck and t.result
576 if appCheck != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700577 main.log.warn( onosCli.apps() )
578 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700579 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
580 onpass="App Ids seem to be correct",
581 onfail="Something is wrong with app Ids" )
582
583 main.step( "Discovering Hosts( Via pingall for now )" )
584 # FIXME: Once we have a host discovery mechanism, use that instead
585 # REACTIVE FWD test
586 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700587 passMsg = "Reactive Pingall test passed"
588 time1 = time.time()
589 pingResult = main.Mininet1.pingall()
590 time2 = time.time()
591 if not pingResult:
592 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700593 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700594 passMsg += " on the second try"
595 utilities.assert_equals(
596 expect=main.TRUE,
597 actual=pingResult,
598 onpass= passMsg,
599 onfail="Reactive Pingall failed, " +
600 "one or more ping pairs failed" )
601 main.log.info( "Time for pingall: %2f seconds" %
602 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700603 # timeout for fwd flows
604 time.sleep( 11 )
605 # uninstall onos-app-fwd
606 main.step( "Uninstall reactive forwarding app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700607 node = main.activeNodes[0]
608 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700609 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
610 onpass="Uninstall fwd successful",
611 onfail="Uninstall fwd failed" )
612
613 main.step( "Check app ids" )
614 threads = []
615 appCheck2 = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700616 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700617 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700618 name="appToIDCheck-" + str( i ),
619 args=[] )
620 threads.append( t )
621 t.start()
622
623 for t in threads:
624 t.join()
625 appCheck2 = appCheck2 and t.result
626 if appCheck2 != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700627 node = main.activeNodes[0]
628 main.log.warn( main.CLIs[node].apps() )
629 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700630 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
631 onpass="App Ids seem to be correct",
632 onfail="Something is wrong with app Ids" )
633
634 main.step( "Add host intents via cli" )
635 intentIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700636 # TODO: move the host numbers to params
637 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700638 intentAddResult = True
639 hostResult = main.TRUE
640 for i in range( 8, 18 ):
641 main.log.info( "Adding host intent between h" + str( i ) +
642 " and h" + str( i + 10 ) )
643 host1 = "00:00:00:00:00:" + \
644 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
645 host2 = "00:00:00:00:00:" + \
646 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
647 # NOTE: getHost can return None
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700648 host1Dict = onosCli.getHost( host1 )
649 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700650 host1Id = None
651 host2Id = None
652 if host1Dict and host2Dict:
653 host1Id = host1Dict.get( 'id', None )
654 host2Id = host2Dict.get( 'id', None )
655 if host1Id and host2Id:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700656 nodeNum = ( i % len( main.activeNodes ) )
657 node = main.activeNodes[nodeNum]
658 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700659 if tmpId:
660 main.log.info( "Added intent with id: " + tmpId )
661 intentIds.append( tmpId )
662 else:
663 main.log.error( "addHostIntent returned: " +
664 repr( tmpId ) )
665 else:
666 main.log.error( "Error, getHost() failed for h" + str( i ) +
667 " and/or h" + str( i + 10 ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700668 node = main.activeNodes[0]
669 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700670 main.log.warn( "Hosts output: " )
671 try:
672 main.log.warn( json.dumps( json.loads( hosts ),
673 sort_keys=True,
674 indent=4,
675 separators=( ',', ': ' ) ) )
676 except ( ValueError, TypeError ):
677 main.log.warn( repr( hosts ) )
678 hostResult = main.FALSE
679 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
680 onpass="Found a host id for each host",
681 onfail="Error looking up host ids" )
682
683 intentStart = time.time()
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700684 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700685 main.log.info( "Submitted intents: " + str( intentIds ) )
686 main.log.info( "Intents in ONOS: " + str( onosIds ) )
687 for intent in intentIds:
688 if intent in onosIds:
689 pass # intent submitted is in onos
690 else:
691 intentAddResult = False
692 if intentAddResult:
693 intentStop = time.time()
694 else:
695 intentStop = None
696 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700697 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700698 intentStates = []
699 installedCheck = True
700 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
701 count = 0
702 try:
703 for intent in json.loads( intents ):
704 state = intent.get( 'state', None )
705 if "INSTALLED" not in state:
706 installedCheck = False
707 intentId = intent.get( 'id', None )
708 intentStates.append( ( intentId, state ) )
709 except ( ValueError, TypeError ):
710 main.log.exception( "Error parsing intents" )
711 # add submitted intents not in the store
712 tmplist = [ i for i, s in intentStates ]
713 missingIntents = False
714 for i in intentIds:
715 if i not in tmplist:
716 intentStates.append( ( i, " - " ) )
717 missingIntents = True
718 intentStates.sort()
719 for i, s in intentStates:
720 count += 1
721 main.log.info( "%-6s%-15s%-15s" %
722 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700723 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700724 try:
725 missing = False
726 if leaders:
727 parsedLeaders = json.loads( leaders )
728 main.log.warn( json.dumps( parsedLeaders,
729 sort_keys=True,
730 indent=4,
731 separators=( ',', ': ' ) ) )
732 # check for all intent partitions
733 topics = []
734 for i in range( 14 ):
735 topics.append( "intent-partition-" + str( i ) )
736 main.log.debug( topics )
737 ONOStopics = [ j['topic'] for j in parsedLeaders ]
738 for topic in topics:
739 if topic not in ONOStopics:
740 main.log.error( "Error: " + topic +
741 " not in leaders" )
742 missing = True
743 else:
744 main.log.error( "leaders() returned None" )
745 except ( ValueError, TypeError ):
746 main.log.exception( "Error parsing leaders" )
747 main.log.error( repr( leaders ) )
748 # Check all nodes
749 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700750 for i in main.activeNodes:
751 response = main.CLIs[i].leaders( jsonFormat=False)
752 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700753 str( response ) )
754
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700755 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700756 try:
757 if partitions :
758 parsedPartitions = json.loads( partitions )
759 main.log.warn( json.dumps( parsedPartitions,
760 sort_keys=True,
761 indent=4,
762 separators=( ',', ': ' ) ) )
763 # TODO check for a leader in all paritions
764 # TODO check for consistency among nodes
765 else:
766 main.log.error( "partitions() returned None" )
767 except ( ValueError, TypeError ):
768 main.log.exception( "Error parsing partitions" )
769 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700770 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700771 try:
772 if pendingMap :
773 parsedPending = json.loads( pendingMap )
774 main.log.warn( json.dumps( parsedPending,
775 sort_keys=True,
776 indent=4,
777 separators=( ',', ': ' ) ) )
778 # TODO check something here?
779 else:
780 main.log.error( "pendingMap() returned None" )
781 except ( ValueError, TypeError ):
782 main.log.exception( "Error parsing pending map" )
783 main.log.error( repr( pendingMap ) )
784
785 intentAddResult = bool( intentAddResult and not missingIntents and
786 installedCheck )
787 if not intentAddResult:
788 main.log.error( "Error in pushing host intents to ONOS" )
789
790 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700791 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700792 correct = True
793 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700794 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700795 onosIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700796 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700797 onosIds.append( ids )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700798 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700799 str( sorted( onosIds ) ) )
800 if sorted( ids ) != sorted( intentIds ):
801 main.log.warn( "Set of intent IDs doesn't match" )
802 correct = False
803 break
804 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700805 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700806 for intent in intents:
807 if intent[ 'state' ] != "INSTALLED":
808 main.log.warn( "Intent " + intent[ 'id' ] +
809 " is " + intent[ 'state' ] )
810 correct = False
811 break
812 if correct:
813 break
814 else:
815 time.sleep(1)
816 if not intentStop:
817 intentStop = time.time()
818 global gossipTime
819 gossipTime = intentStop - intentStart
820 main.log.info( "It took about " + str( gossipTime ) +
821 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700822 gossipPeriod = int( main.params['timers']['gossip'] )
823 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700824 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700825 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700826 onpass="ECM anti-entropy for intents worked within " +
827 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700828 onfail="Intent ECM anti-entropy took too long. " +
829 "Expected time:{}, Actual time:{}".format( maxGossipTime,
830 gossipTime ) )
831 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700832 intentAddResult = True
833
834 if not intentAddResult or "key" in pendingMap:
835 import time
836 installedCheck = True
837 main.log.info( "Sleeping 60 seconds to see if intents are found" )
838 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700839 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700840 main.log.info( "Submitted intents: " + str( intentIds ) )
841 main.log.info( "Intents in ONOS: " + str( onosIds ) )
842 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700843 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700844 intentStates = []
845 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
846 count = 0
847 try:
848 for intent in json.loads( intents ):
849 # Iter through intents of a node
850 state = intent.get( 'state', None )
851 if "INSTALLED" not in state:
852 installedCheck = False
853 intentId = intent.get( 'id', None )
854 intentStates.append( ( intentId, state ) )
855 except ( ValueError, TypeError ):
856 main.log.exception( "Error parsing intents" )
857 # add submitted intents not in the store
858 tmplist = [ i for i, s in intentStates ]
859 for i in intentIds:
860 if i not in tmplist:
861 intentStates.append( ( i, " - " ) )
862 intentStates.sort()
863 for i, s in intentStates:
864 count += 1
865 main.log.info( "%-6s%-15s%-15s" %
866 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700867 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700868 try:
869 missing = False
870 if leaders:
871 parsedLeaders = json.loads( leaders )
872 main.log.warn( json.dumps( parsedLeaders,
873 sort_keys=True,
874 indent=4,
875 separators=( ',', ': ' ) ) )
876 # check for all intent partitions
877 # check for election
878 topics = []
879 for i in range( 14 ):
880 topics.append( "intent-partition-" + str( i ) )
881 # FIXME: this should only be after we start the app
882 topics.append( "org.onosproject.election" )
883 main.log.debug( topics )
884 ONOStopics = [ j['topic'] for j in parsedLeaders ]
885 for topic in topics:
886 if topic not in ONOStopics:
887 main.log.error( "Error: " + topic +
888 " not in leaders" )
889 missing = True
890 else:
891 main.log.error( "leaders() returned None" )
892 except ( ValueError, TypeError ):
893 main.log.exception( "Error parsing leaders" )
894 main.log.error( repr( leaders ) )
895 # Check all nodes
896 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700897 for i in main.activeNodes:
898 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700899 response = node.leaders( jsonFormat=False)
900 main.log.warn( str( node.name ) + " leaders output: \n" +
901 str( response ) )
902
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700903 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700904 try:
905 if partitions :
906 parsedPartitions = json.loads( partitions )
907 main.log.warn( json.dumps( parsedPartitions,
908 sort_keys=True,
909 indent=4,
910 separators=( ',', ': ' ) ) )
911 # TODO check for a leader in all paritions
912 # TODO check for consistency among nodes
913 else:
914 main.log.error( "partitions() returned None" )
915 except ( ValueError, TypeError ):
916 main.log.exception( "Error parsing partitions" )
917 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700918 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700919 try:
920 if pendingMap :
921 parsedPending = json.loads( pendingMap )
922 main.log.warn( json.dumps( parsedPending,
923 sort_keys=True,
924 indent=4,
925 separators=( ',', ': ' ) ) )
926 # TODO check something here?
927 else:
928 main.log.error( "pendingMap() returned None" )
929 except ( ValueError, TypeError ):
930 main.log.exception( "Error parsing pending map" )
931 main.log.error( repr( pendingMap ) )
932
933 def CASE4( self, main ):
934 """
935 Ping across added host intents
936 """
937 import json
938 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700939 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700940 assert main, "main not defined"
941 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700942 assert main.CLIs, "main.CLIs not defined"
943 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700944 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700945 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700946 "functionality and check the state of " +\
947 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700948
Jon Hall41d39f12016-04-11 22:54:35 -0700949 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700950 main.step( "Check Intent state" )
951 installedCheck = False
952 loopCount = 0
953 while not installedCheck and loopCount < 40:
954 installedCheck = True
955 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700956 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700957 intentStates = []
958 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
959 count = 0
960 # Iter through intents of a node
961 try:
962 for intent in json.loads( intents ):
963 state = intent.get( 'state', None )
964 if "INSTALLED" not in state:
965 installedCheck = False
966 intentId = intent.get( 'id', None )
967 intentStates.append( ( intentId, state ) )
968 except ( ValueError, TypeError ):
969 main.log.exception( "Error parsing intents." )
970 # Print states
971 intentStates.sort()
972 for i, s in intentStates:
973 count += 1
974 main.log.info( "%-6s%-15s%-15s" %
975 ( str( count ), str( i ), str( s ) ) )
976 if not installedCheck:
977 time.sleep( 1 )
978 loopCount += 1
979 utilities.assert_equals( expect=True, actual=installedCheck,
980 onpass="Intents are all INSTALLED",
981 onfail="Intents are not all in " +
982 "INSTALLED state" )
983
Jon Hall9d2dcad2016-04-08 10:15:20 -0700984 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700985 PingResult = main.TRUE
986 for i in range( 8, 18 ):
987 ping = main.Mininet1.pingHost( src="h" + str( i ),
988 target="h" + str( i + 10 ) )
989 PingResult = PingResult and ping
990 if ping == main.FALSE:
991 main.log.warn( "Ping failed between h" + str( i ) +
992 " and h" + str( i + 10 ) )
993 elif ping == main.TRUE:
994 main.log.info( "Ping test passed!" )
995 # Don't set PingResult or you'd override failures
996 if PingResult == main.FALSE:
997 main.log.error(
998 "Intents have not been installed correctly, pings failed." )
999 # TODO: pretty print
1000 main.log.warn( "ONOS1 intents: " )
1001 try:
1002 tmpIntents = onosCli.intents()
1003 main.log.warn( json.dumps( json.loads( tmpIntents ),
1004 sort_keys=True,
1005 indent=4,
1006 separators=( ',', ': ' ) ) )
1007 except ( ValueError, TypeError ):
1008 main.log.warn( repr( tmpIntents ) )
1009 utilities.assert_equals(
1010 expect=main.TRUE,
1011 actual=PingResult,
1012 onpass="Intents have been installed correctly and pings work",
1013 onfail="Intents have not been installed correctly, pings failed." )
1014
Jon Hall5cf14d52015-07-16 12:15:19 -07001015 main.step( "Check leadership of topics" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001016 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001017 topicCheck = main.TRUE
1018 try:
1019 if leaders:
1020 parsedLeaders = json.loads( leaders )
1021 main.log.warn( json.dumps( parsedLeaders,
1022 sort_keys=True,
1023 indent=4,
1024 separators=( ',', ': ' ) ) )
1025 # check for all intent partitions
1026 # check for election
1027 # TODO: Look at Devices as topics now that it uses this system
1028 topics = []
1029 for i in range( 14 ):
1030 topics.append( "intent-partition-" + str( i ) )
1031 # FIXME: this should only be after we start the app
1032 # FIXME: topics.append( "org.onosproject.election" )
1033 # Print leaders output
1034 main.log.debug( topics )
1035 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1036 for topic in topics:
1037 if topic not in ONOStopics:
1038 main.log.error( "Error: " + topic +
1039 " not in leaders" )
1040 topicCheck = main.FALSE
1041 else:
1042 main.log.error( "leaders() returned None" )
1043 topicCheck = main.FALSE
1044 except ( ValueError, TypeError ):
1045 topicCheck = main.FALSE
1046 main.log.exception( "Error parsing leaders" )
1047 main.log.error( repr( leaders ) )
1048 # TODO: Check for a leader of these topics
1049 # Check all nodes
1050 if topicCheck:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001051 for i in main.activeNodes:
1052 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001053 response = node.leaders( jsonFormat=False)
1054 main.log.warn( str( node.name ) + " leaders output: \n" +
1055 str( response ) )
1056
1057 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1058 onpass="intent Partitions is in leaders",
1059 onfail="Some topics were lost " )
1060 # Print partitions
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001061 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001062 try:
1063 if partitions :
1064 parsedPartitions = json.loads( partitions )
1065 main.log.warn( json.dumps( parsedPartitions,
1066 sort_keys=True,
1067 indent=4,
1068 separators=( ',', ': ' ) ) )
1069 # TODO check for a leader in all paritions
1070 # TODO check for consistency among nodes
1071 else:
1072 main.log.error( "partitions() returned None" )
1073 except ( ValueError, TypeError ):
1074 main.log.exception( "Error parsing partitions" )
1075 main.log.error( repr( partitions ) )
1076 # Print Pending Map
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001077 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001078 try:
1079 if pendingMap :
1080 parsedPending = json.loads( pendingMap )
1081 main.log.warn( json.dumps( parsedPending,
1082 sort_keys=True,
1083 indent=4,
1084 separators=( ',', ': ' ) ) )
1085 # TODO check something here?
1086 else:
1087 main.log.error( "pendingMap() returned None" )
1088 except ( ValueError, TypeError ):
1089 main.log.exception( "Error parsing pending map" )
1090 main.log.error( repr( pendingMap ) )
1091
1092 if not installedCheck:
1093 main.log.info( "Waiting 60 seconds to see if the state of " +
1094 "intents change" )
1095 time.sleep( 60 )
1096 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001097 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001098 intentStates = []
1099 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1100 count = 0
1101 # Iter through intents of a node
1102 try:
1103 for intent in json.loads( intents ):
1104 state = intent.get( 'state', None )
1105 if "INSTALLED" not in state:
1106 installedCheck = False
1107 intentId = intent.get( 'id', None )
1108 intentStates.append( ( intentId, state ) )
1109 except ( ValueError, TypeError ):
1110 main.log.exception( "Error parsing intents." )
1111 intentStates.sort()
1112 for i, s in intentStates:
1113 count += 1
1114 main.log.info( "%-6s%-15s%-15s" %
1115 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001116 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001117 try:
1118 missing = False
1119 if leaders:
1120 parsedLeaders = json.loads( leaders )
1121 main.log.warn( json.dumps( parsedLeaders,
1122 sort_keys=True,
1123 indent=4,
1124 separators=( ',', ': ' ) ) )
1125 # check for all intent partitions
1126 # check for election
1127 topics = []
1128 for i in range( 14 ):
1129 topics.append( "intent-partition-" + str( i ) )
1130 # FIXME: this should only be after we start the app
1131 topics.append( "org.onosproject.election" )
1132 main.log.debug( topics )
1133 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1134 for topic in topics:
1135 if topic not in ONOStopics:
1136 main.log.error( "Error: " + topic +
1137 " not in leaders" )
1138 missing = True
1139 else:
1140 main.log.error( "leaders() returned None" )
1141 except ( ValueError, TypeError ):
1142 main.log.exception( "Error parsing leaders" )
1143 main.log.error( repr( leaders ) )
1144 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001145 for i in main.activeNodes:
1146 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001147 response = node.leaders( jsonFormat=False)
1148 main.log.warn( str( node.name ) + " leaders output: \n" +
1149 str( response ) )
1150
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001151 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001152 try:
1153 if partitions :
1154 parsedPartitions = json.loads( partitions )
1155 main.log.warn( json.dumps( parsedPartitions,
1156 sort_keys=True,
1157 indent=4,
1158 separators=( ',', ': ' ) ) )
1159 # TODO check for a leader in all paritions
1160 # TODO check for consistency among nodes
1161 else:
1162 main.log.error( "partitions() returned None" )
1163 except ( ValueError, TypeError ):
1164 main.log.exception( "Error parsing partitions" )
1165 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001166 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001167 try:
1168 if pendingMap :
1169 parsedPending = json.loads( pendingMap )
1170 main.log.warn( json.dumps( parsedPending,
1171 sort_keys=True,
1172 indent=4,
1173 separators=( ',', ': ' ) ) )
1174 # TODO check something here?
1175 else:
1176 main.log.error( "pendingMap() returned None" )
1177 except ( ValueError, TypeError ):
1178 main.log.exception( "Error parsing pending map" )
1179 main.log.error( repr( pendingMap ) )
1180 # Print flowrules
Jon Hall41d39f12016-04-11 22:54:35 -07001181 main.log.debug( onosCli.flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001182 main.step( "Wait a minute then ping again" )
1183 # the wait is above
1184 PingResult = main.TRUE
1185 for i in range( 8, 18 ):
1186 ping = main.Mininet1.pingHost( src="h" + str( i ),
1187 target="h" + str( i + 10 ) )
1188 PingResult = PingResult and ping
1189 if ping == main.FALSE:
1190 main.log.warn( "Ping failed between h" + str( i ) +
1191 " and h" + str( i + 10 ) )
1192 elif ping == main.TRUE:
1193 main.log.info( "Ping test passed!" )
1194 # Don't set PingResult or you'd override failures
1195 if PingResult == main.FALSE:
1196 main.log.error(
1197 "Intents have not been installed correctly, pings failed." )
1198 # TODO: pretty print
1199 main.log.warn( "ONOS1 intents: " )
1200 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001201 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001202 main.log.warn( json.dumps( json.loads( tmpIntents ),
1203 sort_keys=True,
1204 indent=4,
1205 separators=( ',', ': ' ) ) )
1206 except ( ValueError, TypeError ):
1207 main.log.warn( repr( tmpIntents ) )
1208 utilities.assert_equals(
1209 expect=main.TRUE,
1210 actual=PingResult,
1211 onpass="Intents have been installed correctly and pings work",
1212 onfail="Intents have not been installed correctly, pings failed." )
1213
1214 def CASE5( self, main ):
1215 """
1216 Reading state of ONOS
1217 """
1218 import json
1219 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001220 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001221 assert main, "main not defined"
1222 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001223 assert main.CLIs, "main.CLIs not defined"
1224 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001225
1226 main.case( "Setting up and gathering data for current state" )
1227 # The general idea for this test case is to pull the state of
1228 # ( intents,flows, topology,... ) from each ONOS node
1229 # We can then compare them with each other and also with past states
1230
1231 main.step( "Check that each switch has a master" )
1232 global mastershipState
1233 mastershipState = '[]'
1234
1235 # Assert that each device has a master
1236 rolesNotNull = main.TRUE
1237 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001238 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001239 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001240 name="rolesNotNull-" + str( i ),
1241 args=[] )
1242 threads.append( t )
1243 t.start()
1244
1245 for t in threads:
1246 t.join()
1247 rolesNotNull = rolesNotNull and t.result
1248 utilities.assert_equals(
1249 expect=main.TRUE,
1250 actual=rolesNotNull,
1251 onpass="Each device has a master",
1252 onfail="Some devices don't have a master assigned" )
1253
1254 main.step( "Get the Mastership of each switch from each controller" )
1255 ONOSMastership = []
1256 mastershipCheck = main.FALSE
1257 consistentMastership = True
1258 rolesResults = True
1259 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001260 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001261 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001262 name="roles-" + str( i ),
1263 args=[] )
1264 threads.append( t )
1265 t.start()
1266
1267 for t in threads:
1268 t.join()
1269 ONOSMastership.append( t.result )
1270
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001271 for i in range( len( ONOSMastership ) ):
1272 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001273 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001274 main.log.error( "Error in getting ONOS" + node + " roles" )
1275 main.log.warn( "ONOS" + node + " mastership response: " +
1276 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001277 rolesResults = False
1278 utilities.assert_equals(
1279 expect=True,
1280 actual=rolesResults,
1281 onpass="No error in reading roles output",
1282 onfail="Error in reading roles from ONOS" )
1283
1284 main.step( "Check for consistency in roles from each controller" )
1285 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1286 main.log.info(
1287 "Switch roles are consistent across all ONOS nodes" )
1288 else:
1289 consistentMastership = False
1290 utilities.assert_equals(
1291 expect=True,
1292 actual=consistentMastership,
1293 onpass="Switch roles are consistent across all ONOS nodes",
1294 onfail="ONOS nodes have different views of switch roles" )
1295
1296 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001297 for i in range( len( main.activeNodes ) ):
1298 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001299 try:
1300 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001301 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001302 json.dumps(
1303 json.loads( ONOSMastership[ i ] ),
1304 sort_keys=True,
1305 indent=4,
1306 separators=( ',', ': ' ) ) )
1307 except ( ValueError, TypeError ):
1308 main.log.warn( repr( ONOSMastership[ i ] ) )
1309 elif rolesResults and consistentMastership:
1310 mastershipCheck = main.TRUE
1311 mastershipState = ONOSMastership[ 0 ]
1312
1313 main.step( "Get the intents from each controller" )
1314 global intentState
1315 intentState = []
1316 ONOSIntents = []
1317 intentCheck = main.FALSE
1318 consistentIntents = True
1319 intentsResults = True
1320 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001321 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001322 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001323 name="intents-" + str( i ),
1324 args=[],
1325 kwargs={ 'jsonFormat': True } )
1326 threads.append( t )
1327 t.start()
1328
1329 for t in threads:
1330 t.join()
1331 ONOSIntents.append( t.result )
1332
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001333 for i in range( len( ONOSIntents ) ):
1334 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001335 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001336 main.log.error( "Error in getting ONOS" + node + " intents" )
1337 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001338 repr( ONOSIntents[ i ] ) )
1339 intentsResults = False
1340 utilities.assert_equals(
1341 expect=True,
1342 actual=intentsResults,
1343 onpass="No error in reading intents output",
1344 onfail="Error in reading intents from ONOS" )
1345
1346 main.step( "Check for consistency in Intents from each controller" )
1347 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1348 main.log.info( "Intents are consistent across all ONOS " +
1349 "nodes" )
1350 else:
1351 consistentIntents = False
1352 main.log.error( "Intents not consistent" )
1353 utilities.assert_equals(
1354 expect=True,
1355 actual=consistentIntents,
1356 onpass="Intents are consistent across all ONOS nodes",
1357 onfail="ONOS nodes have different views of intents" )
1358
1359 if intentsResults:
1360 # Try to make it easy to figure out what is happening
1361 #
1362 # Intent ONOS1 ONOS2 ...
1363 # 0x01 INSTALLED INSTALLING
1364 # ... ... ...
1365 # ... ... ...
1366 title = " Id"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001367 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001368 title += " " * 10 + "ONOS" + str( n + 1 )
1369 main.log.warn( title )
1370 # get all intent keys in the cluster
1371 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001372 try:
1373 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001374 for nodeStr in ONOSIntents:
1375 node = json.loads( nodeStr )
1376 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001377 keys.append( intent.get( 'id' ) )
1378 keys = set( keys )
1379 # For each intent key, print the state on each node
1380 for key in keys:
1381 row = "%-13s" % key
1382 for nodeStr in ONOSIntents:
1383 node = json.loads( nodeStr )
1384 for intent in node:
1385 if intent.get( 'id', "Error" ) == key:
1386 row += "%-15s" % intent.get( 'state' )
1387 main.log.warn( row )
1388 # End of intent state table
1389 except ValueError as e:
1390 main.log.exception( e )
1391 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001392
1393 if intentsResults and not consistentIntents:
1394 # print the json objects
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001395 n = str( main.activeNodes[-1] + 1 )
1396 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001397 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1398 sort_keys=True,
1399 indent=4,
1400 separators=( ',', ': ' ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001401 for i in range( len( ONOSIntents ) ):
1402 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001403 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001404 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001405 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1406 sort_keys=True,
1407 indent=4,
1408 separators=( ',', ': ' ) ) )
1409 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001410 main.log.debug( "ONOS" + node + " intents match ONOS" +
1411 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001412 elif intentsResults and consistentIntents:
1413 intentCheck = main.TRUE
1414 intentState = ONOSIntents[ 0 ]
1415
1416 main.step( "Get the flows from each controller" )
1417 global flowState
1418 flowState = []
1419 ONOSFlows = []
1420 ONOSFlowsJson = []
1421 flowCheck = main.FALSE
1422 consistentFlows = True
1423 flowsResults = True
1424 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001425 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001426 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001427 name="flows-" + str( i ),
1428 args=[],
1429 kwargs={ 'jsonFormat': True } )
1430 threads.append( t )
1431 t.start()
1432
1433 # NOTE: Flows command can take some time to run
1434 time.sleep(30)
1435 for t in threads:
1436 t.join()
1437 result = t.result
1438 ONOSFlows.append( result )
1439
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001440 for i in range( len( ONOSFlows ) ):
1441 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001442 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1443 main.log.error( "Error in getting ONOS" + num + " flows" )
1444 main.log.warn( "ONOS" + num + " flows response: " +
1445 repr( ONOSFlows[ i ] ) )
1446 flowsResults = False
1447 ONOSFlowsJson.append( None )
1448 else:
1449 try:
1450 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1451 except ( ValueError, TypeError ):
1452 # FIXME: change this to log.error?
1453 main.log.exception( "Error in parsing ONOS" + num +
1454 " response as json." )
1455 main.log.error( repr( ONOSFlows[ i ] ) )
1456 ONOSFlowsJson.append( None )
1457 flowsResults = False
1458 utilities.assert_equals(
1459 expect=True,
1460 actual=flowsResults,
1461 onpass="No error in reading flows output",
1462 onfail="Error in reading flows from ONOS" )
1463
1464 main.step( "Check for consistency in Flows from each controller" )
1465 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1466 if all( tmp ):
1467 main.log.info( "Flow count is consistent across all ONOS nodes" )
1468 else:
1469 consistentFlows = False
1470 utilities.assert_equals(
1471 expect=True,
1472 actual=consistentFlows,
1473 onpass="The flow count is consistent across all ONOS nodes",
1474 onfail="ONOS nodes have different flow counts" )
1475
1476 if flowsResults and not consistentFlows:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001477 for i in range( len( ONOSFlows ) ):
1478 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001479 try:
1480 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001481 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001482 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1483 indent=4, separators=( ',', ': ' ) ) )
1484 except ( ValueError, TypeError ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001485 main.log.warn( "ONOS" + node + " flows: " +
1486 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001487 elif flowsResults and consistentFlows:
1488 flowCheck = main.TRUE
1489 flowState = ONOSFlows[ 0 ]
1490
1491 main.step( "Get the OF Table entries" )
1492 global flows
1493 flows = []
1494 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001495 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001496 if flowCheck == main.FALSE:
1497 for table in flows:
1498 main.log.warn( table )
1499 # TODO: Compare switch flow tables with ONOS flow tables
1500
1501 main.step( "Start continuous pings" )
1502 main.Mininet2.pingLong(
1503 src=main.params[ 'PING' ][ 'source1' ],
1504 target=main.params[ 'PING' ][ 'target1' ],
1505 pingTime=500 )
1506 main.Mininet2.pingLong(
1507 src=main.params[ 'PING' ][ 'source2' ],
1508 target=main.params[ 'PING' ][ 'target2' ],
1509 pingTime=500 )
1510 main.Mininet2.pingLong(
1511 src=main.params[ 'PING' ][ 'source3' ],
1512 target=main.params[ 'PING' ][ 'target3' ],
1513 pingTime=500 )
1514 main.Mininet2.pingLong(
1515 src=main.params[ 'PING' ][ 'source4' ],
1516 target=main.params[ 'PING' ][ 'target4' ],
1517 pingTime=500 )
1518 main.Mininet2.pingLong(
1519 src=main.params[ 'PING' ][ 'source5' ],
1520 target=main.params[ 'PING' ][ 'target5' ],
1521 pingTime=500 )
1522 main.Mininet2.pingLong(
1523 src=main.params[ 'PING' ][ 'source6' ],
1524 target=main.params[ 'PING' ][ 'target6' ],
1525 pingTime=500 )
1526 main.Mininet2.pingLong(
1527 src=main.params[ 'PING' ][ 'source7' ],
1528 target=main.params[ 'PING' ][ 'target7' ],
1529 pingTime=500 )
1530 main.Mininet2.pingLong(
1531 src=main.params[ 'PING' ][ 'source8' ],
1532 target=main.params[ 'PING' ][ 'target8' ],
1533 pingTime=500 )
1534 main.Mininet2.pingLong(
1535 src=main.params[ 'PING' ][ 'source9' ],
1536 target=main.params[ 'PING' ][ 'target9' ],
1537 pingTime=500 )
1538 main.Mininet2.pingLong(
1539 src=main.params[ 'PING' ][ 'source10' ],
1540 target=main.params[ 'PING' ][ 'target10' ],
1541 pingTime=500 )
1542
1543 main.step( "Collecting topology information from ONOS" )
1544 devices = []
1545 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001546 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001547 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001548 name="devices-" + str( i ),
1549 args=[ ] )
1550 threads.append( t )
1551 t.start()
1552
1553 for t in threads:
1554 t.join()
1555 devices.append( t.result )
1556 hosts = []
1557 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001558 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001559 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001560 name="hosts-" + str( i ),
1561 args=[ ] )
1562 threads.append( t )
1563 t.start()
1564
1565 for t in threads:
1566 t.join()
1567 try:
1568 hosts.append( json.loads( t.result ) )
1569 except ( ValueError, TypeError ):
1570 # FIXME: better handling of this, print which node
1571 # Maybe use thread name?
1572 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001573 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001574 hosts.append( None )
1575
1576 ports = []
1577 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001578 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001579 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001580 name="ports-" + str( i ),
1581 args=[ ] )
1582 threads.append( t )
1583 t.start()
1584
1585 for t in threads:
1586 t.join()
1587 ports.append( t.result )
1588 links = []
1589 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001590 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001591 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001592 name="links-" + str( i ),
1593 args=[ ] )
1594 threads.append( t )
1595 t.start()
1596
1597 for t in threads:
1598 t.join()
1599 links.append( t.result )
1600 clusters = []
1601 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001602 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001603 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001604 name="clusters-" + str( i ),
1605 args=[ ] )
1606 threads.append( t )
1607 t.start()
1608
1609 for t in threads:
1610 t.join()
1611 clusters.append( t.result )
1612 # Compare json objects for hosts and dataplane clusters
1613
1614 # hosts
1615 main.step( "Host view is consistent across ONOS nodes" )
1616 consistentHostsResult = main.TRUE
1617 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001618 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001619 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001620 if hosts[ controller ] == hosts[ 0 ]:
1621 continue
1622 else: # hosts not consistent
1623 main.log.error( "hosts from ONOS" +
1624 controllerStr +
1625 " is inconsistent with ONOS1" )
1626 main.log.warn( repr( hosts[ controller ] ) )
1627 consistentHostsResult = main.FALSE
1628
1629 else:
1630 main.log.error( "Error in getting ONOS hosts from ONOS" +
1631 controllerStr )
1632 consistentHostsResult = main.FALSE
1633 main.log.warn( "ONOS" + controllerStr +
1634 " hosts response: " +
1635 repr( hosts[ controller ] ) )
1636 utilities.assert_equals(
1637 expect=main.TRUE,
1638 actual=consistentHostsResult,
1639 onpass="Hosts view is consistent across all ONOS nodes",
1640 onfail="ONOS nodes have different views of hosts" )
1641
1642 main.step( "Each host has an IP address" )
1643 ipResult = main.TRUE
1644 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001645 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001646 if hosts[ controller ]:
1647 for host in hosts[ controller ]:
1648 if not host.get( 'ipAddresses', [ ] ):
1649 main.log.error( "Error with host ips on controller" +
1650 controllerStr + ": " + str( host ) )
1651 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001652 utilities.assert_equals(
1653 expect=main.TRUE,
1654 actual=ipResult,
1655 onpass="The ips of the hosts aren't empty",
1656 onfail="The ip of at least one host is missing" )
1657
1658 # Strongly connected clusters of devices
1659 main.step( "Cluster view is consistent across ONOS nodes" )
1660 consistentClustersResult = main.TRUE
1661 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001662 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001663 if "Error" not in clusters[ controller ]:
1664 if clusters[ controller ] == clusters[ 0 ]:
1665 continue
1666 else: # clusters not consistent
1667 main.log.error( "clusters from ONOS" + controllerStr +
1668 " is inconsistent with ONOS1" )
1669 consistentClustersResult = main.FALSE
1670
1671 else:
1672 main.log.error( "Error in getting dataplane clusters " +
1673 "from ONOS" + controllerStr )
1674 consistentClustersResult = main.FALSE
1675 main.log.warn( "ONOS" + controllerStr +
1676 " clusters response: " +
1677 repr( clusters[ controller ] ) )
1678 utilities.assert_equals(
1679 expect=main.TRUE,
1680 actual=consistentClustersResult,
1681 onpass="Clusters view is consistent across all ONOS nodes",
1682 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001683 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001684 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001685
Jon Hall5cf14d52015-07-16 12:15:19 -07001686 # there should always only be one cluster
1687 main.step( "Cluster view correct across ONOS nodes" )
1688 try:
1689 numClusters = len( json.loads( clusters[ 0 ] ) )
1690 except ( ValueError, TypeError ):
1691 main.log.exception( "Error parsing clusters[0]: " +
1692 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001693 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001694 clusterResults = main.FALSE
1695 if numClusters == 1:
1696 clusterResults = main.TRUE
1697 utilities.assert_equals(
1698 expect=1,
1699 actual=numClusters,
1700 onpass="ONOS shows 1 SCC",
1701 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1702
1703 main.step( "Comparing ONOS topology to MN" )
1704 devicesResults = main.TRUE
1705 linksResults = main.TRUE
1706 hostsResults = main.TRUE
1707 mnSwitches = main.Mininet1.getSwitches()
1708 mnLinks = main.Mininet1.getLinks()
1709 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001710 for controller in main.activeNodes:
1711 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001712 if devices[ controller ] and ports[ controller ] and\
1713 "Error" not in devices[ controller ] and\
1714 "Error" not in ports[ controller ]:
Jon Hall6e709752016-02-01 13:38:46 -08001715 currentDevicesResult = main.Mininet1.compareSwitches(
1716 mnSwitches,
1717 json.loads( devices[ controller ] ),
1718 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001719 else:
1720 currentDevicesResult = main.FALSE
1721 utilities.assert_equals( expect=main.TRUE,
1722 actual=currentDevicesResult,
1723 onpass="ONOS" + controllerStr +
1724 " Switches view is correct",
1725 onfail="ONOS" + controllerStr +
1726 " Switches view is incorrect" )
1727 if links[ controller ] and "Error" not in links[ controller ]:
1728 currentLinksResult = main.Mininet1.compareLinks(
1729 mnSwitches, mnLinks,
1730 json.loads( links[ controller ] ) )
1731 else:
1732 currentLinksResult = main.FALSE
1733 utilities.assert_equals( expect=main.TRUE,
1734 actual=currentLinksResult,
1735 onpass="ONOS" + controllerStr +
1736 " links view is correct",
1737 onfail="ONOS" + controllerStr +
1738 " links view is incorrect" )
1739
Jon Hall657cdf62015-12-17 14:40:51 -08001740 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001741 currentHostsResult = main.Mininet1.compareHosts(
1742 mnHosts,
1743 hosts[ controller ] )
1744 else:
1745 currentHostsResult = main.FALSE
1746 utilities.assert_equals( expect=main.TRUE,
1747 actual=currentHostsResult,
1748 onpass="ONOS" + controllerStr +
1749 " hosts exist in Mininet",
1750 onfail="ONOS" + controllerStr +
1751 " hosts don't match Mininet" )
1752
1753 devicesResults = devicesResults and currentDevicesResult
1754 linksResults = linksResults and currentLinksResult
1755 hostsResults = hostsResults and currentHostsResult
1756
1757 main.step( "Device information is correct" )
1758 utilities.assert_equals(
1759 expect=main.TRUE,
1760 actual=devicesResults,
1761 onpass="Device information is correct",
1762 onfail="Device information is incorrect" )
1763
1764 main.step( "Links are correct" )
1765 utilities.assert_equals(
1766 expect=main.TRUE,
1767 actual=linksResults,
1768 onpass="Link are correct",
1769 onfail="Links are incorrect" )
1770
1771 main.step( "Hosts are correct" )
1772 utilities.assert_equals(
1773 expect=main.TRUE,
1774 actual=hostsResults,
1775 onpass="Hosts are correct",
1776 onfail="Hosts are incorrect" )
1777
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001778 def CASE61( self, main ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001779 """
1780 The Failure case.
1781 """
Jon Halle1a3b752015-07-22 13:02:46 -07001782 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001783 assert main, "main not defined"
1784 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001785 assert main.CLIs, "main.CLIs not defined"
1786 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001787 main.case( "Stop minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -07001788
1789 main.step( "Checking ONOS Logs for errors" )
1790 for node in main.nodes:
1791 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1792 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1793
Jon Hall3b489db2015-10-05 14:38:37 -07001794 n = len( main.nodes ) # Number of nodes
1795 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
1796 main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
1797 if n > 3:
1798 main.kill.append( p - 1 )
1799 # NOTE: This only works for cluster sizes of 3,5, or 7.
1800
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001801 main.step( "Stopping " + str( len( main.kill ) ) + " ONOS nodes" )
Jon Hall3b489db2015-10-05 14:38:37 -07001802 killResults = main.TRUE
1803 for i in main.kill:
1804 killResults = killResults and\
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001805 main.ONOSbench.onosStop( main.nodes[i].ip_address )
1806 main.activeNodes.remove( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001807 utilities.assert_equals( expect=main.TRUE, actual=killResults,
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001808 onpass="ONOS nodes stopped successfully",
1809 onfail="ONOS nodes NOT successfully stopped" )
1810
1811 def CASE62( self, main ):
1812 """
1813 The bring up stopped nodes
1814 """
1815 import time
1816 assert main.numCtrls, "main.numCtrls not defined"
1817 assert main, "main not defined"
1818 assert utilities.assert_equals, "utilities.assert_equals not defined"
1819 assert main.CLIs, "main.CLIs not defined"
1820 assert main.nodes, "main.nodes not defined"
1821 assert main.kill, "main.kill not defined"
1822 main.case( "Restart minority of ONOS nodes" )
1823
1824 main.step( "Restarting " + str( len( main.kill ) ) + " ONOS nodes" )
1825 startResults = main.TRUE
1826 restartTime = time.time()
1827 for i in main.kill:
1828 startResults = startResults and\
1829 main.ONOSbench.onosStart( main.nodes[i].ip_address )
1830 utilities.assert_equals( expect=main.TRUE, actual=startResults,
1831 onpass="ONOS nodes started successfully",
1832 onfail="ONOS nodes NOT successfully started" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001833
1834 main.step( "Checking if ONOS is up yet" )
1835 count = 0
1836 onosIsupResult = main.FALSE
1837 while onosIsupResult == main.FALSE and count < 10:
Jon Hall3b489db2015-10-05 14:38:37 -07001838 onosIsupResult = main.TRUE
1839 for i in main.kill:
1840 onosIsupResult = onosIsupResult and\
1841 main.ONOSbench.isup( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001842 count = count + 1
Jon Hall5cf14d52015-07-16 12:15:19 -07001843 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1844 onpass="ONOS restarted successfully",
1845 onfail="ONOS restart NOT successful" )
1846
Jon Halle1a3b752015-07-22 13:02:46 -07001847 main.step( "Restarting ONOS main.CLIs" )
Jon Hall3b489db2015-10-05 14:38:37 -07001848 cliResults = main.TRUE
1849 for i in main.kill:
1850 cliResults = cliResults and\
1851 main.CLIs[i].startOnosCli( main.nodes[i].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001852 main.activeNodes.append( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001853 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1854 onpass="ONOS cli restarted",
1855 onfail="ONOS cli did not restart" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001856 main.activeNodes.sort()
1857 try:
1858 assert list( set( main.activeNodes ) ) == main.activeNodes,\
1859 "List of active nodes has duplicates, this likely indicates something was run out of order"
1860 except AssertionError:
1861 main.log.exception( "" )
1862 main.cleanup()
1863 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -07001864
1865 # Grab the time of restart so we chan check how long the gossip
1866 # protocol has had time to work
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001867 main.restartTime = time.time() - restartTime
Jon Hall5cf14d52015-07-16 12:15:19 -07001868 main.log.debug( "Restart time: " + str( main.restartTime ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001869 # TODO: MAke this configurable. Also, we are breaking the above timer
1870 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001871 node = main.activeNodes[0]
1872 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1873 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1874 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001875
Jon Halla440e872016-03-31 15:15:50 -07001876 main.step( "Rerun for election on the node(s) that were killed" )
1877 runResults = main.TRUE
1878 for i in main.kill:
1879 runResults = runResults and\
1880 main.CLIs[i].electionTestRun()
1881 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1882 onpass="ONOS nodes reran for election topic",
1883 onfail="Errror rerunning for election" )
1884
Jon Hall5cf14d52015-07-16 12:15:19 -07001885 def CASE7( self, main ):
1886 """
1887 Check state after ONOS failure
1888 """
1889 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001890 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001891 assert main, "main not defined"
1892 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001893 assert main.CLIs, "main.CLIs not defined"
1894 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001895 try:
1896 main.kill
1897 except AttributeError:
1898 main.kill = []
1899
Jon Hall5cf14d52015-07-16 12:15:19 -07001900 main.case( "Running ONOS Constant State Tests" )
1901
1902 main.step( "Check that each switch has a master" )
1903 # Assert that each device has a master
1904 rolesNotNull = main.TRUE
1905 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001906 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001907 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001908 name="rolesNotNull-" + str( i ),
1909 args=[ ] )
1910 threads.append( t )
1911 t.start()
1912
1913 for t in threads:
1914 t.join()
1915 rolesNotNull = rolesNotNull and t.result
1916 utilities.assert_equals(
1917 expect=main.TRUE,
1918 actual=rolesNotNull,
1919 onpass="Each device has a master",
1920 onfail="Some devices don't have a master assigned" )
1921
1922 main.step( "Read device roles from ONOS" )
1923 ONOSMastership = []
Jon Halla440e872016-03-31 15:15:50 -07001924 mastershipCheck = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001925 consistentMastership = True
1926 rolesResults = True
1927 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001928 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001929 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001930 name="roles-" + str( i ),
1931 args=[] )
1932 threads.append( t )
1933 t.start()
1934
1935 for t in threads:
1936 t.join()
1937 ONOSMastership.append( t.result )
1938
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001939 for i in range( len( ONOSMastership ) ):
1940 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001941 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001942 main.log.error( "Error in getting ONOS" + node + " roles" )
1943 main.log.warn( "ONOS" + node + " mastership response: " +
1944 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001945 rolesResults = False
1946 utilities.assert_equals(
1947 expect=True,
1948 actual=rolesResults,
1949 onpass="No error in reading roles output",
1950 onfail="Error in reading roles from ONOS" )
1951
1952 main.step( "Check for consistency in roles from each controller" )
1953 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1954 main.log.info(
1955 "Switch roles are consistent across all ONOS nodes" )
1956 else:
1957 consistentMastership = False
1958 utilities.assert_equals(
1959 expect=True,
1960 actual=consistentMastership,
1961 onpass="Switch roles are consistent across all ONOS nodes",
1962 onfail="ONOS nodes have different views of switch roles" )
1963
1964 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001965 for i in range( len( ONOSMastership ) ):
1966 node = str( main.activeNodes[i] + 1 )
1967 main.log.warn( "ONOS" + node + " roles: ",
1968 json.dumps( json.loads( ONOSMastership[ i ] ),
1969 sort_keys=True,
1970 indent=4,
1971 separators=( ',', ': ' ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001972
1973 # NOTE: we expect mastership to change on controller failure
Jon Hall5cf14d52015-07-16 12:15:19 -07001974
1975 main.step( "Get the intents and compare across all nodes" )
1976 ONOSIntents = []
1977 intentCheck = main.FALSE
1978 consistentIntents = True
1979 intentsResults = True
1980 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001981 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001982 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001983 name="intents-" + str( i ),
1984 args=[],
1985 kwargs={ 'jsonFormat': True } )
1986 threads.append( t )
1987 t.start()
1988
1989 for t in threads:
1990 t.join()
1991 ONOSIntents.append( t.result )
1992
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001993 for i in range( len( ONOSIntents) ):
1994 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001995 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001996 main.log.error( "Error in getting ONOS" + node + " intents" )
1997 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001998 repr( ONOSIntents[ i ] ) )
1999 intentsResults = False
2000 utilities.assert_equals(
2001 expect=True,
2002 actual=intentsResults,
2003 onpass="No error in reading intents output",
2004 onfail="Error in reading intents from ONOS" )
2005
2006 main.step( "Check for consistency in Intents from each controller" )
2007 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2008 main.log.info( "Intents are consistent across all ONOS " +
2009 "nodes" )
2010 else:
2011 consistentIntents = False
2012
2013 # Try to make it easy to figure out what is happening
2014 #
2015 # Intent ONOS1 ONOS2 ...
2016 # 0x01 INSTALLED INSTALLING
2017 # ... ... ...
2018 # ... ... ...
2019 title = " ID"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002020 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002021 title += " " * 10 + "ONOS" + str( n + 1 )
2022 main.log.warn( title )
2023 # get all intent keys in the cluster
2024 keys = []
2025 for nodeStr in ONOSIntents:
2026 node = json.loads( nodeStr )
2027 for intent in node:
2028 keys.append( intent.get( 'id' ) )
2029 keys = set( keys )
2030 for key in keys:
2031 row = "%-13s" % key
2032 for nodeStr in ONOSIntents:
2033 node = json.loads( nodeStr )
2034 for intent in node:
2035 if intent.get( 'id' ) == key:
2036 row += "%-15s" % intent.get( 'state' )
2037 main.log.warn( row )
2038 # End table view
2039
2040 utilities.assert_equals(
2041 expect=True,
2042 actual=consistentIntents,
2043 onpass="Intents are consistent across all ONOS nodes",
2044 onfail="ONOS nodes have different views of intents" )
2045 intentStates = []
2046 for node in ONOSIntents: # Iter through ONOS nodes
2047 nodeStates = []
2048 # Iter through intents of a node
2049 try:
2050 for intent in json.loads( node ):
2051 nodeStates.append( intent[ 'state' ] )
2052 except ( ValueError, TypeError ):
2053 main.log.exception( "Error in parsing intents" )
2054 main.log.error( repr( node ) )
2055 intentStates.append( nodeStates )
2056 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2057 main.log.info( dict( out ) )
2058
2059 if intentsResults and not consistentIntents:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002060 for i in range( len( main.activeNodes ) ):
2061 node = str( main.activeNodes[i] + 1 )
2062 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002063 main.log.warn( json.dumps(
2064 json.loads( ONOSIntents[ i ] ),
2065 sort_keys=True,
2066 indent=4,
2067 separators=( ',', ': ' ) ) )
2068 elif intentsResults and consistentIntents:
2069 intentCheck = main.TRUE
2070
2071 # NOTE: Store has no durability, so intents are lost across system
2072 # restarts
2073 main.step( "Compare current intents with intents before the failure" )
2074 # NOTE: this requires case 5 to pass for intentState to be set.
2075 # maybe we should stop the test if that fails?
2076 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002077 try:
2078 intentState
2079 except NameError:
2080 main.log.warn( "No previous intent state was saved" )
2081 else:
2082 if intentState and intentState == ONOSIntents[ 0 ]:
2083 sameIntents = main.TRUE
2084 main.log.info( "Intents are consistent with before failure" )
2085 # TODO: possibly the states have changed? we may need to figure out
2086 # what the acceptable states are
2087 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2088 sameIntents = main.TRUE
2089 try:
2090 before = json.loads( intentState )
2091 after = json.loads( ONOSIntents[ 0 ] )
2092 for intent in before:
2093 if intent not in after:
2094 sameIntents = main.FALSE
2095 main.log.debug( "Intent is not currently in ONOS " +
2096 "(at least in the same form):" )
2097 main.log.debug( json.dumps( intent ) )
2098 except ( ValueError, TypeError ):
2099 main.log.exception( "Exception printing intents" )
2100 main.log.debug( repr( ONOSIntents[0] ) )
2101 main.log.debug( repr( intentState ) )
2102 if sameIntents == main.FALSE:
2103 try:
2104 main.log.debug( "ONOS intents before: " )
2105 main.log.debug( json.dumps( json.loads( intentState ),
2106 sort_keys=True, indent=4,
2107 separators=( ',', ': ' ) ) )
2108 main.log.debug( "Current ONOS intents: " )
2109 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2110 sort_keys=True, indent=4,
2111 separators=( ',', ': ' ) ) )
2112 except ( ValueError, TypeError ):
2113 main.log.exception( "Exception printing intents" )
2114 main.log.debug( repr( ONOSIntents[0] ) )
2115 main.log.debug( repr( intentState ) )
2116 utilities.assert_equals(
2117 expect=main.TRUE,
2118 actual=sameIntents,
2119 onpass="Intents are consistent with before failure",
2120 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002121 intentCheck = intentCheck and sameIntents
2122
2123 main.step( "Get the OF Table entries and compare to before " +
2124 "component failure" )
2125 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002126 for i in range( 28 ):
2127 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002128 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002129 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2130 FlowTables = FlowTables and curSwitch
2131 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002132 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002133 utilities.assert_equals(
2134 expect=main.TRUE,
2135 actual=FlowTables,
2136 onpass="No changes were found in the flow tables",
2137 onfail="Changes were found in the flow tables" )
2138
2139 main.Mininet2.pingLongKill()
2140 '''
2141 main.step( "Check the continuous pings to ensure that no packets " +
2142 "were dropped during component failure" )
2143 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2144 main.params[ 'TESTONIP' ] )
2145 LossInPings = main.FALSE
2146 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2147 for i in range( 8, 18 ):
2148 main.log.info(
2149 "Checking for a loss in pings along flow from s" +
2150 str( i ) )
2151 LossInPings = main.Mininet2.checkForLoss(
2152 "/tmp/ping.h" +
2153 str( i ) ) or LossInPings
2154 if LossInPings == main.TRUE:
2155 main.log.info( "Loss in ping detected" )
2156 elif LossInPings == main.ERROR:
2157 main.log.info( "There are multiple mininet process running" )
2158 elif LossInPings == main.FALSE:
2159 main.log.info( "No Loss in the pings" )
2160 main.log.info( "No loss of dataplane connectivity" )
2161 utilities.assert_equals(
2162 expect=main.FALSE,
2163 actual=LossInPings,
2164 onpass="No Loss of connectivity",
2165 onfail="Loss of dataplane connectivity detected" )
2166 '''
2167
2168 main.step( "Leadership Election is still functional" )
2169 # Test of LeadershipElection
2170 leaderList = []
Jon Hall5cf14d52015-07-16 12:15:19 -07002171
Jon Hall3b489db2015-10-05 14:38:37 -07002172 restarted = []
2173 for i in main.kill:
2174 restarted.append( main.nodes[i].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002175 leaderResult = main.TRUE
Jon Hall3b489db2015-10-05 14:38:37 -07002176
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002177 for i in main.activeNodes:
2178 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002179 leaderN = cli.electionTestLeader()
2180 leaderList.append( leaderN )
2181 if leaderN == main.FALSE:
2182 # error in response
2183 main.log.error( "Something is wrong with " +
2184 "electionTestLeader function, check the" +
2185 " error logs" )
2186 leaderResult = main.FALSE
2187 elif leaderN is None:
2188 main.log.error( cli.name +
2189 " shows no leader for the election-app was" +
2190 " elected after the old one died" )
2191 leaderResult = main.FALSE
2192 elif leaderN in restarted:
2193 main.log.error( cli.name + " shows " + str( leaderN ) +
2194 " as leader for the election-app, but it " +
2195 "was restarted" )
2196 leaderResult = main.FALSE
2197 if len( set( leaderList ) ) != 1:
2198 leaderResult = main.FALSE
2199 main.log.error(
2200 "Inconsistent view of leader for the election test app" )
2201 # TODO: print the list
2202 utilities.assert_equals(
2203 expect=main.TRUE,
2204 actual=leaderResult,
2205 onpass="Leadership election passed",
2206 onfail="Something went wrong with Leadership election" )
2207
2208 def CASE8( self, main ):
2209 """
2210 Compare topo
2211 """
2212 import json
2213 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002214 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002215 assert main, "main not defined"
2216 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002217 assert main.CLIs, "main.CLIs not defined"
2218 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002219
2220 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002221 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002222 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002223 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002224 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002225 elapsed = 0
2226 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002227 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002228 startTime = time.time()
2229 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002230 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002231 devicesResults = main.TRUE
2232 linksResults = main.TRUE
2233 hostsResults = main.TRUE
2234 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002235 count += 1
2236 cliStart = time.time()
2237 devices = []
2238 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002239 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002240 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002241 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002242 args=[ main.CLIs[i].devices, [ None ] ],
2243 kwargs= { 'sleep': 5, 'attempts': 5,
2244 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002245 threads.append( t )
2246 t.start()
2247
2248 for t in threads:
2249 t.join()
2250 devices.append( t.result )
2251 hosts = []
2252 ipResult = main.TRUE
2253 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002254 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002255 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002256 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002257 args=[ main.CLIs[i].hosts, [ None ] ],
2258 kwargs= { 'sleep': 5, 'attempts': 5,
2259 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002260 threads.append( t )
2261 t.start()
2262
2263 for t in threads:
2264 t.join()
2265 try:
2266 hosts.append( json.loads( t.result ) )
2267 except ( ValueError, TypeError ):
2268 main.log.exception( "Error parsing hosts results" )
2269 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002270 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002271 for controller in range( 0, len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002272 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002273 if hosts[ controller ]:
2274 for host in hosts[ controller ]:
2275 if host is None or host.get( 'ipAddresses', [] ) == []:
2276 main.log.error(
2277 "Error with host ipAddresses on controller" +
2278 controllerStr + ": " + str( host ) )
2279 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002280 ports = []
2281 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002282 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002283 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002284 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002285 args=[ main.CLIs[i].ports, [ None ] ],
2286 kwargs= { 'sleep': 5, 'attempts': 5,
2287 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002288 threads.append( t )
2289 t.start()
2290
2291 for t in threads:
2292 t.join()
2293 ports.append( t.result )
2294 links = []
2295 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002296 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002297 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002298 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002299 args=[ main.CLIs[i].links, [ None ] ],
2300 kwargs= { 'sleep': 5, 'attempts': 5,
2301 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002302 threads.append( t )
2303 t.start()
2304
2305 for t in threads:
2306 t.join()
2307 links.append( t.result )
2308 clusters = []
2309 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002310 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002311 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002312 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002313 args=[ main.CLIs[i].clusters, [ None ] ],
2314 kwargs= { 'sleep': 5, 'attempts': 5,
2315 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002316 threads.append( t )
2317 t.start()
2318
2319 for t in threads:
2320 t.join()
2321 clusters.append( t.result )
2322
2323 elapsed = time.time() - startTime
2324 cliTime = time.time() - cliStart
2325 print "Elapsed time: " + str( elapsed )
2326 print "CLI time: " + str( cliTime )
2327
Jon Hall6e709752016-02-01 13:38:46 -08002328 if all( e is None for e in devices ) and\
2329 all( e is None for e in hosts ) and\
2330 all( e is None for e in ports ) and\
2331 all( e is None for e in links ) and\
2332 all( e is None for e in clusters ):
2333 topoFailMsg = "Could not get topology from ONOS"
2334 main.log.error( topoFailMsg )
2335 continue # Try again, No use trying to compare
2336
Jon Hall5cf14d52015-07-16 12:15:19 -07002337 mnSwitches = main.Mininet1.getSwitches()
2338 mnLinks = main.Mininet1.getLinks()
2339 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002340 for controller in range( len( main.activeNodes ) ):
2341 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002342 if devices[ controller ] and ports[ controller ] and\
2343 "Error" not in devices[ controller ] and\
2344 "Error" not in ports[ controller ]:
2345
Jon Hallc6793552016-01-19 14:18:37 -08002346 try:
2347 currentDevicesResult = main.Mininet1.compareSwitches(
2348 mnSwitches,
2349 json.loads( devices[ controller ] ),
2350 json.loads( ports[ controller ] ) )
2351 except ( TypeError, ValueError ) as e:
2352 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2353 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002354 else:
2355 currentDevicesResult = main.FALSE
2356 utilities.assert_equals( expect=main.TRUE,
2357 actual=currentDevicesResult,
2358 onpass="ONOS" + controllerStr +
2359 " Switches view is correct",
2360 onfail="ONOS" + controllerStr +
2361 " Switches view is incorrect" )
2362
2363 if links[ controller ] and "Error" not in links[ controller ]:
2364 currentLinksResult = main.Mininet1.compareLinks(
2365 mnSwitches, mnLinks,
2366 json.loads( links[ controller ] ) )
2367 else:
2368 currentLinksResult = main.FALSE
2369 utilities.assert_equals( expect=main.TRUE,
2370 actual=currentLinksResult,
2371 onpass="ONOS" + controllerStr +
2372 " links view is correct",
2373 onfail="ONOS" + controllerStr +
2374 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002375 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002376 currentHostsResult = main.Mininet1.compareHosts(
2377 mnHosts,
2378 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002379 elif hosts[ controller ] == []:
2380 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002381 else:
2382 currentHostsResult = main.FALSE
2383 utilities.assert_equals( expect=main.TRUE,
2384 actual=currentHostsResult,
2385 onpass="ONOS" + controllerStr +
2386 " hosts exist in Mininet",
2387 onfail="ONOS" + controllerStr +
2388 " hosts don't match Mininet" )
2389 # CHECKING HOST ATTACHMENT POINTS
2390 hostAttachment = True
2391 zeroHosts = False
2392 # FIXME: topo-HA/obelisk specific mappings:
2393 # key is mac and value is dpid
2394 mappings = {}
2395 for i in range( 1, 29 ): # hosts 1 through 28
2396 # set up correct variables:
2397 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2398 if i == 1:
2399 deviceId = "1000".zfill(16)
2400 elif i == 2:
2401 deviceId = "2000".zfill(16)
2402 elif i == 3:
2403 deviceId = "3000".zfill(16)
2404 elif i == 4:
2405 deviceId = "3004".zfill(16)
2406 elif i == 5:
2407 deviceId = "5000".zfill(16)
2408 elif i == 6:
2409 deviceId = "6000".zfill(16)
2410 elif i == 7:
2411 deviceId = "6007".zfill(16)
2412 elif i >= 8 and i <= 17:
2413 dpid = '3' + str( i ).zfill( 3 )
2414 deviceId = dpid.zfill(16)
2415 elif i >= 18 and i <= 27:
2416 dpid = '6' + str( i ).zfill( 3 )
2417 deviceId = dpid.zfill(16)
2418 elif i == 28:
2419 deviceId = "2800".zfill(16)
2420 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002421 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002422 if hosts[ controller ] == []:
2423 main.log.warn( "There are no hosts discovered" )
2424 zeroHosts = True
2425 else:
2426 for host in hosts[ controller ]:
2427 mac = None
2428 location = None
2429 device = None
2430 port = None
2431 try:
2432 mac = host.get( 'mac' )
2433 assert mac, "mac field could not be found for this host object"
2434
2435 location = host.get( 'location' )
2436 assert location, "location field could not be found for this host object"
2437
2438 # Trim the protocol identifier off deviceId
2439 device = str( location.get( 'elementId' ) ).split(':')[1]
2440 assert device, "elementId field could not be found for this host location object"
2441
2442 port = location.get( 'port' )
2443 assert port, "port field could not be found for this host location object"
2444
2445 # Now check if this matches where they should be
2446 if mac and device and port:
2447 if str( port ) != "1":
2448 main.log.error( "The attachment port is incorrect for " +
2449 "host " + str( mac ) +
2450 ". Expected: 1 Actual: " + str( port) )
2451 hostAttachment = False
2452 if device != mappings[ str( mac ) ]:
2453 main.log.error( "The attachment device is incorrect for " +
2454 "host " + str( mac ) +
2455 ". Expected: " + mappings[ str( mac ) ] +
2456 " Actual: " + device )
2457 hostAttachment = False
2458 else:
2459 hostAttachment = False
2460 except AssertionError:
2461 main.log.exception( "Json object not as expected" )
2462 main.log.error( repr( host ) )
2463 hostAttachment = False
2464 else:
2465 main.log.error( "No hosts json output or \"Error\"" +
2466 " in output. hosts = " +
2467 repr( hosts[ controller ] ) )
2468 if zeroHosts is False:
2469 hostAttachment = True
2470
2471 # END CHECKING HOST ATTACHMENT POINTS
2472 devicesResults = devicesResults and currentDevicesResult
2473 linksResults = linksResults and currentLinksResult
2474 hostsResults = hostsResults and currentHostsResult
2475 hostAttachmentResults = hostAttachmentResults and\
2476 hostAttachment
Jon Halla440e872016-03-31 15:15:50 -07002477 topoResult = ( devicesResults and linksResults
2478 and hostsResults and ipResult and
2479 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002480 utilities.assert_equals( expect=True,
2481 actual=topoResult,
2482 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002483 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002484 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002485
2486 # Compare json objects for hosts and dataplane clusters
2487
2488 # hosts
2489 main.step( "Hosts view is consistent across all ONOS nodes" )
2490 consistentHostsResult = main.TRUE
2491 for controller in range( len( hosts ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002492 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002493 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002494 if hosts[ controller ] == hosts[ 0 ]:
2495 continue
2496 else: # hosts not consistent
2497 main.log.error( "hosts from ONOS" + controllerStr +
2498 " is inconsistent with ONOS1" )
2499 main.log.warn( repr( hosts[ controller ] ) )
2500 consistentHostsResult = main.FALSE
2501
2502 else:
2503 main.log.error( "Error in getting ONOS hosts from ONOS" +
2504 controllerStr )
2505 consistentHostsResult = main.FALSE
2506 main.log.warn( "ONOS" + controllerStr +
2507 " hosts response: " +
2508 repr( hosts[ controller ] ) )
2509 utilities.assert_equals(
2510 expect=main.TRUE,
2511 actual=consistentHostsResult,
2512 onpass="Hosts view is consistent across all ONOS nodes",
2513 onfail="ONOS nodes have different views of hosts" )
2514
2515 main.step( "Hosts information is correct" )
2516 hostsResults = hostsResults and ipResult
2517 utilities.assert_equals(
2518 expect=main.TRUE,
2519 actual=hostsResults,
2520 onpass="Host information is correct",
2521 onfail="Host information is incorrect" )
2522
2523 main.step( "Host attachment points to the network" )
2524 utilities.assert_equals(
2525 expect=True,
2526 actual=hostAttachmentResults,
2527 onpass="Hosts are correctly attached to the network",
2528 onfail="ONOS did not correctly attach hosts to the network" )
2529
2530 # Strongly connected clusters of devices
2531 main.step( "Clusters view is consistent across all ONOS nodes" )
2532 consistentClustersResult = main.TRUE
2533 for controller in range( len( clusters ) ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002534 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002535 if "Error" not in clusters[ controller ]:
2536 if clusters[ controller ] == clusters[ 0 ]:
2537 continue
2538 else: # clusters not consistent
2539 main.log.error( "clusters from ONOS" +
2540 controllerStr +
2541 " is inconsistent with ONOS1" )
2542 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002543 else:
2544 main.log.error( "Error in getting dataplane clusters " +
2545 "from ONOS" + controllerStr )
2546 consistentClustersResult = main.FALSE
2547 main.log.warn( "ONOS" + controllerStr +
2548 " clusters response: " +
2549 repr( clusters[ controller ] ) )
2550 utilities.assert_equals(
2551 expect=main.TRUE,
2552 actual=consistentClustersResult,
2553 onpass="Clusters view is consistent across all ONOS nodes",
2554 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002555 if not consistentClustersResult:
2556 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002557
2558 main.step( "There is only one SCC" )
2559 # there should always only be one cluster
2560 try:
2561 numClusters = len( json.loads( clusters[ 0 ] ) )
2562 except ( ValueError, TypeError ):
2563 main.log.exception( "Error parsing clusters[0]: " +
2564 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002565 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002566 clusterResults = main.FALSE
2567 if numClusters == 1:
2568 clusterResults = main.TRUE
2569 utilities.assert_equals(
2570 expect=1,
2571 actual=numClusters,
2572 onpass="ONOS shows 1 SCC",
2573 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2574
2575 topoResult = ( devicesResults and linksResults
2576 and hostsResults and consistentHostsResult
2577 and consistentClustersResult and clusterResults
2578 and ipResult and hostAttachmentResults )
2579
2580 topoResult = topoResult and int( count <= 2 )
2581 note = "note it takes about " + str( int( cliTime ) ) + \
2582 " seconds for the test to make all the cli calls to fetch " +\
2583 "the topology from each ONOS instance"
2584 main.log.info(
2585 "Very crass estimate for topology discovery/convergence( " +
2586 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2587 str( count ) + " tries" )
2588
2589 main.step( "Device information is correct" )
2590 utilities.assert_equals(
2591 expect=main.TRUE,
2592 actual=devicesResults,
2593 onpass="Device information is correct",
2594 onfail="Device information is incorrect" )
2595
2596 main.step( "Links are correct" )
2597 utilities.assert_equals(
2598 expect=main.TRUE,
2599 actual=linksResults,
2600 onpass="Link are correct",
2601 onfail="Links are incorrect" )
2602
Jon Halla440e872016-03-31 15:15:50 -07002603 main.step( "Hosts are correct" )
2604 utilities.assert_equals(
2605 expect=main.TRUE,
2606 actual=hostsResults,
2607 onpass="Hosts are correct",
2608 onfail="Hosts are incorrect" )
2609
Jon Hall5cf14d52015-07-16 12:15:19 -07002610 # FIXME: move this to an ONOS state case
2611 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002612 nodeResults = utilities.retry( main.HA.nodesCheck,
2613 False,
2614 args=[main.activeNodes],
2615 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002616
Jon Hall41d39f12016-04-11 22:54:35 -07002617 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002618 onpass="Nodes check successful",
2619 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002620 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002621 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002622 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002623 main.CLIs[i].name,
2624 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002625
2626 def CASE9( self, main ):
2627 """
2628 Link s3-s28 down
2629 """
2630 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002631 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002632 assert main, "main not defined"
2633 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002634 assert main.CLIs, "main.CLIs not defined"
2635 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002636 # NOTE: You should probably run a topology check after this
2637
2638 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2639
2640 description = "Turn off a link to ensure that Link Discovery " +\
2641 "is working properly"
2642 main.case( description )
2643
2644 main.step( "Kill Link between s3 and s28" )
2645 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2646 main.log.info( "Waiting " + str( linkSleep ) +
2647 " seconds for link down to be discovered" )
2648 time.sleep( linkSleep )
2649 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2650 onpass="Link down successful",
2651 onfail="Failed to bring link down" )
2652 # TODO do some sort of check here
2653
2654 def CASE10( self, main ):
2655 """
2656 Link s3-s28 up
2657 """
2658 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002659 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002660 assert main, "main not defined"
2661 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002662 assert main.CLIs, "main.CLIs not defined"
2663 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002664 # NOTE: You should probably run a topology check after this
2665
2666 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2667
2668 description = "Restore a link to ensure that Link Discovery is " + \
2669 "working properly"
2670 main.case( description )
2671
2672 main.step( "Bring link between s3 and s28 back up" )
2673 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2674 main.log.info( "Waiting " + str( linkSleep ) +
2675 " seconds for link up to be discovered" )
2676 time.sleep( linkSleep )
2677 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2678 onpass="Link up successful",
2679 onfail="Failed to bring link up" )
2680 # TODO do some sort of check here
2681
2682 def CASE11( self, main ):
2683 """
2684 Switch Down
2685 """
2686 # NOTE: You should probably run a topology check after this
2687 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002688 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002689 assert main, "main not defined"
2690 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002691 assert main.CLIs, "main.CLIs not defined"
2692 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002693
2694 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2695
2696 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002697 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002698 main.case( description )
2699 switch = main.params[ 'kill' ][ 'switch' ]
2700 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2701
2702 # TODO: Make this switch parameterizable
2703 main.step( "Kill " + switch )
2704 main.log.info( "Deleting " + switch )
2705 main.Mininet1.delSwitch( switch )
2706 main.log.info( "Waiting " + str( switchSleep ) +
2707 " seconds for switch down to be discovered" )
2708 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002709 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002710 # Peek at the deleted switch
2711 main.log.warn( str( device ) )
2712 result = main.FALSE
2713 if device and device[ 'available' ] is False:
2714 result = main.TRUE
2715 utilities.assert_equals( expect=main.TRUE, actual=result,
2716 onpass="Kill switch successful",
2717 onfail="Failed to kill switch?" )
2718
2719 def CASE12( self, main ):
2720 """
2721 Switch Up
2722 """
2723 # NOTE: You should probably run a topology check after this
2724 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002725 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002726 assert main, "main not defined"
2727 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002728 assert main.CLIs, "main.CLIs not defined"
2729 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002730 assert ONOS1Port, "ONOS1Port not defined"
2731 assert ONOS2Port, "ONOS2Port not defined"
2732 assert ONOS3Port, "ONOS3Port not defined"
2733 assert ONOS4Port, "ONOS4Port not defined"
2734 assert ONOS5Port, "ONOS5Port not defined"
2735 assert ONOS6Port, "ONOS6Port not defined"
2736 assert ONOS7Port, "ONOS7Port not defined"
2737
2738 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2739 switch = main.params[ 'kill' ][ 'switch' ]
2740 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2741 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002742 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002743 description = "Adding a switch to ensure it is discovered correctly"
2744 main.case( description )
2745
2746 main.step( "Add back " + switch )
2747 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2748 for peer in links:
2749 main.Mininet1.addLink( switch, peer )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002750 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002751 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2752 main.log.info( "Waiting " + str( switchSleep ) +
2753 " seconds for switch up to be discovered" )
2754 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002755 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002756 # Peek at the deleted switch
2757 main.log.warn( str( device ) )
2758 result = main.FALSE
2759 if device and device[ 'available' ]:
2760 result = main.TRUE
2761 utilities.assert_equals( expect=main.TRUE, actual=result,
2762 onpass="add switch successful",
2763 onfail="Failed to add switch?" )
2764
2765 def CASE13( self, main ):
2766 """
2767 Clean up
2768 """
2769 import os
2770 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002771 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002772 assert main, "main not defined"
2773 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002774 assert main.CLIs, "main.CLIs not defined"
2775 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002776
2777 # printing colors to terminal
2778 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2779 'blue': '\033[94m', 'green': '\033[92m',
2780 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2781 main.case( "Test Cleanup" )
2782 main.step( "Killing tcpdumps" )
2783 main.Mininet2.stopTcpdump()
2784
2785 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002786 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002787 main.step( "Copying MN pcap and ONOS log files to test station" )
2788 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2789 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002790 # NOTE: MN Pcap file is being saved to logdir.
2791 # We scp this file as MN and TestON aren't necessarily the same vm
2792
2793 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002794 # TODO: Load these from params
2795 # NOTE: must end in /
2796 logFolder = "/opt/onos/log/"
2797 logFiles = [ "karaf.log", "karaf.log.1" ]
2798 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002799 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002800 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002801 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002802 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2803 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002804 # std*.log's
2805 # NOTE: must end in /
2806 logFolder = "/opt/onos/var/"
2807 logFiles = [ "stderr.log", "stdout.log" ]
2808 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002809 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002810 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002811 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002812 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2813 logFolder + f, dstName )
2814 else:
2815 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002816
2817 main.step( "Stopping Mininet" )
2818 mnResult = main.Mininet1.stopNet()
2819 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2820 onpass="Mininet stopped",
2821 onfail="MN cleanup NOT successful" )
2822
2823 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002824 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002825 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2826 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002827
2828 try:
2829 timerLog = open( main.logdir + "/Timers.csv", 'w')
2830 # Overwrite with empty line and close
2831 labels = "Gossip Intents, Restart"
2832 data = str( gossipTime ) + ", " + str( main.restartTime )
2833 timerLog.write( labels + "\n" + data )
2834 timerLog.close()
2835 except NameError, e:
2836 main.log.exception(e)
2837
2838 def CASE14( self, main ):
2839 """
2840 start election app on all onos nodes
2841 """
Jon Halle1a3b752015-07-22 13:02:46 -07002842 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002843 assert main, "main not defined"
2844 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002845 assert main.CLIs, "main.CLIs not defined"
2846 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002847
2848 main.case("Start Leadership Election app")
2849 main.step( "Install leadership election app" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002850 onosCli = main.CLIs[ main.activeNodes[0] ]
2851 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002852 utilities.assert_equals(
2853 expect=main.TRUE,
2854 actual=appResult,
2855 onpass="Election app installed",
2856 onfail="Something went wrong with installing Leadership election" )
2857
2858 main.step( "Run for election on each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002859 for i in main.activeNodes:
2860 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002861 time.sleep(5)
2862 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2863 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002864 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002865 expect=True,
2866 actual=sameResult,
2867 onpass="All nodes see the same leaderboards",
2868 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002869
Jon Hall25463a82016-04-13 14:03:52 -07002870 if sameResult:
2871 leader = leaders[ 0 ][ 0 ]
2872 if main.nodes[main.activeNodes[0]].ip_address in leader:
2873 correctLeader = True
2874 else:
2875 correctLeader = False
2876 main.step( "First node was elected leader" )
2877 utilities.assert_equals(
2878 expect=True,
2879 actual=correctLeader,
2880 onpass="Correct leader was elected",
2881 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002882
2883 def CASE15( self, main ):
2884 """
2885 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002886 15.1 Run election on each node
2887 15.2 Check that each node has the same leaders and candidates
2888 15.3 Find current leader and withdraw
2889 15.4 Check that a new node was elected leader
2890 15.5 Check that that new leader was the candidate of old leader
2891 15.6 Run for election on old leader
2892 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2893 15.8 Make sure that the old leader was added to the candidate list
2894
2895 old and new variable prefixes refer to data from before vs after
2896 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002897 """
2898 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002899 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002900 assert main, "main not defined"
2901 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002902 assert main.CLIs, "main.CLIs not defined"
2903 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002904
Jon Hall5cf14d52015-07-16 12:15:19 -07002905 description = "Check that Leadership Election is still functional"
2906 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002907 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002908
Jon Halla440e872016-03-31 15:15:50 -07002909 oldLeaders = [] # list of lists of each nodes' candidates before
2910 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002911 oldLeader = '' # the old leader from oldLeaders, None if not same
2912 newLeader = '' # the new leaders fron newLoeaders, None if not same
2913 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2914 expectNoLeader = False # True when there is only one leader
2915 if main.numCtrls == 1:
2916 expectNoLeader = True
2917
2918 main.step( "Run for election on each node" )
2919 electionResult = main.TRUE
2920
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002921 for i in main.activeNodes: # run test election on each node
2922 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002923 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002924 utilities.assert_equals(
2925 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002926 actual=electionResult,
2927 onpass="All nodes successfully ran for leadership",
2928 onfail="At least one node failed to run for leadership" )
2929
acsmars3a72bde2015-09-02 14:16:22 -07002930 if electionResult == main.FALSE:
2931 main.log.error(
2932 "Skipping Test Case because Election Test App isn't loaded" )
2933 main.skipCase()
2934
acsmars71adceb2015-08-31 15:09:26 -07002935 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002936 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002937 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002938 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002939 if sameResult:
2940 oldLeader = oldLeaders[ 0 ][ 0 ]
2941 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07002942 else:
Jon Halla440e872016-03-31 15:15:50 -07002943 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002944 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002945 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002946 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002947 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002948 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002949
2950 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002951 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002952 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002953 if oldLeader is None:
2954 main.log.error( "Leadership isn't consistent." )
2955 withdrawResult = main.FALSE
2956 # Get the CLI of the oldLeader
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002957 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002958 if oldLeader == main.nodes[ i ].ip_address:
2959 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002960 break
2961 else: # FOR/ELSE statement
2962 main.log.error( "Leader election, could not find current leader" )
2963 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002964 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002965 utilities.assert_equals(
2966 expect=main.TRUE,
2967 actual=withdrawResult,
2968 onpass="Node was withdrawn from election",
2969 onfail="Node was not withdrawn from election" )
2970
acsmars71adceb2015-08-31 15:09:26 -07002971 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002972 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07002973 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002974 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002975 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002976 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002977 if newLeaders[ 0 ][ 0 ] == 'none':
2978 main.log.error( "No leader was elected on at least 1 node" )
2979 if not expectNoLeader:
2980 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002981 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002982
2983 # Check that the new leader is not the older leader, which was withdrawn
2984 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002985 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08002986 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07002987 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002988 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002989 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002990 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002991 onpass="Leadership election passed",
2992 onfail="Something went wrong with Leadership election" )
2993
Jon Halla440e872016-03-31 15:15:50 -07002994 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08002995 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07002996 correctCandidateResult = main.TRUE
2997 if expectNoLeader:
2998 if newLeader == 'none':
2999 main.log.info( "No leader expected. None found. Pass" )
3000 correctCandidateResult = main.TRUE
3001 else:
3002 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3003 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003004 elif len( oldLeaders[0] ) >= 3:
3005 if newLeader == oldLeaders[ 0 ][ 2 ]:
3006 # correct leader was elected
3007 correctCandidateResult = main.TRUE
3008 else:
3009 correctCandidateResult = main.FALSE
3010 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3011 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003012 else:
3013 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003014 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003015 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07003016 utilities.assert_equals(
3017 expect=main.TRUE,
3018 actual=correctCandidateResult,
3019 onpass="Correct Candidate Elected",
3020 onfail="Incorrect Candidate Elected" )
3021
Jon Hall5cf14d52015-07-16 12:15:19 -07003022 main.step( "Run for election on old leader( just so everyone " +
3023 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07003024 if oldLeaderCLI is not None:
3025 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003026 else:
acsmars71adceb2015-08-31 15:09:26 -07003027 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003028 runResult = main.FALSE
3029 utilities.assert_equals(
3030 expect=main.TRUE,
3031 actual=runResult,
3032 onpass="App re-ran for election",
3033 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003034
acsmars71adceb2015-08-31 15:09:26 -07003035 main.step(
3036 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003037 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003038 # Get new leaders and candidates
3039 reRunLeaders = []
3040 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003041 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003042
3043 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07003044 if not reRunLeaders[0]:
3045 positionResult = main.FALSE
3046 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07003047 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3048 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07003049 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003050 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003051 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003052 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003053 onpass="Old leader successfully re-ran for election",
3054 onfail="Something went wrong with Leadership election after " +
3055 "the old leader re-ran for election" )
3056
3057 def CASE16( self, main ):
3058 """
3059 Install Distributed Primitives app
3060 """
3061 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003062 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003063 assert main, "main not defined"
3064 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003065 assert main.CLIs, "main.CLIs not defined"
3066 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003067
3068 # Variables for the distributed primitives tests
3069 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003070 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003071 global onosSet
3072 global onosSetName
3073 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003074 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003075 onosSet = set([])
3076 onosSetName = "TestON-set"
3077
3078 description = "Install Primitives app"
3079 main.case( description )
3080 main.step( "Install Primitives app" )
3081 appName = "org.onosproject.distributedprimitives"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003082 node = main.activeNodes[0]
3083 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003084 utilities.assert_equals( expect=main.TRUE,
3085 actual=appResults,
3086 onpass="Primitives app activated",
3087 onfail="Primitives app not activated" )
3088 time.sleep( 5 ) # To allow all nodes to activate
3089
3090 def CASE17( self, main ):
3091 """
3092 Check for basic functionality with distributed primitives
3093 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003094 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003095 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003096 assert main, "main not defined"
3097 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003098 assert main.CLIs, "main.CLIs not defined"
3099 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003100 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003101 assert onosSetName, "onosSetName not defined"
3102 # NOTE: assert fails if value is 0/None/Empty/False
3103 try:
3104 pCounterValue
3105 except NameError:
3106 main.log.error( "pCounterValue not defined, setting to 0" )
3107 pCounterValue = 0
3108 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003109 onosSet
3110 except NameError:
3111 main.log.error( "onosSet not defined, setting to empty Set" )
3112 onosSet = set([])
3113 # Variables for the distributed primitives tests. These are local only
3114 addValue = "a"
3115 addAllValue = "a b c d e f"
3116 retainValue = "c d e f"
3117
3118 description = "Check for basic functionality with distributed " +\
3119 "primitives"
3120 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003121 main.caseExplanation = "Test the methods of the distributed " +\
3122 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003123 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003124 # Partitioned counters
3125 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003126 pCounters = []
3127 threads = []
3128 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003129 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003130 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3131 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003132 args=[ pCounterName ] )
3133 pCounterValue += 1
3134 addedPValues.append( pCounterValue )
3135 threads.append( t )
3136 t.start()
3137
3138 for t in threads:
3139 t.join()
3140 pCounters.append( t.result )
3141 # Check that counter incremented numController times
3142 pCounterResults = True
3143 for i in addedPValues:
3144 tmpResult = i in pCounters
3145 pCounterResults = pCounterResults and tmpResult
3146 if not tmpResult:
3147 main.log.error( str( i ) + " is not in partitioned "
3148 "counter incremented results" )
3149 utilities.assert_equals( expect=True,
3150 actual=pCounterResults,
3151 onpass="Default counter incremented",
3152 onfail="Error incrementing default" +
3153 " counter" )
3154
Jon Halle1a3b752015-07-22 13:02:46 -07003155 main.step( "Get then Increment a default counter on each node" )
3156 pCounters = []
3157 threads = []
3158 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003159 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003160 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3161 name="counterGetAndAdd-" + str( i ),
3162 args=[ pCounterName ] )
3163 addedPValues.append( pCounterValue )
3164 pCounterValue += 1
3165 threads.append( t )
3166 t.start()
3167
3168 for t in threads:
3169 t.join()
3170 pCounters.append( t.result )
3171 # Check that counter incremented numController times
3172 pCounterResults = True
3173 for i in addedPValues:
3174 tmpResult = i in pCounters
3175 pCounterResults = pCounterResults and tmpResult
3176 if not tmpResult:
3177 main.log.error( str( i ) + " is not in partitioned "
3178 "counter incremented results" )
3179 utilities.assert_equals( expect=True,
3180 actual=pCounterResults,
3181 onpass="Default counter incremented",
3182 onfail="Error incrementing default" +
3183 " counter" )
3184
3185 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003186 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003187 utilities.assert_equals( expect=main.TRUE,
3188 actual=incrementCheck,
3189 onpass="Added counters are correct",
3190 onfail="Added counters are incorrect" )
3191
3192 main.step( "Add -8 to then get a default counter on each node" )
3193 pCounters = []
3194 threads = []
3195 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003196 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003197 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3198 name="counterIncrement-" + str( i ),
3199 args=[ pCounterName ],
3200 kwargs={ "delta": -8 } )
3201 pCounterValue += -8
3202 addedPValues.append( pCounterValue )
3203 threads.append( t )
3204 t.start()
3205
3206 for t in threads:
3207 t.join()
3208 pCounters.append( t.result )
3209 # Check that counter incremented numController times
3210 pCounterResults = True
3211 for i in addedPValues:
3212 tmpResult = i in pCounters
3213 pCounterResults = pCounterResults and tmpResult
3214 if not tmpResult:
3215 main.log.error( str( i ) + " is not in partitioned "
3216 "counter incremented results" )
3217 utilities.assert_equals( expect=True,
3218 actual=pCounterResults,
3219 onpass="Default counter incremented",
3220 onfail="Error incrementing default" +
3221 " counter" )
3222
3223 main.step( "Add 5 to then get a default counter on each node" )
3224 pCounters = []
3225 threads = []
3226 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003227 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003228 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3229 name="counterIncrement-" + str( i ),
3230 args=[ pCounterName ],
3231 kwargs={ "delta": 5 } )
3232 pCounterValue += 5
3233 addedPValues.append( pCounterValue )
3234 threads.append( t )
3235 t.start()
3236
3237 for t in threads:
3238 t.join()
3239 pCounters.append( t.result )
3240 # Check that counter incremented numController times
3241 pCounterResults = True
3242 for i in addedPValues:
3243 tmpResult = i in pCounters
3244 pCounterResults = pCounterResults and tmpResult
3245 if not tmpResult:
3246 main.log.error( str( i ) + " is not in partitioned "
3247 "counter incremented results" )
3248 utilities.assert_equals( expect=True,
3249 actual=pCounterResults,
3250 onpass="Default counter incremented",
3251 onfail="Error incrementing default" +
3252 " counter" )
3253
3254 main.step( "Get then add 5 to a default counter on each node" )
3255 pCounters = []
3256 threads = []
3257 addedPValues = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003258 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003259 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3260 name="counterIncrement-" + str( i ),
3261 args=[ pCounterName ],
3262 kwargs={ "delta": 5 } )
3263 addedPValues.append( pCounterValue )
3264 pCounterValue += 5
3265 threads.append( t )
3266 t.start()
3267
3268 for t in threads:
3269 t.join()
3270 pCounters.append( t.result )
3271 # Check that counter incremented numController times
3272 pCounterResults = True
3273 for i in addedPValues:
3274 tmpResult = i in pCounters
3275 pCounterResults = pCounterResults and tmpResult
3276 if not tmpResult:
3277 main.log.error( str( i ) + " is not in partitioned "
3278 "counter incremented results" )
3279 utilities.assert_equals( expect=True,
3280 actual=pCounterResults,
3281 onpass="Default counter incremented",
3282 onfail="Error incrementing default" +
3283 " counter" )
3284
3285 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003286 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003287 utilities.assert_equals( expect=main.TRUE,
3288 actual=incrementCheck,
3289 onpass="Added counters are correct",
3290 onfail="Added counters are incorrect" )
3291
Jon Hall5cf14d52015-07-16 12:15:19 -07003292 # DISTRIBUTED SETS
3293 main.step( "Distributed Set get" )
3294 size = len( onosSet )
3295 getResponses = []
3296 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003297 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003298 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003299 name="setTestGet-" + str( i ),
3300 args=[ onosSetName ] )
3301 threads.append( t )
3302 t.start()
3303 for t in threads:
3304 t.join()
3305 getResponses.append( t.result )
3306
3307 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003308 for i in range( len( main.activeNodes ) ):
3309 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003310 if isinstance( getResponses[ i ], list):
3311 current = set( getResponses[ i ] )
3312 if len( current ) == len( getResponses[ i ] ):
3313 # no repeats
3314 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003315 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003316 " has incorrect view" +
3317 " of set " + onosSetName + ":\n" +
3318 str( getResponses[ i ] ) )
3319 main.log.debug( "Expected: " + str( onosSet ) )
3320 main.log.debug( "Actual: " + str( current ) )
3321 getResults = main.FALSE
3322 else:
3323 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003324 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003325 " has repeat elements in" +
3326 " set " + onosSetName + ":\n" +
3327 str( getResponses[ i ] ) )
3328 getResults = main.FALSE
3329 elif getResponses[ i ] == main.ERROR:
3330 getResults = main.FALSE
3331 utilities.assert_equals( expect=main.TRUE,
3332 actual=getResults,
3333 onpass="Set elements are correct",
3334 onfail="Set elements are incorrect" )
3335
3336 main.step( "Distributed Set size" )
3337 sizeResponses = []
3338 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003339 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003340 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003341 name="setTestSize-" + str( i ),
3342 args=[ onosSetName ] )
3343 threads.append( t )
3344 t.start()
3345 for t in threads:
3346 t.join()
3347 sizeResponses.append( t.result )
3348
3349 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003350 for i in range( len( main.activeNodes ) ):
3351 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003352 if size != sizeResponses[ i ]:
3353 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003354 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003355 " expected a size of " + str( size ) +
3356 " for set " + onosSetName +
3357 " but got " + str( sizeResponses[ i ] ) )
3358 utilities.assert_equals( expect=main.TRUE,
3359 actual=sizeResults,
3360 onpass="Set sizes are correct",
3361 onfail="Set sizes are incorrect" )
3362
3363 main.step( "Distributed Set add()" )
3364 onosSet.add( addValue )
3365 addResponses = []
3366 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003367 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003368 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003369 name="setTestAdd-" + str( i ),
3370 args=[ onosSetName, addValue ] )
3371 threads.append( t )
3372 t.start()
3373 for t in threads:
3374 t.join()
3375 addResponses.append( t.result )
3376
3377 # main.TRUE = successfully changed the set
3378 # main.FALSE = action resulted in no change in set
3379 # main.ERROR - Some error in executing the function
3380 addResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003381 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003382 if addResponses[ i ] == main.TRUE:
3383 # All is well
3384 pass
3385 elif addResponses[ i ] == main.FALSE:
3386 # Already in set, probably fine
3387 pass
3388 elif addResponses[ i ] == main.ERROR:
3389 # Error in execution
3390 addResults = main.FALSE
3391 else:
3392 # unexpected result
3393 addResults = main.FALSE
3394 if addResults != main.TRUE:
3395 main.log.error( "Error executing set add" )
3396
3397 # Check if set is still correct
3398 size = len( onosSet )
3399 getResponses = []
3400 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003401 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003402 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003403 name="setTestGet-" + str( i ),
3404 args=[ onosSetName ] )
3405 threads.append( t )
3406 t.start()
3407 for t in threads:
3408 t.join()
3409 getResponses.append( t.result )
3410 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003411 for i in range( len( main.activeNodes ) ):
3412 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003413 if isinstance( getResponses[ i ], list):
3414 current = set( getResponses[ i ] )
3415 if len( current ) == len( getResponses[ i ] ):
3416 # no repeats
3417 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003418 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003419 " of set " + onosSetName + ":\n" +
3420 str( getResponses[ i ] ) )
3421 main.log.debug( "Expected: " + str( onosSet ) )
3422 main.log.debug( "Actual: " + str( current ) )
3423 getResults = main.FALSE
3424 else:
3425 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003426 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003427 " set " + onosSetName + ":\n" +
3428 str( getResponses[ i ] ) )
3429 getResults = main.FALSE
3430 elif getResponses[ i ] == main.ERROR:
3431 getResults = main.FALSE
3432 sizeResponses = []
3433 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003434 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003435 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003436 name="setTestSize-" + str( i ),
3437 args=[ onosSetName ] )
3438 threads.append( t )
3439 t.start()
3440 for t in threads:
3441 t.join()
3442 sizeResponses.append( t.result )
3443 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003444 for i in range( len( main.activeNodes ) ):
3445 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003446 if size != sizeResponses[ i ]:
3447 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003448 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003449 " expected a size of " + str( size ) +
3450 " for set " + onosSetName +
3451 " but got " + str( sizeResponses[ i ] ) )
3452 addResults = addResults and getResults and sizeResults
3453 utilities.assert_equals( expect=main.TRUE,
3454 actual=addResults,
3455 onpass="Set add correct",
3456 onfail="Set add was incorrect" )
3457
3458 main.step( "Distributed Set addAll()" )
3459 onosSet.update( addAllValue.split() )
3460 addResponses = []
3461 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003462 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003463 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003464 name="setTestAddAll-" + str( i ),
3465 args=[ onosSetName, addAllValue ] )
3466 threads.append( t )
3467 t.start()
3468 for t in threads:
3469 t.join()
3470 addResponses.append( t.result )
3471
3472 # main.TRUE = successfully changed the set
3473 # main.FALSE = action resulted in no change in set
3474 # main.ERROR - Some error in executing the function
3475 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003476 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003477 if addResponses[ i ] == main.TRUE:
3478 # All is well
3479 pass
3480 elif addResponses[ i ] == main.FALSE:
3481 # Already in set, probably fine
3482 pass
3483 elif addResponses[ i ] == main.ERROR:
3484 # Error in execution
3485 addAllResults = main.FALSE
3486 else:
3487 # unexpected result
3488 addAllResults = main.FALSE
3489 if addAllResults != main.TRUE:
3490 main.log.error( "Error executing set addAll" )
3491
3492 # Check if set is still correct
3493 size = len( onosSet )
3494 getResponses = []
3495 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003496 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003497 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003498 name="setTestGet-" + str( i ),
3499 args=[ onosSetName ] )
3500 threads.append( t )
3501 t.start()
3502 for t in threads:
3503 t.join()
3504 getResponses.append( t.result )
3505 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003506 for i in range( len( main.activeNodes ) ):
3507 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003508 if isinstance( getResponses[ i ], list):
3509 current = set( getResponses[ i ] )
3510 if len( current ) == len( getResponses[ i ] ):
3511 # no repeats
3512 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003513 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003514 " has incorrect view" +
3515 " of set " + onosSetName + ":\n" +
3516 str( getResponses[ i ] ) )
3517 main.log.debug( "Expected: " + str( onosSet ) )
3518 main.log.debug( "Actual: " + str( current ) )
3519 getResults = main.FALSE
3520 else:
3521 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003522 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003523 " has repeat elements in" +
3524 " set " + onosSetName + ":\n" +
3525 str( getResponses[ i ] ) )
3526 getResults = main.FALSE
3527 elif getResponses[ i ] == main.ERROR:
3528 getResults = main.FALSE
3529 sizeResponses = []
3530 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003531 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003532 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003533 name="setTestSize-" + str( i ),
3534 args=[ onosSetName ] )
3535 threads.append( t )
3536 t.start()
3537 for t in threads:
3538 t.join()
3539 sizeResponses.append( t.result )
3540 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003541 for i in range( len( main.activeNodes ) ):
3542 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003543 if size != sizeResponses[ i ]:
3544 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003545 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003546 " expected a size of " + str( size ) +
3547 " for set " + onosSetName +
3548 " but got " + str( sizeResponses[ i ] ) )
3549 addAllResults = addAllResults and getResults and sizeResults
3550 utilities.assert_equals( expect=main.TRUE,
3551 actual=addAllResults,
3552 onpass="Set addAll correct",
3553 onfail="Set addAll was incorrect" )
3554
3555 main.step( "Distributed Set contains()" )
3556 containsResponses = []
3557 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003558 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003559 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003560 name="setContains-" + str( i ),
3561 args=[ onosSetName ],
3562 kwargs={ "values": addValue } )
3563 threads.append( t )
3564 t.start()
3565 for t in threads:
3566 t.join()
3567 # NOTE: This is the tuple
3568 containsResponses.append( t.result )
3569
3570 containsResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003571 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003572 if containsResponses[ i ] == main.ERROR:
3573 containsResults = main.FALSE
3574 else:
3575 containsResults = containsResults and\
3576 containsResponses[ i ][ 1 ]
3577 utilities.assert_equals( expect=main.TRUE,
3578 actual=containsResults,
3579 onpass="Set contains is functional",
3580 onfail="Set contains failed" )
3581
3582 main.step( "Distributed Set containsAll()" )
3583 containsAllResponses = []
3584 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003585 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003586 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003587 name="setContainsAll-" + str( i ),
3588 args=[ onosSetName ],
3589 kwargs={ "values": addAllValue } )
3590 threads.append( t )
3591 t.start()
3592 for t in threads:
3593 t.join()
3594 # NOTE: This is the tuple
3595 containsAllResponses.append( t.result )
3596
3597 containsAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003598 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003599 if containsResponses[ i ] == main.ERROR:
3600 containsResults = main.FALSE
3601 else:
3602 containsResults = containsResults and\
3603 containsResponses[ i ][ 1 ]
3604 utilities.assert_equals( expect=main.TRUE,
3605 actual=containsAllResults,
3606 onpass="Set containsAll is functional",
3607 onfail="Set containsAll failed" )
3608
3609 main.step( "Distributed Set remove()" )
3610 onosSet.remove( addValue )
3611 removeResponses = []
3612 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003613 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003614 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003615 name="setTestRemove-" + str( i ),
3616 args=[ onosSetName, addValue ] )
3617 threads.append( t )
3618 t.start()
3619 for t in threads:
3620 t.join()
3621 removeResponses.append( t.result )
3622
3623 # main.TRUE = successfully changed the set
3624 # main.FALSE = action resulted in no change in set
3625 # main.ERROR - Some error in executing the function
3626 removeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003627 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003628 if removeResponses[ i ] == main.TRUE:
3629 # All is well
3630 pass
3631 elif removeResponses[ i ] == main.FALSE:
3632 # not in set, probably fine
3633 pass
3634 elif removeResponses[ i ] == main.ERROR:
3635 # Error in execution
3636 removeResults = main.FALSE
3637 else:
3638 # unexpected result
3639 removeResults = main.FALSE
3640 if removeResults != main.TRUE:
3641 main.log.error( "Error executing set remove" )
3642
3643 # Check if set is still correct
3644 size = len( onosSet )
3645 getResponses = []
3646 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003647 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003648 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003649 name="setTestGet-" + str( i ),
3650 args=[ onosSetName ] )
3651 threads.append( t )
3652 t.start()
3653 for t in threads:
3654 t.join()
3655 getResponses.append( t.result )
3656 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003657 for i in range( len( main.activeNodes ) ):
3658 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003659 if isinstance( getResponses[ i ], list):
3660 current = set( getResponses[ i ] )
3661 if len( current ) == len( getResponses[ i ] ):
3662 # no repeats
3663 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003664 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003665 " has incorrect view" +
3666 " of set " + onosSetName + ":\n" +
3667 str( getResponses[ i ] ) )
3668 main.log.debug( "Expected: " + str( onosSet ) )
3669 main.log.debug( "Actual: " + str( current ) )
3670 getResults = main.FALSE
3671 else:
3672 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003673 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003674 " has repeat elements in" +
3675 " set " + onosSetName + ":\n" +
3676 str( getResponses[ i ] ) )
3677 getResults = main.FALSE
3678 elif getResponses[ i ] == main.ERROR:
3679 getResults = main.FALSE
3680 sizeResponses = []
3681 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003682 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003683 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003684 name="setTestSize-" + str( i ),
3685 args=[ onosSetName ] )
3686 threads.append( t )
3687 t.start()
3688 for t in threads:
3689 t.join()
3690 sizeResponses.append( t.result )
3691 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003692 for i in range( len( main.activeNodes ) ):
3693 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003694 if size != sizeResponses[ i ]:
3695 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003696 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003697 " expected a size of " + str( size ) +
3698 " for set " + onosSetName +
3699 " but got " + str( sizeResponses[ i ] ) )
3700 removeResults = removeResults and getResults and sizeResults
3701 utilities.assert_equals( expect=main.TRUE,
3702 actual=removeResults,
3703 onpass="Set remove correct",
3704 onfail="Set remove was incorrect" )
3705
3706 main.step( "Distributed Set removeAll()" )
3707 onosSet.difference_update( addAllValue.split() )
3708 removeAllResponses = []
3709 threads = []
3710 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003711 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003712 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003713 name="setTestRemoveAll-" + str( i ),
3714 args=[ onosSetName, addAllValue ] )
3715 threads.append( t )
3716 t.start()
3717 for t in threads:
3718 t.join()
3719 removeAllResponses.append( t.result )
3720 except Exception, e:
3721 main.log.exception(e)
3722
3723 # main.TRUE = successfully changed the set
3724 # main.FALSE = action resulted in no change in set
3725 # main.ERROR - Some error in executing the function
3726 removeAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003727 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003728 if removeAllResponses[ i ] == main.TRUE:
3729 # All is well
3730 pass
3731 elif removeAllResponses[ i ] == main.FALSE:
3732 # not in set, probably fine
3733 pass
3734 elif removeAllResponses[ i ] == main.ERROR:
3735 # Error in execution
3736 removeAllResults = main.FALSE
3737 else:
3738 # unexpected result
3739 removeAllResults = main.FALSE
3740 if removeAllResults != main.TRUE:
3741 main.log.error( "Error executing set removeAll" )
3742
3743 # Check if set is still correct
3744 size = len( onosSet )
3745 getResponses = []
3746 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003747 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003748 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003749 name="setTestGet-" + str( i ),
3750 args=[ onosSetName ] )
3751 threads.append( t )
3752 t.start()
3753 for t in threads:
3754 t.join()
3755 getResponses.append( t.result )
3756 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003757 for i in range( len( main.activeNodes ) ):
3758 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003759 if isinstance( getResponses[ i ], list):
3760 current = set( getResponses[ i ] )
3761 if len( current ) == len( getResponses[ i ] ):
3762 # no repeats
3763 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003764 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003765 " has incorrect view" +
3766 " of set " + onosSetName + ":\n" +
3767 str( getResponses[ i ] ) )
3768 main.log.debug( "Expected: " + str( onosSet ) )
3769 main.log.debug( "Actual: " + str( current ) )
3770 getResults = main.FALSE
3771 else:
3772 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003773 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003774 " has repeat elements in" +
3775 " set " + onosSetName + ":\n" +
3776 str( getResponses[ i ] ) )
3777 getResults = main.FALSE
3778 elif getResponses[ i ] == main.ERROR:
3779 getResults = main.FALSE
3780 sizeResponses = []
3781 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003782 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003783 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003784 name="setTestSize-" + str( i ),
3785 args=[ onosSetName ] )
3786 threads.append( t )
3787 t.start()
3788 for t in threads:
3789 t.join()
3790 sizeResponses.append( t.result )
3791 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003792 for i in range( len( main.activeNodes ) ):
3793 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003794 if size != sizeResponses[ i ]:
3795 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003796 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003797 " expected a size of " + str( size ) +
3798 " for set " + onosSetName +
3799 " but got " + str( sizeResponses[ i ] ) )
3800 removeAllResults = removeAllResults and getResults and sizeResults
3801 utilities.assert_equals( expect=main.TRUE,
3802 actual=removeAllResults,
3803 onpass="Set removeAll correct",
3804 onfail="Set removeAll was incorrect" )
3805
3806 main.step( "Distributed Set addAll()" )
3807 onosSet.update( addAllValue.split() )
3808 addResponses = []
3809 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003810 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003811 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003812 name="setTestAddAll-" + str( i ),
3813 args=[ onosSetName, addAllValue ] )
3814 threads.append( t )
3815 t.start()
3816 for t in threads:
3817 t.join()
3818 addResponses.append( t.result )
3819
3820 # main.TRUE = successfully changed the set
3821 # main.FALSE = action resulted in no change in set
3822 # main.ERROR - Some error in executing the function
3823 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003824 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003825 if addResponses[ i ] == main.TRUE:
3826 # All is well
3827 pass
3828 elif addResponses[ i ] == main.FALSE:
3829 # Already in set, probably fine
3830 pass
3831 elif addResponses[ i ] == main.ERROR:
3832 # Error in execution
3833 addAllResults = main.FALSE
3834 else:
3835 # unexpected result
3836 addAllResults = main.FALSE
3837 if addAllResults != main.TRUE:
3838 main.log.error( "Error executing set addAll" )
3839
3840 # Check if set is still correct
3841 size = len( onosSet )
3842 getResponses = []
3843 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003844 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003845 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003846 name="setTestGet-" + str( i ),
3847 args=[ onosSetName ] )
3848 threads.append( t )
3849 t.start()
3850 for t in threads:
3851 t.join()
3852 getResponses.append( t.result )
3853 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003854 for i in range( len( main.activeNodes ) ):
3855 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003856 if isinstance( getResponses[ i ], list):
3857 current = set( getResponses[ i ] )
3858 if len( current ) == len( getResponses[ i ] ):
3859 # no repeats
3860 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003861 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003862 " has incorrect view" +
3863 " of set " + onosSetName + ":\n" +
3864 str( getResponses[ i ] ) )
3865 main.log.debug( "Expected: " + str( onosSet ) )
3866 main.log.debug( "Actual: " + str( current ) )
3867 getResults = main.FALSE
3868 else:
3869 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003870 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003871 " has repeat elements in" +
3872 " set " + onosSetName + ":\n" +
3873 str( getResponses[ i ] ) )
3874 getResults = main.FALSE
3875 elif getResponses[ i ] == main.ERROR:
3876 getResults = main.FALSE
3877 sizeResponses = []
3878 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003879 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003880 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003881 name="setTestSize-" + str( i ),
3882 args=[ onosSetName ] )
3883 threads.append( t )
3884 t.start()
3885 for t in threads:
3886 t.join()
3887 sizeResponses.append( t.result )
3888 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003889 for i in range( len( main.activeNodes ) ):
3890 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003891 if size != sizeResponses[ i ]:
3892 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003893 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003894 " expected a size of " + str( size ) +
3895 " for set " + onosSetName +
3896 " but got " + str( sizeResponses[ i ] ) )
3897 addAllResults = addAllResults and getResults and sizeResults
3898 utilities.assert_equals( expect=main.TRUE,
3899 actual=addAllResults,
3900 onpass="Set addAll correct",
3901 onfail="Set addAll was incorrect" )
3902
3903 main.step( "Distributed Set clear()" )
3904 onosSet.clear()
3905 clearResponses = []
3906 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003907 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003908 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003909 name="setTestClear-" + str( i ),
3910 args=[ onosSetName, " "], # Values doesn't matter
3911 kwargs={ "clear": True } )
3912 threads.append( t )
3913 t.start()
3914 for t in threads:
3915 t.join()
3916 clearResponses.append( t.result )
3917
3918 # main.TRUE = successfully changed the set
3919 # main.FALSE = action resulted in no change in set
3920 # main.ERROR - Some error in executing the function
3921 clearResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003922 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003923 if clearResponses[ i ] == main.TRUE:
3924 # All is well
3925 pass
3926 elif clearResponses[ i ] == main.FALSE:
3927 # Nothing set, probably fine
3928 pass
3929 elif clearResponses[ i ] == main.ERROR:
3930 # Error in execution
3931 clearResults = main.FALSE
3932 else:
3933 # unexpected result
3934 clearResults = main.FALSE
3935 if clearResults != main.TRUE:
3936 main.log.error( "Error executing set clear" )
3937
3938 # Check if set is still correct
3939 size = len( onosSet )
3940 getResponses = []
3941 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003942 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003943 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003944 name="setTestGet-" + str( i ),
3945 args=[ onosSetName ] )
3946 threads.append( t )
3947 t.start()
3948 for t in threads:
3949 t.join()
3950 getResponses.append( t.result )
3951 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003952 for i in range( len( main.activeNodes ) ):
3953 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003954 if isinstance( getResponses[ i ], list):
3955 current = set( getResponses[ i ] )
3956 if len( current ) == len( getResponses[ i ] ):
3957 # no repeats
3958 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003959 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003960 " has incorrect view" +
3961 " of set " + onosSetName + ":\n" +
3962 str( getResponses[ i ] ) )
3963 main.log.debug( "Expected: " + str( onosSet ) )
3964 main.log.debug( "Actual: " + str( current ) )
3965 getResults = main.FALSE
3966 else:
3967 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003968 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003969 " has repeat elements in" +
3970 " set " + onosSetName + ":\n" +
3971 str( getResponses[ i ] ) )
3972 getResults = main.FALSE
3973 elif getResponses[ i ] == main.ERROR:
3974 getResults = main.FALSE
3975 sizeResponses = []
3976 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003977 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003978 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003979 name="setTestSize-" + str( i ),
3980 args=[ onosSetName ] )
3981 threads.append( t )
3982 t.start()
3983 for t in threads:
3984 t.join()
3985 sizeResponses.append( t.result )
3986 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003987 for i in range( len( main.activeNodes ) ):
3988 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003989 if size != sizeResponses[ i ]:
3990 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003991 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003992 " expected a size of " + str( size ) +
3993 " for set " + onosSetName +
3994 " but got " + str( sizeResponses[ i ] ) )
3995 clearResults = clearResults and getResults and sizeResults
3996 utilities.assert_equals( expect=main.TRUE,
3997 actual=clearResults,
3998 onpass="Set clear correct",
3999 onfail="Set clear was incorrect" )
4000
4001 main.step( "Distributed Set addAll()" )
4002 onosSet.update( addAllValue.split() )
4003 addResponses = []
4004 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004005 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004006 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004007 name="setTestAddAll-" + str( i ),
4008 args=[ onosSetName, addAllValue ] )
4009 threads.append( t )
4010 t.start()
4011 for t in threads:
4012 t.join()
4013 addResponses.append( t.result )
4014
4015 # main.TRUE = successfully changed the set
4016 # main.FALSE = action resulted in no change in set
4017 # main.ERROR - Some error in executing the function
4018 addAllResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004019 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004020 if addResponses[ i ] == main.TRUE:
4021 # All is well
4022 pass
4023 elif addResponses[ i ] == main.FALSE:
4024 # Already in set, probably fine
4025 pass
4026 elif addResponses[ i ] == main.ERROR:
4027 # Error in execution
4028 addAllResults = main.FALSE
4029 else:
4030 # unexpected result
4031 addAllResults = main.FALSE
4032 if addAllResults != main.TRUE:
4033 main.log.error( "Error executing set addAll" )
4034
4035 # Check if set is still correct
4036 size = len( onosSet )
4037 getResponses = []
4038 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004039 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004040 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004041 name="setTestGet-" + str( i ),
4042 args=[ onosSetName ] )
4043 threads.append( t )
4044 t.start()
4045 for t in threads:
4046 t.join()
4047 getResponses.append( t.result )
4048 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004049 for i in range( len( main.activeNodes ) ):
4050 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004051 if isinstance( getResponses[ i ], list):
4052 current = set( getResponses[ i ] )
4053 if len( current ) == len( getResponses[ i ] ):
4054 # no repeats
4055 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004056 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004057 " has incorrect view" +
4058 " of set " + onosSetName + ":\n" +
4059 str( getResponses[ i ] ) )
4060 main.log.debug( "Expected: " + str( onosSet ) )
4061 main.log.debug( "Actual: " + str( current ) )
4062 getResults = main.FALSE
4063 else:
4064 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004065 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004066 " has repeat elements in" +
4067 " set " + onosSetName + ":\n" +
4068 str( getResponses[ i ] ) )
4069 getResults = main.FALSE
4070 elif getResponses[ i ] == main.ERROR:
4071 getResults = main.FALSE
4072 sizeResponses = []
4073 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004074 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004075 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004076 name="setTestSize-" + str( i ),
4077 args=[ onosSetName ] )
4078 threads.append( t )
4079 t.start()
4080 for t in threads:
4081 t.join()
4082 sizeResponses.append( t.result )
4083 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004084 for i in range( len( main.activeNodes ) ):
4085 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004086 if size != sizeResponses[ i ]:
4087 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004088 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004089 " expected a size of " + str( size ) +
4090 " for set " + onosSetName +
4091 " but got " + str( sizeResponses[ i ] ) )
4092 addAllResults = addAllResults and getResults and sizeResults
4093 utilities.assert_equals( expect=main.TRUE,
4094 actual=addAllResults,
4095 onpass="Set addAll correct",
4096 onfail="Set addAll was incorrect" )
4097
4098 main.step( "Distributed Set retain()" )
4099 onosSet.intersection_update( retainValue.split() )
4100 retainResponses = []
4101 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004102 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004103 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004104 name="setTestRetain-" + str( i ),
4105 args=[ onosSetName, retainValue ],
4106 kwargs={ "retain": True } )
4107 threads.append( t )
4108 t.start()
4109 for t in threads:
4110 t.join()
4111 retainResponses.append( t.result )
4112
4113 # main.TRUE = successfully changed the set
4114 # main.FALSE = action resulted in no change in set
4115 # main.ERROR - Some error in executing the function
4116 retainResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004117 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004118 if retainResponses[ i ] == main.TRUE:
4119 # All is well
4120 pass
4121 elif retainResponses[ i ] == main.FALSE:
4122 # Already in set, probably fine
4123 pass
4124 elif retainResponses[ i ] == main.ERROR:
4125 # Error in execution
4126 retainResults = main.FALSE
4127 else:
4128 # unexpected result
4129 retainResults = main.FALSE
4130 if retainResults != main.TRUE:
4131 main.log.error( "Error executing set retain" )
4132
4133 # Check if set is still correct
4134 size = len( onosSet )
4135 getResponses = []
4136 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004137 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004138 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004139 name="setTestGet-" + str( i ),
4140 args=[ onosSetName ] )
4141 threads.append( t )
4142 t.start()
4143 for t in threads:
4144 t.join()
4145 getResponses.append( t.result )
4146 getResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004147 for i in range( len( main.activeNodes ) ):
4148 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004149 if isinstance( getResponses[ i ], list):
4150 current = set( getResponses[ i ] )
4151 if len( current ) == len( getResponses[ i ] ):
4152 # no repeats
4153 if onosSet != current:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004154 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004155 " has incorrect view" +
4156 " of set " + onosSetName + ":\n" +
4157 str( getResponses[ i ] ) )
4158 main.log.debug( "Expected: " + str( onosSet ) )
4159 main.log.debug( "Actual: " + str( current ) )
4160 getResults = main.FALSE
4161 else:
4162 # error, set is not a set
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004163 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004164 " has repeat elements in" +
4165 " set " + onosSetName + ":\n" +
4166 str( getResponses[ i ] ) )
4167 getResults = main.FALSE
4168 elif getResponses[ i ] == main.ERROR:
4169 getResults = main.FALSE
4170 sizeResponses = []
4171 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004172 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004173 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004174 name="setTestSize-" + str( i ),
4175 args=[ onosSetName ] )
4176 threads.append( t )
4177 t.start()
4178 for t in threads:
4179 t.join()
4180 sizeResponses.append( t.result )
4181 sizeResults = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004182 for i in range( len( main.activeNodes ) ):
4183 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004184 if size != sizeResponses[ i ]:
4185 sizeResults = main.FALSE
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004186 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004187 str( size ) + " for set " + onosSetName +
4188 " but got " + str( sizeResponses[ i ] ) )
4189 retainResults = retainResults and getResults and sizeResults
4190 utilities.assert_equals( expect=main.TRUE,
4191 actual=retainResults,
4192 onpass="Set retain correct",
4193 onfail="Set retain was incorrect" )
4194
Jon Hall2a5002c2015-08-21 16:49:11 -07004195 # Transactional maps
4196 main.step( "Partitioned Transactional maps put" )
4197 tMapValue = "Testing"
4198 numKeys = 100
4199 putResult = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004200 node = main.activeNodes[0]
4201 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004202 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004203 for i in putResponses:
4204 if putResponses[ i ][ 'value' ] != tMapValue:
4205 putResult = False
4206 else:
4207 putResult = False
4208 if not putResult:
4209 main.log.debug( "Put response values: " + str( putResponses ) )
4210 utilities.assert_equals( expect=True,
4211 actual=putResult,
4212 onpass="Partitioned Transactional Map put successful",
4213 onfail="Partitioned Transactional Map put values are incorrect" )
4214
4215 main.step( "Partitioned Transactional maps get" )
Jon Hall9bfadd22016-05-11 14:48:07 -07004216 # FIXME: is this sleep needed?
4217 time.sleep( 5 )
4218
Jon Hall2a5002c2015-08-21 16:49:11 -07004219 getCheck = True
4220 for n in range( 1, numKeys + 1 ):
4221 getResponses = []
4222 threads = []
4223 valueCheck = True
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004224 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004225 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4226 name="TMap-get-" + str( i ),
Jon Hallb3ed8ed2015-10-28 16:43:55 -07004227 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004228 threads.append( t )
4229 t.start()
4230 for t in threads:
4231 t.join()
4232 getResponses.append( t.result )
4233 for node in getResponses:
4234 if node != tMapValue:
4235 valueCheck = False
4236 if not valueCheck:
4237 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4238 main.log.warn( getResponses )
4239 getCheck = getCheck and valueCheck
4240 utilities.assert_equals( expect=True,
4241 actual=getCheck,
4242 onpass="Partitioned Transactional Map get values were correct",
4243 onfail="Partitioned Transactional Map values incorrect" )