blob: c229b03122679c21ad1b06d14e7fef86f1a715ff [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAclusterRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080051 import time
Jon Halla440e872016-03-31 15:15:50 -070052 import json
Jon Hall5cf14d52015-07-16 12:15:19 -070053 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
54 "initialization" )
55 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070056 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070057 "installing ONOS, starting Mininet and ONOS" +\
58 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070059
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
71 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79 # These are for csv plotting in jenkins
80 global labels
81 global data
82 labels = []
83 data = []
84
85 # FIXME: just get controller port from params?
86 # TODO: do we really need all these?
87 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
88 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
89 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
90 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
91 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
92 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
93 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
94
Jon Halle1a3b752015-07-22 13:02:46 -070095 try:
Jon Hall53c5e662016-04-13 16:06:56 -070096 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070097 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070098 except Exception as e:
99 main.log.exception( e )
100 main.cleanup()
101 main.exit()
102
103 main.CLIs = []
104 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700105 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700106 for i in range( 1, main.numCtrls + 1 ):
107 try:
108 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
109 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
110 ipList.append( main.nodes[ -1 ].ip_address )
111 except AttributeError:
112 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700113
114 main.step( "Create cell file" )
115 cellAppString = main.params[ 'ENV' ][ 'appString' ]
116 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
117 main.Mininet1.ip_address,
118 cellAppString, ipList )
119 main.step( "Applying cell variable to environment" )
120 cellResult = main.ONOSbench.setCell( cellName )
121 verifyResult = main.ONOSbench.verifyCell()
122
123 # FIXME:this is short term fix
124 main.log.info( "Removing raft logs" )
125 main.ONOSbench.onosRemoveRaftLogs()
126
127 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700128 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700129 main.ONOSbench.onosUninstall( node.ip_address )
130
131 # Make sure ONOS is DEAD
132 main.log.info( "Killing any ONOS processes" )
133 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700134 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700135 killed = main.ONOSbench.onosKill( node.ip_address )
136 killResults = killResults and killed
137
138 cleanInstallResult = main.TRUE
139 gitPullResult = main.TRUE
140
141 main.step( "Starting Mininet" )
142 # scp topo file to mininet
143 # TODO: move to params?
144 topoName = "obelisk.py"
145 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700146 main.ONOSbench.scp( main.Mininet1,
147 filePath + topoName,
148 main.Mininet1.home,
149 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700150 mnResult = main.Mininet1.startNet( )
151 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
152 onpass="Mininet Started",
153 onfail="Error starting Mininet" )
154
155 main.step( "Git checkout and pull " + gitBranch )
156 if PULLCODE:
157 main.ONOSbench.gitCheckout( gitBranch )
158 gitPullResult = main.ONOSbench.gitPull()
159 # values of 1 or 3 are good
160 utilities.assert_lesser( expect=0, actual=gitPullResult,
161 onpass="Git pull successful",
162 onfail="Git pull failed" )
163 main.ONOSbench.getVersion( report=True )
164
165 main.step( "Using mvn clean install" )
166 cleanInstallResult = main.TRUE
167 if PULLCODE and gitPullResult == main.TRUE:
168 cleanInstallResult = main.ONOSbench.cleanInstall()
169 else:
170 main.log.warn( "Did not pull new code so skipping mvn " +
171 "clean install" )
172 utilities.assert_equals( expect=main.TRUE,
173 actual=cleanInstallResult,
174 onpass="MCI successful",
175 onfail="MCI failed" )
176 # GRAPHS
177 # NOTE: important params here:
178 # job = name of Jenkins job
179 # Plot Name = Plot-HA, only can be used if multiple plots
180 # index = The number of the graph under plot name
181 job = "HAclusterRestart"
182 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700183 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700184 graphs = '<ac:structured-macro ac:name="html">\n'
185 graphs += '<ac:plain-text-body><![CDATA[\n'
186 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800187 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700188 '&width=500&height=300"' +\
189 'noborder="0" width="500" height="300" scrolling="yes" ' +\
190 'seamless="seamless"></iframe>\n'
191 graphs += ']]></ac:plain-text-body>\n'
192 graphs += '</ac:structured-macro>\n'
193 main.log.wiki(graphs)
194
195 main.step( "Creating ONOS package" )
196 packageResult = main.ONOSbench.onosPackage()
197 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
198 onpass="ONOS package successful",
199 onfail="ONOS package failed" )
200
201 main.step( "Installing ONOS package" )
202 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700203 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700204 tmpResult = main.ONOSbench.onosInstall( options="-f",
205 node=node.ip_address )
206 onosInstallResult = onosInstallResult and tmpResult
207 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
208 onpass="ONOS install successful",
209 onfail="ONOS install failed" )
210
211 main.step( "Checking if ONOS is up yet" )
212 for i in range( 2 ):
213 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700214 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700215 started = main.ONOSbench.isup( node.ip_address )
216 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800217 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700218 onosIsupResult = onosIsupResult and started
219 if onosIsupResult == main.TRUE:
220 break
221 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
222 onpass="ONOS startup successful",
223 onfail="ONOS startup failed" )
224
225 main.log.step( "Starting ONOS CLI sessions" )
226 cliResults = main.TRUE
227 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700228 for i in range( main.numCtrls ):
229 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700231 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700232 threads.append( t )
233 t.start()
234
235 for t in threads:
236 t.join()
237 cliResults = cliResults and t.result
238 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
239 onpass="ONOS cli startup successful",
240 onfail="ONOS cli startup failed" )
241
Jon Halla440e872016-03-31 15:15:50 -0700242 # Create a list of active nodes for use when some nodes are stopped
243 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
244
Jon Hall5cf14d52015-07-16 12:15:19 -0700245 if main.params[ 'tcpdump' ].lower() == "true":
246 main.step( "Start Packet Capture MN" )
247 main.Mininet2.startTcpdump(
248 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
249 + "-MN.pcap",
250 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
251 port=main.params[ 'MNtcpdump' ][ 'port' ] )
252
Jon Halla440e872016-03-31 15:15:50 -0700253 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700254 nodeResults = utilities.retry( main.HA.nodesCheck,
255 False,
256 args=[main.activeNodes],
257 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700258
Jon Hall41d39f12016-04-11 22:54:35 -0700259 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700260 onpass="Nodes check successful",
261 onfail="Nodes check NOT successful" )
262
263 if not nodeResults:
264 for cli in main.CLIs:
265 main.log.debug( "{} components not ACTIVE: \n{}".format(
266 cli.name,
267 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
268
Jon Hall5cf14d52015-07-16 12:15:19 -0700269 if cliResults == main.FALSE:
270 main.log.error( "Failed to start ONOS, stopping test" )
271 main.cleanup()
272 main.exit()
273
Jon Hall172b7ba2016-04-07 18:12:20 -0700274 main.step( "Activate apps defined in the params file" )
275 # get data from the params
276 apps = main.params.get( 'apps' )
277 if apps:
278 apps = apps.split(',')
279 main.log.warn( apps )
280 activateResult = True
281 for app in apps:
282 main.CLIs[ 0 ].app( app, "Activate" )
283 # TODO: check this worked
284 time.sleep( 10 ) # wait for apps to activate
285 for app in apps:
286 state = main.CLIs[ 0 ].appStatus( app )
287 if state == "ACTIVE":
288 activateResult = activeResult and True
289 else:
290 main.log.error( "{} is in {} state".format( app, state ) )
291 activeResult = False
292 utilities.assert_equals( expect=True,
293 actual=activateResult,
294 onpass="Successfully activated apps",
295 onfail="Failed to activate apps" )
296 else:
297 main.log.warn( "No apps were specified to be loaded after startup" )
298
299 main.step( "Set ONOS configurations" )
300 config = main.params.get( 'ONOS_Configuration' )
301 if config:
302 main.log.debug( config )
303 checkResult = main.TRUE
304 for component in config:
305 for setting in config[component]:
306 value = config[component][setting]
307 check = main.CLIs[ 0 ].setCfg( component, setting, value )
308 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
309 checkResult = check and checkResult
310 utilities.assert_equals( expect=main.TRUE,
311 actual=checkResult,
312 onpass="Successfully set config",
313 onfail="Failed to set config" )
314 else:
315 main.log.warn( "No configurations were specified to be changed after startup" )
316
Jon Hall9d2dcad2016-04-08 10:15:20 -0700317 main.step( "App Ids check" )
318 appCheck = main.TRUE
319 threads = []
320 for i in main.activeNodes:
321 t = main.Thread( target=main.CLIs[i].appToIDCheck,
322 name="appToIDCheck-" + str( i ),
323 args=[] )
324 threads.append( t )
325 t.start()
326
327 for t in threads:
328 t.join()
329 appCheck = appCheck and t.result
330 if appCheck != main.TRUE:
331 node = main.activeNodes[0]
332 main.log.warn( main.CLIs[node].apps() )
333 main.log.warn( main.CLIs[node].appIDs() )
334 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
335 onpass="App Ids seem to be correct",
336 onfail="Something is wrong with app Ids" )
337
Jon Hall5cf14d52015-07-16 12:15:19 -0700338 def CASE2( self, main ):
339 """
340 Assign devices to controllers
341 """
342 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700343 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700344 assert main, "main not defined"
345 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700346 assert main.CLIs, "main.CLIs not defined"
347 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700348 assert ONOS1Port, "ONOS1Port not defined"
349 assert ONOS2Port, "ONOS2Port not defined"
350 assert ONOS3Port, "ONOS3Port not defined"
351 assert ONOS4Port, "ONOS4Port not defined"
352 assert ONOS5Port, "ONOS5Port not defined"
353 assert ONOS6Port, "ONOS6Port not defined"
354 assert ONOS7Port, "ONOS7Port not defined"
355
356 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700357 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700358 "and check that an ONOS node becomes the " +\
359 "master of the device."
360 main.step( "Assign switches to controllers" )
361
362 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700363 for i in range( main.numCtrls ):
364 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700365 swList = []
366 for i in range( 1, 29 ):
367 swList.append( "s" + str( i ) )
368 main.Mininet1.assignSwController( sw=swList, ip=ipList )
369
370 mastershipCheck = main.TRUE
371 for i in range( 1, 29 ):
372 response = main.Mininet1.getSwController( "s" + str( i ) )
373 try:
374 main.log.info( str( response ) )
375 except Exception:
376 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700377 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700378 if re.search( "tcp:" + node.ip_address, response ):
379 mastershipCheck = mastershipCheck and main.TRUE
380 else:
381 main.log.error( "Error, node " + node.ip_address + " is " +
382 "not in the list of controllers s" +
383 str( i ) + " is connecting to." )
384 mastershipCheck = main.FALSE
385 utilities.assert_equals(
386 expect=main.TRUE,
387 actual=mastershipCheck,
388 onpass="Switch mastership assigned correctly",
389 onfail="Switches not assigned correctly to controllers" )
390
391 def CASE21( self, main ):
392 """
393 Assign mastership to controllers
394 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700395 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700396 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700397 assert main, "main not defined"
398 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700399 assert main.CLIs, "main.CLIs not defined"
400 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700401 assert ONOS1Port, "ONOS1Port not defined"
402 assert ONOS2Port, "ONOS2Port not defined"
403 assert ONOS3Port, "ONOS3Port not defined"
404 assert ONOS4Port, "ONOS4Port not defined"
405 assert ONOS5Port, "ONOS5Port not defined"
406 assert ONOS6Port, "ONOS6Port not defined"
407 assert ONOS7Port, "ONOS7Port not defined"
408
409 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700410 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700411 "device. Then manually assign" +\
412 " mastership to specific ONOS nodes using" +\
413 " 'device-role'"
414 main.step( "Assign mastership of switches to specific controllers" )
415 # Manually assign mastership to the controller we want
416 roleCall = main.TRUE
417
418 ipList = [ ]
419 deviceList = []
Jon Halla440e872016-03-31 15:15:50 -0700420 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700421 try:
422 # Assign mastership to specific controllers. This assignment was
423 # determined for a 7 node cluser, but will work with any sized
424 # cluster
425 for i in range( 1, 29 ): # switches 1 through 28
426 # set up correct variables:
427 if i == 1:
428 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700429 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700430 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700431 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700432 c = 1 % main.numCtrls
433 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700434 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700435 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700436 c = 1 % main.numCtrls
437 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700438 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700439 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700440 c = 3 % main.numCtrls
441 ip = main.nodes[ c ].ip_address # ONOS4
Jon Halla440e872016-03-31 15:15:50 -0700442 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700443 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700444 c = 2 % main.numCtrls
445 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700446 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700447 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700448 c = 2 % main.numCtrls
449 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700450 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700451 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700452 c = 5 % main.numCtrls
453 ip = main.nodes[ c ].ip_address # ONOS6
Jon Halla440e872016-03-31 15:15:50 -0700454 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700455 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700456 c = 4 % main.numCtrls
457 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700458 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700459 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700460 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700461 c = 6 % main.numCtrls
462 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700464 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700465 elif i == 28:
466 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700467 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700468 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700469 else:
470 main.log.error( "You didn't write an else statement for " +
471 "switch s" + str( i ) )
472 roleCall = main.FALSE
473 # Assign switch
474 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
475 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700476 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700477 ipList.append( ip )
478 deviceList.append( deviceId )
479 except ( AttributeError, AssertionError ):
480 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700481 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700482 utilities.assert_equals(
483 expect=main.TRUE,
484 actual=roleCall,
485 onpass="Re-assigned switch mastership to designated controller",
486 onfail="Something wrong with deviceRole calls" )
487
488 main.step( "Check mastership was correctly assigned" )
489 roleCheck = main.TRUE
490 # NOTE: This is due to the fact that device mastership change is not
491 # atomic and is actually a multi step process
492 time.sleep( 5 )
493 for i in range( len( ipList ) ):
494 ip = ipList[i]
495 deviceId = deviceList[i]
496 # Check assignment
Jon Halla440e872016-03-31 15:15:50 -0700497 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700498 if ip in master:
499 roleCheck = roleCheck and main.TRUE
500 else:
501 roleCheck = roleCheck and main.FALSE
502 main.log.error( "Error, controller " + ip + " is not" +
503 " master " + "of device " +
504 str( deviceId ) + ". Master is " +
505 repr( master ) + "." )
506 utilities.assert_equals(
507 expect=main.TRUE,
508 actual=roleCheck,
509 onpass="Switches were successfully reassigned to designated " +
510 "controller",
511 onfail="Switches were not successfully reassigned" )
512
513 def CASE3( self, main ):
514 """
515 Assign intents
516 """
517 import time
518 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700519 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700520 assert main, "main not defined"
521 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700522 assert main.CLIs, "main.CLIs not defined"
523 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700524 try:
525 labels
526 except NameError:
527 main.log.error( "labels not defined, setting to []" )
528 labels = []
529 try:
530 data
531 except NameError:
532 main.log.error( "data not defined, setting to []" )
533 data = []
534 # NOTE: we must reinstall intents until we have a persistant intent
535 # datastore!
536 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700537 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700538 "assign predetermined host-to-host intents." +\
539 " After installation, check that the intent" +\
540 " is distributed to all nodes and the state" +\
541 " is INSTALLED"
542
543 # install onos-app-fwd
544 main.step( "Install reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700545 onosCli = main.CLIs[ main.activeNodes[0] ]
546 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700547 utilities.assert_equals( expect=main.TRUE, actual=installResults,
548 onpass="Install fwd successful",
549 onfail="Install fwd failed" )
550
551 main.step( "Check app ids" )
552 appCheck = main.TRUE
553 threads = []
Jon Halla440e872016-03-31 15:15:50 -0700554 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700555 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700556 name="appToIDCheck-" + str( i ),
557 args=[] )
558 threads.append( t )
559 t.start()
560
561 for t in threads:
562 t.join()
563 appCheck = appCheck and t.result
564 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700565 main.log.warn( onosCli.apps() )
566 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700567 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
568 onpass="App Ids seem to be correct",
569 onfail="Something is wrong with app Ids" )
570
571 main.step( "Discovering Hosts( Via pingall for now )" )
572 # FIXME: Once we have a host discovery mechanism, use that instead
573 # REACTIVE FWD test
574 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700575 passMsg = "Reactive Pingall test passed"
576 time1 = time.time()
577 pingResult = main.Mininet1.pingall()
578 time2 = time.time()
579 if not pingResult:
580 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700581 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700582 passMsg += " on the second try"
583 utilities.assert_equals(
584 expect=main.TRUE,
585 actual=pingResult,
586 onpass= passMsg,
587 onfail="Reactive Pingall failed, " +
588 "one or more ping pairs failed" )
589 main.log.info( "Time for pingall: %2f seconds" %
590 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700591 # timeout for fwd flows
592 time.sleep( 11 )
593 # uninstall onos-app-fwd
594 main.step( "Uninstall reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700595 node = main.activeNodes[0]
596 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700597 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
598 onpass="Uninstall fwd successful",
599 onfail="Uninstall fwd failed" )
600
601 main.step( "Check app ids" )
602 threads = []
603 appCheck2 = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700604 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700605 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700606 name="appToIDCheck-" + str( i ),
607 args=[] )
608 threads.append( t )
609 t.start()
610
611 for t in threads:
612 t.join()
613 appCheck2 = appCheck2 and t.result
614 if appCheck2 != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700615 node = main.activeNodes[0]
616 main.log.warn( main.CLIs[node].apps() )
617 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700618 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
619 onpass="App Ids seem to be correct",
620 onfail="Something is wrong with app Ids" )
621
622 main.step( "Add host intents via cli" )
623 intentIds = []
Jon Hall6e709752016-02-01 13:38:46 -0800624 # TODO: move the host numbers to params
625 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700626 intentAddResult = True
627 hostResult = main.TRUE
628 for i in range( 8, 18 ):
629 main.log.info( "Adding host intent between h" + str( i ) +
630 " and h" + str( i + 10 ) )
631 host1 = "00:00:00:00:00:" + \
632 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
633 host2 = "00:00:00:00:00:" + \
634 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
635 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700636 host1Dict = onosCli.getHost( host1 )
637 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700638 host1Id = None
639 host2Id = None
640 if host1Dict and host2Dict:
641 host1Id = host1Dict.get( 'id', None )
642 host2Id = host2Dict.get( 'id', None )
643 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700644 nodeNum = ( i % len( main.activeNodes ) )
645 node = main.activeNodes[nodeNum]
646 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700647 if tmpId:
648 main.log.info( "Added intent with id: " + tmpId )
649 intentIds.append( tmpId )
650 else:
651 main.log.error( "addHostIntent returned: " +
652 repr( tmpId ) )
653 else:
654 main.log.error( "Error, getHost() failed for h" + str( i ) +
655 " and/or h" + str( i + 10 ) )
Jon Halla440e872016-03-31 15:15:50 -0700656 node = main.activeNodes[0]
657 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700658 main.log.warn( "Hosts output: " )
659 try:
660 main.log.warn( json.dumps( json.loads( hosts ),
661 sort_keys=True,
662 indent=4,
663 separators=( ',', ': ' ) ) )
664 except ( ValueError, TypeError ):
665 main.log.warn( repr( hosts ) )
666 hostResult = main.FALSE
667 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
668 onpass="Found a host id for each host",
669 onfail="Error looking up host ids" )
670
671 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700672 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700673 main.log.info( "Submitted intents: " + str( intentIds ) )
674 main.log.info( "Intents in ONOS: " + str( onosIds ) )
675 for intent in intentIds:
676 if intent in onosIds:
677 pass # intent submitted is in onos
678 else:
679 intentAddResult = False
680 if intentAddResult:
681 intentStop = time.time()
682 else:
683 intentStop = None
684 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700685 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700686 intentStates = []
687 installedCheck = True
688 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
689 count = 0
690 try:
691 for intent in json.loads( intents ):
692 state = intent.get( 'state', None )
693 if "INSTALLED" not in state:
694 installedCheck = False
695 intentId = intent.get( 'id', None )
696 intentStates.append( ( intentId, state ) )
697 except ( ValueError, TypeError ):
698 main.log.exception( "Error parsing intents" )
699 # add submitted intents not in the store
700 tmplist = [ i for i, s in intentStates ]
701 missingIntents = False
702 for i in intentIds:
703 if i not in tmplist:
704 intentStates.append( ( i, " - " ) )
705 missingIntents = True
706 intentStates.sort()
707 for i, s in intentStates:
708 count += 1
709 main.log.info( "%-6s%-15s%-15s" %
710 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700711 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700712 try:
713 missing = False
714 if leaders:
715 parsedLeaders = json.loads( leaders )
716 main.log.warn( json.dumps( parsedLeaders,
717 sort_keys=True,
718 indent=4,
719 separators=( ',', ': ' ) ) )
720 # check for all intent partitions
721 topics = []
722 for i in range( 14 ):
723 topics.append( "intent-partition-" + str( i ) )
724 main.log.debug( topics )
725 ONOStopics = [ j['topic'] for j in parsedLeaders ]
726 for topic in topics:
727 if topic not in ONOStopics:
728 main.log.error( "Error: " + topic +
729 " not in leaders" )
730 missing = True
731 else:
732 main.log.error( "leaders() returned None" )
733 except ( ValueError, TypeError ):
734 main.log.exception( "Error parsing leaders" )
735 main.log.error( repr( leaders ) )
736 # Check all nodes
737 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700738 for i in main.activeNodes:
739 response = main.CLIs[i].leaders( jsonFormat=False)
740 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700741 str( response ) )
742
Jon Halla440e872016-03-31 15:15:50 -0700743 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700744 try:
745 if partitions :
746 parsedPartitions = json.loads( partitions )
747 main.log.warn( json.dumps( parsedPartitions,
748 sort_keys=True,
749 indent=4,
750 separators=( ',', ': ' ) ) )
751 # TODO check for a leader in all paritions
752 # TODO check for consistency among nodes
753 else:
754 main.log.error( "partitions() returned None" )
755 except ( ValueError, TypeError ):
756 main.log.exception( "Error parsing partitions" )
757 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700758 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700759 try:
760 if pendingMap :
761 parsedPending = json.loads( pendingMap )
762 main.log.warn( json.dumps( parsedPending,
763 sort_keys=True,
764 indent=4,
765 separators=( ',', ': ' ) ) )
766 # TODO check something here?
767 else:
768 main.log.error( "pendingMap() returned None" )
769 except ( ValueError, TypeError ):
770 main.log.exception( "Error parsing pending map" )
771 main.log.error( repr( pendingMap ) )
772
773 intentAddResult = bool( intentAddResult and not missingIntents and
774 installedCheck )
775 if not intentAddResult:
776 main.log.error( "Error in pushing host intents to ONOS" )
777
778 main.step( "Intent Anti-Entropy dispersion" )
Jon Halla440e872016-03-31 15:15:50 -0700779 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700780 correct = True
781 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700782 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700783 onosIds = []
Jon Halla440e872016-03-31 15:15:50 -0700784 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700785 onosIds.append( ids )
Jon Halla440e872016-03-31 15:15:50 -0700786 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700787 str( sorted( onosIds ) ) )
788 if sorted( ids ) != sorted( intentIds ):
789 main.log.warn( "Set of intent IDs doesn't match" )
790 correct = False
791 break
792 else:
Jon Halla440e872016-03-31 15:15:50 -0700793 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700794 for intent in intents:
795 if intent[ 'state' ] != "INSTALLED":
796 main.log.warn( "Intent " + intent[ 'id' ] +
797 " is " + intent[ 'state' ] )
798 correct = False
799 break
800 if correct:
801 break
802 else:
803 time.sleep(1)
804 if not intentStop:
805 intentStop = time.time()
806 global gossipTime
807 gossipTime = intentStop - intentStart
808 main.log.info( "It took about " + str( gossipTime ) +
809 " seconds for all intents to appear in each node" )
810 append = False
811 title = "Gossip Intents"
812 count = 1
813 while append is False:
814 curTitle = title + str( count )
815 if curTitle not in labels:
816 labels.append( curTitle )
817 data.append( str( gossipTime ) )
818 append = True
819 else:
820 count += 1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700821 gossipPeriod = int( main.params['timers']['gossip'] )
Jon Halla440e872016-03-31 15:15:50 -0700822 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700823 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700824 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700825 onpass="ECM anti-entropy for intents worked within " +
826 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700827 onfail="Intent ECM anti-entropy took too long. " +
828 "Expected time:{}, Actual time:{}".format( maxGossipTime,
829 gossipTime ) )
830 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700831 intentAddResult = True
832
833 if not intentAddResult or "key" in pendingMap:
834 import time
835 installedCheck = True
836 main.log.info( "Sleeping 60 seconds to see if intents are found" )
837 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700838 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700839 main.log.info( "Submitted intents: " + str( intentIds ) )
840 main.log.info( "Intents in ONOS: " + str( onosIds ) )
841 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700842 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700843 intentStates = []
844 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
845 count = 0
846 try:
847 for intent in json.loads( intents ):
848 # Iter through intents of a node
849 state = intent.get( 'state', None )
850 if "INSTALLED" not in state:
851 installedCheck = False
852 intentId = intent.get( 'id', None )
853 intentStates.append( ( intentId, state ) )
854 except ( ValueError, TypeError ):
855 main.log.exception( "Error parsing intents" )
856 # add submitted intents not in the store
857 tmplist = [ i for i, s in intentStates ]
858 for i in intentIds:
859 if i not in tmplist:
860 intentStates.append( ( i, " - " ) )
861 intentStates.sort()
862 for i, s in intentStates:
863 count += 1
864 main.log.info( "%-6s%-15s%-15s" %
865 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700866 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700867 try:
868 missing = False
869 if leaders:
870 parsedLeaders = json.loads( leaders )
871 main.log.warn( json.dumps( parsedLeaders,
872 sort_keys=True,
873 indent=4,
874 separators=( ',', ': ' ) ) )
875 # check for all intent partitions
876 # check for election
877 topics = []
878 for i in range( 14 ):
879 topics.append( "intent-partition-" + str( i ) )
880 # FIXME: this should only be after we start the app
881 topics.append( "org.onosproject.election" )
882 main.log.debug( topics )
883 ONOStopics = [ j['topic'] for j in parsedLeaders ]
884 for topic in topics:
885 if topic not in ONOStopics:
886 main.log.error( "Error: " + topic +
887 " not in leaders" )
888 missing = True
889 else:
890 main.log.error( "leaders() returned None" )
891 except ( ValueError, TypeError ):
892 main.log.exception( "Error parsing leaders" )
893 main.log.error( repr( leaders ) )
894 # Check all nodes
895 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700896 for i in main.activeNodes:
897 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700898 response = node.leaders( jsonFormat=False)
899 main.log.warn( str( node.name ) + " leaders output: \n" +
900 str( response ) )
901
Jon Halla440e872016-03-31 15:15:50 -0700902 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700903 try:
904 if partitions :
905 parsedPartitions = json.loads( partitions )
906 main.log.warn( json.dumps( parsedPartitions,
907 sort_keys=True,
908 indent=4,
909 separators=( ',', ': ' ) ) )
910 # TODO check for a leader in all paritions
911 # TODO check for consistency among nodes
912 else:
913 main.log.error( "partitions() returned None" )
914 except ( ValueError, TypeError ):
915 main.log.exception( "Error parsing partitions" )
916 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700917 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700918 try:
919 if pendingMap :
920 parsedPending = json.loads( pendingMap )
921 main.log.warn( json.dumps( parsedPending,
922 sort_keys=True,
923 indent=4,
924 separators=( ',', ': ' ) ) )
925 # TODO check something here?
926 else:
927 main.log.error( "pendingMap() returned None" )
928 except ( ValueError, TypeError ):
929 main.log.exception( "Error parsing pending map" )
930 main.log.error( repr( pendingMap ) )
931
932 def CASE4( self, main ):
933 """
934 Ping across added host intents
935 """
936 import json
937 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700938 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700939 assert main, "main not defined"
940 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700941 assert main.CLIs, "main.CLIs not defined"
942 assert main.nodes, "main.nodes not defined"
Jon Hall6e709752016-02-01 13:38:46 -0800943 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700944 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700945 "functionality and check the state of " +\
946 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700947
Jon Hall41d39f12016-04-11 22:54:35 -0700948 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700949 main.step( "Check Intent state" )
950 installedCheck = False
951 loopCount = 0
952 while not installedCheck and loopCount < 40:
953 installedCheck = True
954 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700955 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700956 intentStates = []
957 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700958 count = 0
Jon Hall5cf14d52015-07-16 12:15:19 -0700959 # Iter through intents of a node
960 try:
961 for intent in json.loads( intents ):
962 state = intent.get( 'state', None )
963 if "INSTALLED" not in state:
964 installedCheck = False
965 intentId = intent.get( 'id', None )
966 intentStates.append( ( intentId, state ) )
967 except ( ValueError, TypeError ):
968 main.log.exception( "Error parsing intents." )
969 # Print states
970 intentStates.sort()
971 for i, s in intentStates:
972 count += 1
973 main.log.info( "%-6s%-15s%-15s" %
974 ( str( count ), str( i ), str( s ) ) )
975 if not installedCheck:
976 time.sleep( 1 )
977 loopCount += 1
978 utilities.assert_equals( expect=True, actual=installedCheck,
979 onpass="Intents are all INSTALLED",
980 onfail="Intents are not all in " +
981 "INSTALLED state" )
982
Jon Hall9d2dcad2016-04-08 10:15:20 -0700983 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700984 PingResult = main.TRUE
985 for i in range( 8, 18 ):
986 ping = main.Mininet1.pingHost( src="h" + str( i ),
987 target="h" + str( i + 10 ) )
988 PingResult = PingResult and ping
989 if ping == main.FALSE:
990 main.log.warn( "Ping failed between h" + str( i ) +
991 " and h" + str( i + 10 ) )
992 elif ping == main.TRUE:
993 main.log.info( "Ping test passed!" )
994 # Don't set PingResult or you'd override failures
995 if PingResult == main.FALSE:
996 main.log.error(
997 "Intents have not been installed correctly, pings failed." )
998 # TODO: pretty print
999 main.log.warn( "ONOS1 intents: " )
1000 try:
1001 tmpIntents = onosCli.intents()
1002 main.log.warn( json.dumps( json.loads( tmpIntents ),
1003 sort_keys=True,
1004 indent=4,
1005 separators=( ',', ': ' ) ) )
1006 except ( ValueError, TypeError ):
1007 main.log.warn( repr( tmpIntents ) )
1008 utilities.assert_equals(
1009 expect=main.TRUE,
1010 actual=PingResult,
1011 onpass="Intents have been installed correctly and pings work",
1012 onfail="Intents have not been installed correctly, pings failed." )
1013
Jon Hall5cf14d52015-07-16 12:15:19 -07001014 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -07001015 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001016 topicCheck = main.TRUE
1017 try:
1018 if leaders:
1019 parsedLeaders = json.loads( leaders )
1020 main.log.warn( json.dumps( parsedLeaders,
1021 sort_keys=True,
1022 indent=4,
1023 separators=( ',', ': ' ) ) )
1024 # check for all intent partitions
1025 # check for election
1026 # TODO: Look at Devices as topics now that it uses this system
1027 topics = []
1028 for i in range( 14 ):
1029 topics.append( "intent-partition-" + str( i ) )
1030 # FIXME: this should only be after we start the app
1031 # FIXME: topics.append( "org.onosproject.election" )
1032 # Print leaders output
1033 main.log.debug( topics )
1034 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1035 for topic in topics:
1036 if topic not in ONOStopics:
1037 main.log.error( "Error: " + topic +
1038 " not in leaders" )
1039 topicCheck = main.FALSE
1040 else:
1041 main.log.error( "leaders() returned None" )
1042 topicCheck = main.FALSE
1043 except ( ValueError, TypeError ):
1044 topicCheck = main.FALSE
1045 main.log.exception( "Error parsing leaders" )
1046 main.log.error( repr( leaders ) )
1047 # TODO: Check for a leader of these topics
1048 # Check all nodes
1049 if topicCheck:
Jon Halla440e872016-03-31 15:15:50 -07001050 for i in main.activeNodes:
1051 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001052 response = node.leaders( jsonFormat=False)
1053 main.log.warn( str( node.name ) + " leaders output: \n" +
1054 str( response ) )
1055
1056 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1057 onpass="intent Partitions is in leaders",
1058 onfail="Some topics were lost " )
1059 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -07001060 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001061 try:
1062 if partitions :
1063 parsedPartitions = json.loads( partitions )
1064 main.log.warn( json.dumps( parsedPartitions,
1065 sort_keys=True,
1066 indent=4,
1067 separators=( ',', ': ' ) ) )
1068 # TODO check for a leader in all paritions
1069 # TODO check for consistency among nodes
1070 else:
1071 main.log.error( "partitions() returned None" )
1072 except ( ValueError, TypeError ):
1073 main.log.exception( "Error parsing partitions" )
1074 main.log.error( repr( partitions ) )
1075 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001076 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001077 try:
1078 if pendingMap :
1079 parsedPending = json.loads( pendingMap )
1080 main.log.warn( json.dumps( parsedPending,
1081 sort_keys=True,
1082 indent=4,
1083 separators=( ',', ': ' ) ) )
1084 # TODO check something here?
1085 else:
1086 main.log.error( "pendingMap() returned None" )
1087 except ( ValueError, TypeError ):
1088 main.log.exception( "Error parsing pending map" )
1089 main.log.error( repr( pendingMap ) )
1090
1091 if not installedCheck:
1092 main.log.info( "Waiting 60 seconds to see if the state of " +
1093 "intents change" )
1094 time.sleep( 60 )
1095 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001096 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001097 intentStates = []
1098 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1099 count = 0
1100 # Iter through intents of a node
1101 try:
1102 for intent in json.loads( intents ):
1103 state = intent.get( 'state', None )
1104 if "INSTALLED" not in state:
1105 installedCheck = False
1106 intentId = intent.get( 'id', None )
1107 intentStates.append( ( intentId, state ) )
1108 except ( ValueError, TypeError ):
1109 main.log.exception( "Error parsing intents." )
1110 intentStates.sort()
1111 for i, s in intentStates:
1112 count += 1
1113 main.log.info( "%-6s%-15s%-15s" %
1114 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001115 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001116 try:
1117 missing = False
1118 if leaders:
1119 parsedLeaders = json.loads( leaders )
1120 main.log.warn( json.dumps( parsedLeaders,
1121 sort_keys=True,
1122 indent=4,
1123 separators=( ',', ': ' ) ) )
1124 # check for all intent partitions
1125 # check for election
1126 topics = []
1127 for i in range( 14 ):
1128 topics.append( "intent-partition-" + str( i ) )
1129 # FIXME: this should only be after we start the app
1130 topics.append( "org.onosproject.election" )
1131 main.log.debug( topics )
1132 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1133 for topic in topics:
1134 if topic not in ONOStopics:
1135 main.log.error( "Error: " + topic +
1136 " not in leaders" )
1137 missing = True
1138 else:
1139 main.log.error( "leaders() returned None" )
1140 except ( ValueError, TypeError ):
1141 main.log.exception( "Error parsing leaders" )
1142 main.log.error( repr( leaders ) )
1143 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001144 for i in main.activeNodes:
1145 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001146 response = node.leaders( jsonFormat=False)
1147 main.log.warn( str( node.name ) + " leaders output: \n" +
1148 str( response ) )
1149
Jon Halla440e872016-03-31 15:15:50 -07001150 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001151 try:
1152 if partitions :
1153 parsedPartitions = json.loads( partitions )
1154 main.log.warn( json.dumps( parsedPartitions,
1155 sort_keys=True,
1156 indent=4,
1157 separators=( ',', ': ' ) ) )
1158 # TODO check for a leader in all paritions
1159 # TODO check for consistency among nodes
1160 else:
1161 main.log.error( "partitions() returned None" )
1162 except ( ValueError, TypeError ):
1163 main.log.exception( "Error parsing partitions" )
1164 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001165 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001166 try:
1167 if pendingMap :
1168 parsedPending = json.loads( pendingMap )
1169 main.log.warn( json.dumps( parsedPending,
1170 sort_keys=True,
1171 indent=4,
1172 separators=( ',', ': ' ) ) )
1173 # TODO check something here?
1174 else:
1175 main.log.error( "pendingMap() returned None" )
1176 except ( ValueError, TypeError ):
1177 main.log.exception( "Error parsing pending map" )
1178 main.log.error( repr( pendingMap ) )
1179 # Print flowrules
Jon Halla440e872016-03-31 15:15:50 -07001180 node = main.activeNodes[0]
1181 main.log.debug( main.CLIs[node].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001182 main.step( "Wait a minute then ping again" )
1183 # the wait is above
1184 PingResult = main.TRUE
1185 for i in range( 8, 18 ):
1186 ping = main.Mininet1.pingHost( src="h" + str( i ),
1187 target="h" + str( i + 10 ) )
1188 PingResult = PingResult and ping
1189 if ping == main.FALSE:
1190 main.log.warn( "Ping failed between h" + str( i ) +
1191 " and h" + str( i + 10 ) )
1192 elif ping == main.TRUE:
1193 main.log.info( "Ping test passed!" )
1194 # Don't set PingResult or you'd override failures
1195 if PingResult == main.FALSE:
1196 main.log.error(
1197 "Intents have not been installed correctly, pings failed." )
1198 # TODO: pretty print
1199 main.log.warn( "ONOS1 intents: " )
1200 try:
Jon Halla440e872016-03-31 15:15:50 -07001201 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001202 main.log.warn( json.dumps( json.loads( tmpIntents ),
1203 sort_keys=True,
1204 indent=4,
1205 separators=( ',', ': ' ) ) )
1206 except ( ValueError, TypeError ):
1207 main.log.warn( repr( tmpIntents ) )
1208 utilities.assert_equals(
1209 expect=main.TRUE,
1210 actual=PingResult,
1211 onpass="Intents have been installed correctly and pings work",
1212 onfail="Intents have not been installed correctly, pings failed." )
1213
1214 def CASE5( self, main ):
1215 """
1216 Reading state of ONOS
1217 """
1218 import json
1219 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001220 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001221 assert main, "main not defined"
1222 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001223 assert main.CLIs, "main.CLIs not defined"
1224 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001225
1226 main.case( "Setting up and gathering data for current state" )
1227 # The general idea for this test case is to pull the state of
1228 # ( intents,flows, topology,... ) from each ONOS node
1229 # We can then compare them with each other and also with past states
1230
1231 main.step( "Check that each switch has a master" )
1232 global mastershipState
1233 mastershipState = '[]'
1234
1235 # Assert that each device has a master
1236 rolesNotNull = main.TRUE
1237 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001238 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001239 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001240 name="rolesNotNull-" + str( i ),
1241 args=[] )
1242 threads.append( t )
1243 t.start()
1244
1245 for t in threads:
1246 t.join()
1247 rolesNotNull = rolesNotNull and t.result
1248 utilities.assert_equals(
1249 expect=main.TRUE,
1250 actual=rolesNotNull,
1251 onpass="Each device has a master",
1252 onfail="Some devices don't have a master assigned" )
1253
1254 main.step( "Get the Mastership of each switch from each controller" )
1255 ONOSMastership = []
1256 mastershipCheck = main.FALSE
1257 consistentMastership = True
1258 rolesResults = True
1259 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001260 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001261 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001262 name="roles-" + str( i ),
1263 args=[] )
1264 threads.append( t )
1265 t.start()
1266
1267 for t in threads:
1268 t.join()
1269 ONOSMastership.append( t.result )
1270
Jon Halla440e872016-03-31 15:15:50 -07001271 for i in range( len( ONOSMastership ) ):
1272 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001273 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001274 main.log.error( "Error in getting ONOS" + node + " roles" )
1275 main.log.warn( "ONOS" + node + " mastership response: " +
1276 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001277 rolesResults = False
1278 utilities.assert_equals(
1279 expect=True,
1280 actual=rolesResults,
1281 onpass="No error in reading roles output",
1282 onfail="Error in reading roles from ONOS" )
1283
1284 main.step( "Check for consistency in roles from each controller" )
1285 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1286 main.log.info(
1287 "Switch roles are consistent across all ONOS nodes" )
1288 else:
1289 consistentMastership = False
1290 utilities.assert_equals(
1291 expect=True,
1292 actual=consistentMastership,
1293 onpass="Switch roles are consistent across all ONOS nodes",
1294 onfail="ONOS nodes have different views of switch roles" )
1295
1296 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001297 for i in range( len( main.activeNodes ) ):
1298 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001299 try:
1300 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001301 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001302 json.dumps(
1303 json.loads( ONOSMastership[ i ] ),
1304 sort_keys=True,
1305 indent=4,
1306 separators=( ',', ': ' ) ) )
1307 except ( ValueError, TypeError ):
1308 main.log.warn( repr( ONOSMastership[ i ] ) )
1309 elif rolesResults and consistentMastership:
1310 mastershipCheck = main.TRUE
1311 mastershipState = ONOSMastership[ 0 ]
1312
1313 main.step( "Get the intents from each controller" )
1314 global intentState
1315 intentState = []
1316 ONOSIntents = []
1317 intentCheck = main.FALSE
1318 consistentIntents = True
1319 intentsResults = True
1320 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001321 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001322 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001323 name="intents-" + str( i ),
1324 args=[],
1325 kwargs={ 'jsonFormat': True } )
1326 threads.append( t )
1327 t.start()
1328
1329 for t in threads:
1330 t.join()
1331 ONOSIntents.append( t.result )
1332
Jon Halla440e872016-03-31 15:15:50 -07001333 for i in range( len( ONOSIntents ) ):
1334 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001335 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hall6e709752016-02-01 13:38:46 -08001336 main.log.error( "Error in getting ONOS" + node + " intents" )
1337 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001338 repr( ONOSIntents[ i ] ) )
1339 intentsResults = False
1340 utilities.assert_equals(
1341 expect=True,
1342 actual=intentsResults,
1343 onpass="No error in reading intents output",
1344 onfail="Error in reading intents from ONOS" )
1345
1346 main.step( "Check for consistency in Intents from each controller" )
1347 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1348 main.log.info( "Intents are consistent across all ONOS " +
1349 "nodes" )
1350 else:
1351 consistentIntents = False
1352 main.log.error( "Intents not consistent" )
1353 utilities.assert_equals(
1354 expect=True,
1355 actual=consistentIntents,
1356 onpass="Intents are consistent across all ONOS nodes",
1357 onfail="ONOS nodes have different views of intents" )
1358
1359 if intentsResults:
1360 # Try to make it easy to figure out what is happening
1361 #
1362 # Intent ONOS1 ONOS2 ...
1363 # 0x01 INSTALLED INSTALLING
1364 # ... ... ...
1365 # ... ... ...
1366 title = " Id"
Jon Halla440e872016-03-31 15:15:50 -07001367 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001368 title += " " * 10 + "ONOS" + str( n + 1 )
1369 main.log.warn( title )
1370 # get all intent keys in the cluster
1371 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001372 try:
1373 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001374 for nodeStr in ONOSIntents:
1375 node = json.loads( nodeStr )
1376 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001377 keys.append( intent.get( 'id' ) )
1378 keys = set( keys )
1379 # For each intent key, print the state on each node
1380 for key in keys:
1381 row = "%-13s" % key
1382 for nodeStr in ONOSIntents:
1383 node = json.loads( nodeStr )
1384 for intent in node:
1385 if intent.get( 'id', "Error" ) == key:
1386 row += "%-15s" % intent.get( 'state' )
1387 main.log.warn( row )
1388 # End of intent state table
1389 except ValueError as e:
1390 main.log.exception( e )
1391 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001392
1393 if intentsResults and not consistentIntents:
1394 # print the json objects
Jon Halla440e872016-03-31 15:15:50 -07001395 n = str( main.activeNodes[-1] + 1 )
1396 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001397 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1398 sort_keys=True,
1399 indent=4,
1400 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001401 for i in range( len( ONOSIntents ) ):
1402 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001403 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07001404 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001405 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1406 sort_keys=True,
1407 indent=4,
1408 separators=( ',', ': ' ) ) )
1409 else:
Jon Halla440e872016-03-31 15:15:50 -07001410 main.log.debug( "ONOS" + node + " intents match ONOS" +
1411 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001412 elif intentsResults and consistentIntents:
1413 intentCheck = main.TRUE
1414 intentState = ONOSIntents[ 0 ]
1415
1416 main.step( "Get the flows from each controller" )
1417 global flowState
1418 flowState = []
1419 ONOSFlows = []
1420 ONOSFlowsJson = []
1421 flowCheck = main.FALSE
1422 consistentFlows = True
1423 flowsResults = True
1424 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001425 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001426 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001427 name="flows-" + str( i ),
1428 args=[],
1429 kwargs={ 'jsonFormat': True } )
1430 threads.append( t )
1431 t.start()
1432
1433 # NOTE: Flows command can take some time to run
1434 time.sleep(30)
1435 for t in threads:
1436 t.join()
1437 result = t.result
1438 ONOSFlows.append( result )
1439
Jon Halla440e872016-03-31 15:15:50 -07001440 for i in range( len( ONOSFlows ) ):
1441 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001442 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1443 main.log.error( "Error in getting ONOS" + num + " flows" )
1444 main.log.warn( "ONOS" + num + " flows response: " +
1445 repr( ONOSFlows[ i ] ) )
1446 flowsResults = False
1447 ONOSFlowsJson.append( None )
1448 else:
1449 try:
1450 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1451 except ( ValueError, TypeError ):
1452 # FIXME: change this to log.error?
1453 main.log.exception( "Error in parsing ONOS" + num +
1454 " response as json." )
1455 main.log.error( repr( ONOSFlows[ i ] ) )
1456 ONOSFlowsJson.append( None )
1457 flowsResults = False
1458 utilities.assert_equals(
1459 expect=True,
1460 actual=flowsResults,
1461 onpass="No error in reading flows output",
1462 onfail="Error in reading flows from ONOS" )
1463
1464 main.step( "Check for consistency in Flows from each controller" )
1465 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1466 if all( tmp ):
1467 main.log.info( "Flow count is consistent across all ONOS nodes" )
1468 else:
1469 consistentFlows = False
1470 utilities.assert_equals(
1471 expect=True,
1472 actual=consistentFlows,
1473 onpass="The flow count is consistent across all ONOS nodes",
1474 onfail="ONOS nodes have different flow counts" )
1475
1476 if flowsResults and not consistentFlows:
Jon Halla440e872016-03-31 15:15:50 -07001477 for i in range( len( ONOSFlows ) ):
1478 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001479 try:
1480 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001481 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001482 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1483 indent=4, separators=( ',', ': ' ) ) )
1484 except ( ValueError, TypeError ):
Jon Halla440e872016-03-31 15:15:50 -07001485 main.log.warn( "ONOS" + node + " flows: " +
1486 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001487 elif flowsResults and consistentFlows:
1488 flowCheck = main.TRUE
1489 flowState = ONOSFlows[ 0 ]
1490
1491 main.step( "Get the OF Table entries" )
1492 global flows
1493 flows = []
1494 for i in range( 1, 29 ):
Jon Halla440e872016-03-31 15:15:50 -07001495 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001496 if flowCheck == main.FALSE:
1497 for table in flows:
1498 main.log.warn( table )
1499 # TODO: Compare switch flow tables with ONOS flow tables
1500
1501 main.step( "Start continuous pings" )
1502 main.Mininet2.pingLong(
1503 src=main.params[ 'PING' ][ 'source1' ],
1504 target=main.params[ 'PING' ][ 'target1' ],
1505 pingTime=500 )
1506 main.Mininet2.pingLong(
1507 src=main.params[ 'PING' ][ 'source2' ],
1508 target=main.params[ 'PING' ][ 'target2' ],
1509 pingTime=500 )
1510 main.Mininet2.pingLong(
1511 src=main.params[ 'PING' ][ 'source3' ],
1512 target=main.params[ 'PING' ][ 'target3' ],
1513 pingTime=500 )
1514 main.Mininet2.pingLong(
1515 src=main.params[ 'PING' ][ 'source4' ],
1516 target=main.params[ 'PING' ][ 'target4' ],
1517 pingTime=500 )
1518 main.Mininet2.pingLong(
1519 src=main.params[ 'PING' ][ 'source5' ],
1520 target=main.params[ 'PING' ][ 'target5' ],
1521 pingTime=500 )
1522 main.Mininet2.pingLong(
1523 src=main.params[ 'PING' ][ 'source6' ],
1524 target=main.params[ 'PING' ][ 'target6' ],
1525 pingTime=500 )
1526 main.Mininet2.pingLong(
1527 src=main.params[ 'PING' ][ 'source7' ],
1528 target=main.params[ 'PING' ][ 'target7' ],
1529 pingTime=500 )
1530 main.Mininet2.pingLong(
1531 src=main.params[ 'PING' ][ 'source8' ],
1532 target=main.params[ 'PING' ][ 'target8' ],
1533 pingTime=500 )
1534 main.Mininet2.pingLong(
1535 src=main.params[ 'PING' ][ 'source9' ],
1536 target=main.params[ 'PING' ][ 'target9' ],
1537 pingTime=500 )
1538 main.Mininet2.pingLong(
1539 src=main.params[ 'PING' ][ 'source10' ],
1540 target=main.params[ 'PING' ][ 'target10' ],
1541 pingTime=500 )
1542
1543 main.step( "Collecting topology information from ONOS" )
1544 devices = []
1545 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001546 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001547 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001548 name="devices-" + str( i ),
1549 args=[ ] )
1550 threads.append( t )
1551 t.start()
1552
1553 for t in threads:
1554 t.join()
1555 devices.append( t.result )
1556 hosts = []
1557 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001558 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001559 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001560 name="hosts-" + str( i ),
1561 args=[ ] )
1562 threads.append( t )
1563 t.start()
1564
1565 for t in threads:
1566 t.join()
1567 try:
1568 hosts.append( json.loads( t.result ) )
1569 except ( ValueError, TypeError ):
1570 # FIXME: better handling of this, print which node
1571 # Maybe use thread name?
1572 main.log.exception( "Error parsing json output of hosts" )
Jon Hall3afe4c92015-12-14 19:30:38 -08001573 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001574 hosts.append( None )
1575
1576 ports = []
1577 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001578 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001579 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001580 name="ports-" + str( i ),
1581 args=[ ] )
1582 threads.append( t )
1583 t.start()
1584
1585 for t in threads:
1586 t.join()
1587 ports.append( t.result )
1588 links = []
1589 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001590 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001591 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001592 name="links-" + str( i ),
1593 args=[ ] )
1594 threads.append( t )
1595 t.start()
1596
1597 for t in threads:
1598 t.join()
1599 links.append( t.result )
1600 clusters = []
1601 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001602 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001603 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001604 name="clusters-" + str( i ),
1605 args=[ ] )
1606 threads.append( t )
1607 t.start()
1608
1609 for t in threads:
1610 t.join()
1611 clusters.append( t.result )
1612 # Compare json objects for hosts and dataplane clusters
1613
1614 # hosts
1615 main.step( "Host view is consistent across ONOS nodes" )
1616 consistentHostsResult = main.TRUE
1617 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001618 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001619 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001620 if hosts[ controller ] == hosts[ 0 ]:
1621 continue
1622 else: # hosts not consistent
1623 main.log.error( "hosts from ONOS" +
1624 controllerStr +
1625 " is inconsistent with ONOS1" )
1626 main.log.warn( repr( hosts[ controller ] ) )
1627 consistentHostsResult = main.FALSE
1628
1629 else:
1630 main.log.error( "Error in getting ONOS hosts from ONOS" +
1631 controllerStr )
1632 consistentHostsResult = main.FALSE
1633 main.log.warn( "ONOS" + controllerStr +
1634 " hosts response: " +
1635 repr( hosts[ controller ] ) )
1636 utilities.assert_equals(
1637 expect=main.TRUE,
1638 actual=consistentHostsResult,
1639 onpass="Hosts view is consistent across all ONOS nodes",
1640 onfail="ONOS nodes have different views of hosts" )
1641
1642 main.step( "Each host has an IP address" )
1643 ipResult = main.TRUE
1644 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001645 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001646 if hosts[ controller ]:
1647 for host in hosts[ controller ]:
1648 if not host.get( 'ipAddresses', [ ] ):
Jon Hallf3d16e72015-12-16 17:45:08 -08001649 main.log.error( "Error with host ips on controller" +
Jon Hall3afe4c92015-12-14 19:30:38 -08001650 controllerStr + ": " + str( host ) )
1651 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001652 utilities.assert_equals(
1653 expect=main.TRUE,
1654 actual=ipResult,
1655 onpass="The ips of the hosts aren't empty",
1656 onfail="The ip of at least one host is missing" )
1657
1658 # Strongly connected clusters of devices
1659 main.step( "Cluster view is consistent across ONOS nodes" )
1660 consistentClustersResult = main.TRUE
1661 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07001662 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001663 if "Error" not in clusters[ controller ]:
1664 if clusters[ controller ] == clusters[ 0 ]:
1665 continue
1666 else: # clusters not consistent
1667 main.log.error( "clusters from ONOS" + controllerStr +
1668 " is inconsistent with ONOS1" )
1669 consistentClustersResult = main.FALSE
1670
1671 else:
1672 main.log.error( "Error in getting dataplane clusters " +
1673 "from ONOS" + controllerStr )
1674 consistentClustersResult = main.FALSE
1675 main.log.warn( "ONOS" + controllerStr +
1676 " clusters response: " +
1677 repr( clusters[ controller ] ) )
1678 utilities.assert_equals(
1679 expect=main.TRUE,
1680 actual=consistentClustersResult,
1681 onpass="Clusters view is consistent across all ONOS nodes",
1682 onfail="ONOS nodes have different views of clusters" )
Jon Hall172b7ba2016-04-07 18:12:20 -07001683 if consistentClustersResult != main.TRUE:
1684 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07001685 # there should always only be one cluster
1686 main.step( "Cluster view correct across ONOS nodes" )
1687 try:
1688 numClusters = len( json.loads( clusters[ 0 ] ) )
1689 except ( ValueError, TypeError ):
1690 main.log.exception( "Error parsing clusters[0]: " +
1691 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001692 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001693 clusterResults = main.FALSE
1694 if numClusters == 1:
1695 clusterResults = main.TRUE
1696 utilities.assert_equals(
1697 expect=1,
1698 actual=numClusters,
1699 onpass="ONOS shows 1 SCC",
1700 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1701
1702 main.step( "Comparing ONOS topology to MN" )
1703 devicesResults = main.TRUE
1704 linksResults = main.TRUE
1705 hostsResults = main.TRUE
1706 mnSwitches = main.Mininet1.getSwitches()
1707 mnLinks = main.Mininet1.getLinks()
1708 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001709 for controller in main.activeNodes:
1710 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001711 if devices[ controller ] and ports[ controller ] and\
1712 "Error" not in devices[ controller ] and\
1713 "Error" not in ports[ controller ]:
Jon Halle1a3b752015-07-22 13:02:46 -07001714 currentDevicesResult = main.Mininet1.compareSwitches(
1715 mnSwitches,
1716 json.loads( devices[ controller ] ),
1717 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001718 else:
1719 currentDevicesResult = main.FALSE
1720 utilities.assert_equals( expect=main.TRUE,
1721 actual=currentDevicesResult,
1722 onpass="ONOS" + controllerStr +
1723 " Switches view is correct",
1724 onfail="ONOS" + controllerStr +
1725 " Switches view is incorrect" )
1726 if links[ controller ] and "Error" not in links[ controller ]:
1727 currentLinksResult = main.Mininet1.compareLinks(
1728 mnSwitches, mnLinks,
1729 json.loads( links[ controller ] ) )
1730 else:
1731 currentLinksResult = main.FALSE
1732 utilities.assert_equals( expect=main.TRUE,
1733 actual=currentLinksResult,
1734 onpass="ONOS" + controllerStr +
1735 " links view is correct",
1736 onfail="ONOS" + controllerStr +
1737 " links view is incorrect" )
1738
Jon Hall657cdf62015-12-17 14:40:51 -08001739 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001740 currentHostsResult = main.Mininet1.compareHosts(
1741 mnHosts,
1742 hosts[ controller ] )
1743 else:
1744 currentHostsResult = main.FALSE
1745 utilities.assert_equals( expect=main.TRUE,
1746 actual=currentHostsResult,
1747 onpass="ONOS" + controllerStr +
1748 " hosts exist in Mininet",
1749 onfail="ONOS" + controllerStr +
1750 " hosts don't match Mininet" )
1751
1752 devicesResults = devicesResults and currentDevicesResult
1753 linksResults = linksResults and currentLinksResult
1754 hostsResults = hostsResults and currentHostsResult
1755
1756 main.step( "Device information is correct" )
1757 utilities.assert_equals(
1758 expect=main.TRUE,
1759 actual=devicesResults,
1760 onpass="Device information is correct",
1761 onfail="Device information is incorrect" )
1762
1763 main.step( "Links are correct" )
1764 utilities.assert_equals(
1765 expect=main.TRUE,
1766 actual=linksResults,
1767 onpass="Link are correct",
1768 onfail="Links are incorrect" )
1769
1770 main.step( "Hosts are correct" )
1771 utilities.assert_equals(
1772 expect=main.TRUE,
1773 actual=hostsResults,
1774 onpass="Hosts are correct",
1775 onfail="Hosts are incorrect" )
1776
1777 def CASE6( self, main ):
1778 """
1779 The Failure case.
1780 """
1781 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001782 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001783 assert main, "main not defined"
1784 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001785 assert main.CLIs, "main.CLIs not defined"
1786 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001787 try:
1788 labels
1789 except NameError:
1790 main.log.error( "labels not defined, setting to []" )
1791 global labels
1792 labels = []
1793 try:
1794 data
1795 except NameError:
1796 main.log.error( "data not defined, setting to []" )
1797 global data
1798 data = []
1799 # Reset non-persistent variables
1800 try:
1801 iCounterValue = 0
1802 except NameError:
1803 main.log.error( "iCounterValue not defined, setting to 0" )
1804 iCounterValue = 0
1805
1806 main.case( "Restart entire ONOS cluster" )
1807
Jon Hall5ec6b1b2015-09-17 18:20:14 -07001808 main.step( "Checking ONOS Logs for errors" )
1809 for node in main.nodes:
1810 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1811 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1812
Jon Hall5cf14d52015-07-16 12:15:19 -07001813 main.step( "Killing ONOS nodes" )
1814 killResults = main.TRUE
1815 killTime = time.time()
Jon Halle1a3b752015-07-22 13:02:46 -07001816 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001817 killed = main.ONOSbench.onosKill( node.ip_address )
1818 killResults = killResults and killed
1819 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1820 onpass="ONOS nodes killed",
1821 onfail="ONOS kill unsuccessful" )
1822
1823 main.step( "Checking if ONOS is up yet" )
1824 for i in range( 2 ):
1825 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001826 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001827 started = main.ONOSbench.isup( node.ip_address )
1828 if not started:
1829 main.log.error( node.name + " didn't start!" )
1830 onosIsupResult = onosIsupResult and started
1831 if onosIsupResult == main.TRUE:
1832 break
1833 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1834 onpass="ONOS restarted",
1835 onfail="ONOS restart NOT successful" )
1836
1837 main.log.step( "Starting ONOS CLI sessions" )
1838 cliResults = main.TRUE
1839 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001840 for i in range( main.numCtrls ):
1841 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -07001842 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -07001843 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001844 threads.append( t )
1845 t.start()
1846
1847 for t in threads:
1848 t.join()
1849 cliResults = cliResults and t.result
1850 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1851 onpass="ONOS cli started",
1852 onfail="ONOS clis did not restart" )
1853
Jon Hall6e709752016-02-01 13:38:46 -08001854 for i in range( 10 ):
1855 ready = True
1856 for cli in main.CLIs:
1857 output = cli.summary()
1858 if not output:
1859 ready = False
1860 time.sleep( 30 )
1861 utilities.assert_equals( expect=True, actual=ready,
1862 onpass="ONOS summary command succeded",
1863 onfail="ONOS summary command failed" )
1864 if not ready:
1865 main.cleanup()
1866 main.exit()
1867
Jon Hall5cf14d52015-07-16 12:15:19 -07001868 # Grab the time of restart so we chan check how long the gossip
1869 # protocol has had time to work
1870 main.restartTime = time.time() - killTime
1871 main.log.debug( "Restart time: " + str( main.restartTime ) )
1872 labels.append( "Restart" )
1873 data.append( str( main.restartTime ) )
1874
Jon Hall5cf14d52015-07-16 12:15:19 -07001875 # Rerun for election on restarted nodes
1876 runResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001877 for cli in main.CLIs:
Jon Halla440e872016-03-31 15:15:50 -07001878 run = cli.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001879 if run != main.TRUE:
1880 main.log.error( "Error running for election on " + cli.name )
1881 runResults = runResults and run
1882 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1883 onpass="Reran for election",
1884 onfail="Failed to rerun for election" )
1885
1886 # TODO: Make this configurable
1887 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -07001888 node = main.activeNodes[0]
1889 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1890 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1891 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001892
1893 def CASE7( self, main ):
1894 """
1895 Check state after ONOS failure
1896 """
1897 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001898 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001899 assert main, "main not defined"
1900 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001901 assert main.CLIs, "main.CLIs not defined"
1902 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001903 main.case( "Running ONOS Constant State Tests" )
1904
1905 main.step( "Check that each switch has a master" )
1906 # Assert that each device has a master
1907 rolesNotNull = main.TRUE
1908 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001909 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001910 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001911 name="rolesNotNull-" + str( i ),
1912 args=[ ] )
1913 threads.append( t )
1914 t.start()
1915
1916 for t in threads:
1917 t.join()
1918 rolesNotNull = rolesNotNull and t.result
1919 utilities.assert_equals(
1920 expect=main.TRUE,
1921 actual=rolesNotNull,
1922 onpass="Each device has a master",
1923 onfail="Some devices don't have a master assigned" )
1924
1925 main.step( "Read device roles from ONOS" )
1926 ONOSMastership = []
1927 mastershipCheck = main.FALSE
1928 consistentMastership = True
1929 rolesResults = True
1930 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001931 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001932 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001933 name="roles-" + str( i ),
1934 args=[] )
1935 threads.append( t )
1936 t.start()
1937
1938 for t in threads:
1939 t.join()
1940 ONOSMastership.append( t.result )
1941
Jon Halla440e872016-03-31 15:15:50 -07001942 for i in range( len( ONOSMastership ) ):
1943 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001944 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001945 main.log.error( "Error in getting ONOS" + node + " roles" )
1946 main.log.warn( "ONOS" + node + " mastership response: " +
1947 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001948 rolesResults = False
1949 utilities.assert_equals(
1950 expect=True,
1951 actual=rolesResults,
1952 onpass="No error in reading roles output",
1953 onfail="Error in reading roles from ONOS" )
1954
1955 main.step( "Check for consistency in roles from each controller" )
1956 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1957 main.log.info(
1958 "Switch roles are consistent across all ONOS nodes" )
1959 else:
1960 consistentMastership = False
1961 utilities.assert_equals(
1962 expect=True,
1963 actual=consistentMastership,
1964 onpass="Switch roles are consistent across all ONOS nodes",
1965 onfail="ONOS nodes have different views of switch roles" )
1966
1967 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001968 for i in range( len( ONOSMastership ) ):
1969 node = str( main.activeNodes[i] + 1 )
1970 main.log.warn( "ONOS" + node + " roles: ",
Jon Hall6e709752016-02-01 13:38:46 -08001971 json.dumps( json.loads( ONOSMastership[ i ] ),
1972 sort_keys=True,
1973 indent=4,
1974 separators=( ',', ': ' ) ) )
1975 elif rolesResults and consistentMastership:
Jon Hall5cf14d52015-07-16 12:15:19 -07001976 mastershipCheck = main.TRUE
1977
Jon Hall5cf14d52015-07-16 12:15:19 -07001978 # NOTE: we expect mastership to change on controller failure
1979
1980 main.step( "Get the intents and compare across all nodes" )
1981 ONOSIntents = []
1982 intentCheck = main.FALSE
1983 consistentIntents = True
1984 intentsResults = True
1985 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001986 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001987 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001988 name="intents-" + str( i ),
1989 args=[],
1990 kwargs={ 'jsonFormat': True } )
1991 threads.append( t )
1992 t.start()
1993
1994 for t in threads:
1995 t.join()
1996 ONOSIntents.append( t.result )
1997
Jon Halla440e872016-03-31 15:15:50 -07001998 for i in range( len( ONOSIntents) ):
1999 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002000 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07002001 main.log.error( "Error in getting ONOS" + node + " intents" )
2002 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002003 repr( ONOSIntents[ i ] ) )
2004 intentsResults = False
2005 utilities.assert_equals(
2006 expect=True,
2007 actual=intentsResults,
2008 onpass="No error in reading intents output",
2009 onfail="Error in reading intents from ONOS" )
2010
2011 main.step( "Check for consistency in Intents from each controller" )
2012 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2013 main.log.info( "Intents are consistent across all ONOS " +
2014 "nodes" )
2015 else:
2016 consistentIntents = False
2017
2018 # Try to make it easy to figure out what is happening
2019 #
2020 # Intent ONOS1 ONOS2 ...
2021 # 0x01 INSTALLED INSTALLING
2022 # ... ... ...
2023 # ... ... ...
2024 title = " ID"
Jon Halla440e872016-03-31 15:15:50 -07002025 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002026 title += " " * 10 + "ONOS" + str( n + 1 )
2027 main.log.warn( title )
2028 # get all intent keys in the cluster
2029 keys = []
2030 for nodeStr in ONOSIntents:
2031 node = json.loads( nodeStr )
2032 for intent in node:
2033 keys.append( intent.get( 'id' ) )
2034 keys = set( keys )
2035 for key in keys:
2036 row = "%-13s" % key
2037 for nodeStr in ONOSIntents:
2038 node = json.loads( nodeStr )
2039 for intent in node:
2040 if intent.get( 'id' ) == key:
2041 row += "%-15s" % intent.get( 'state' )
2042 main.log.warn( row )
2043 # End table view
2044
2045 utilities.assert_equals(
2046 expect=True,
2047 actual=consistentIntents,
2048 onpass="Intents are consistent across all ONOS nodes",
2049 onfail="ONOS nodes have different views of intents" )
2050 intentStates = []
2051 for node in ONOSIntents: # Iter through ONOS nodes
2052 nodeStates = []
2053 # Iter through intents of a node
2054 try:
2055 for intent in json.loads( node ):
2056 nodeStates.append( intent[ 'state' ] )
2057 except ( ValueError, TypeError ):
2058 main.log.exception( "Error in parsing intents" )
2059 main.log.error( repr( node ) )
2060 intentStates.append( nodeStates )
2061 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2062 main.log.info( dict( out ) )
2063
2064 if intentsResults and not consistentIntents:
Jon Halla440e872016-03-31 15:15:50 -07002065 for i in range( len( main.activeNodes ) ):
2066 node = str( main.activeNodes[i] + 1 )
2067 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002068 main.log.warn( json.dumps(
2069 json.loads( ONOSIntents[ i ] ),
2070 sort_keys=True,
2071 indent=4,
2072 separators=( ',', ': ' ) ) )
2073 elif intentsResults and consistentIntents:
2074 intentCheck = main.TRUE
2075
2076 # NOTE: Store has no durability, so intents are lost across system
2077 # restarts
2078 """
2079 main.step( "Compare current intents with intents before the failure" )
2080 # NOTE: this requires case 5 to pass for intentState to be set.
2081 # maybe we should stop the test if that fails?
2082 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002083 try:
2084 intentState
2085 except NameError:
2086 main.log.warn( "No previous intent state was saved" )
2087 else:
2088 if intentState and intentState == ONOSIntents[ 0 ]:
2089 sameIntents = main.TRUE
2090 main.log.info( "Intents are consistent with before failure" )
2091 # TODO: possibly the states have changed? we may need to figure out
2092 # what the acceptable states are
2093 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2094 sameIntents = main.TRUE
2095 try:
2096 before = json.loads( intentState )
2097 after = json.loads( ONOSIntents[ 0 ] )
2098 for intent in before:
2099 if intent not in after:
2100 sameIntents = main.FALSE
2101 main.log.debug( "Intent is not currently in ONOS " +
2102 "(at least in the same form):" )
2103 main.log.debug( json.dumps( intent ) )
2104 except ( ValueError, TypeError ):
2105 main.log.exception( "Exception printing intents" )
2106 main.log.debug( repr( ONOSIntents[0] ) )
2107 main.log.debug( repr( intentState ) )
2108 if sameIntents == main.FALSE:
2109 try:
2110 main.log.debug( "ONOS intents before: " )
2111 main.log.debug( json.dumps( json.loads( intentState ),
2112 sort_keys=True, indent=4,
2113 separators=( ',', ': ' ) ) )
2114 main.log.debug( "Current ONOS intents: " )
2115 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2116 sort_keys=True, indent=4,
2117 separators=( ',', ': ' ) ) )
2118 except ( ValueError, TypeError ):
2119 main.log.exception( "Exception printing intents" )
2120 main.log.debug( repr( ONOSIntents[0] ) )
2121 main.log.debug( repr( intentState ) )
2122 utilities.assert_equals(
2123 expect=main.TRUE,
2124 actual=sameIntents,
2125 onpass="Intents are consistent with before failure",
2126 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002127 intentCheck = intentCheck and sameIntents
2128 """
2129 main.step( "Get the OF Table entries and compare to before " +
2130 "component failure" )
2131 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002132 for i in range( 28 ):
2133 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002134 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002135 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2136 FlowTables = FlowTables and curSwitch
2137 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002138 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002139 utilities.assert_equals(
2140 expect=main.TRUE,
2141 actual=FlowTables,
2142 onpass="No changes were found in the flow tables",
2143 onfail="Changes were found in the flow tables" )
2144
2145 main.Mininet2.pingLongKill()
2146 '''
2147 # main.step( "Check the continuous pings to ensure that no packets " +
2148 # "were dropped during component failure" )
2149 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2150 main.params[ 'TESTONIP' ] )
2151 LossInPings = main.FALSE
2152 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2153 for i in range( 8, 18 ):
2154 main.log.info(
2155 "Checking for a loss in pings along flow from s" +
2156 str( i ) )
2157 LossInPings = main.Mininet2.checkForLoss(
2158 "/tmp/ping.h" +
2159 str( i ) ) or LossInPings
2160 if LossInPings == main.TRUE:
2161 main.log.info( "Loss in ping detected" )
2162 elif LossInPings == main.ERROR:
2163 main.log.info( "There are multiple mininet process running" )
2164 elif LossInPings == main.FALSE:
2165 main.log.info( "No Loss in the pings" )
2166 main.log.info( "No loss of dataplane connectivity" )
2167 # utilities.assert_equals(
2168 # expect=main.FALSE,
2169 # actual=LossInPings,
2170 # onpass="No Loss of connectivity",
2171 # onfail="Loss of dataplane connectivity detected" )
2172
2173 # NOTE: Since intents are not persisted with IntnentStore,
2174 # we expect loss in dataplane connectivity
2175 LossInPings = main.FALSE
2176 '''
2177
2178 main.step( "Leadership Election is still functional" )
2179 # Test of LeadershipElection
2180 leaderList = []
2181 leaderResult = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002182
2183 for i in main.activeNodes:
2184 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002185 leaderN = cli.electionTestLeader()
2186 leaderList.append( leaderN )
2187 if leaderN == main.FALSE:
2188 # error in response
2189 main.log.error( "Something is wrong with " +
2190 "electionTestLeader function, check the" +
2191 " error logs" )
2192 leaderResult = main.FALSE
2193 elif leaderN is None:
2194 main.log.error( cli.name +
2195 " shows no leader for the election-app." )
2196 leaderResult = main.FALSE
2197 if len( set( leaderList ) ) != 1:
2198 leaderResult = main.FALSE
2199 main.log.error(
2200 "Inconsistent view of leader for the election test app" )
2201 # TODO: print the list
2202 utilities.assert_equals(
2203 expect=main.TRUE,
2204 actual=leaderResult,
2205 onpass="Leadership election passed",
2206 onfail="Something went wrong with Leadership election" )
2207
2208 def CASE8( self, main ):
2209 """
2210 Compare topo
2211 """
2212 import json
2213 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002214 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002215 assert main, "main not defined"
2216 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002217 assert main.CLIs, "main.CLIs not defined"
2218 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002219
2220 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002221 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002222 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002223 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002224 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002225 elapsed = 0
2226 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002227 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002228 startTime = time.time()
2229 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002230 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hallba609822015-09-18 12:00:21 -07002231 devicesResults = main.TRUE
2232 linksResults = main.TRUE
2233 hostsResults = main.TRUE
2234 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002235 count += 1
2236 cliStart = time.time()
2237 devices = []
2238 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002239 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002240 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002241 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002242 args=[ main.CLIs[i].devices, [ None ] ],
2243 kwargs= { 'sleep': 5, 'attempts': 5,
2244 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002245 threads.append( t )
2246 t.start()
2247
2248 for t in threads:
2249 t.join()
2250 devices.append( t.result )
2251 hosts = []
2252 ipResult = main.TRUE
2253 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002254 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002255 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002256 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002257 args=[ main.CLIs[i].hosts, [ None ] ],
2258 kwargs= { 'sleep': 5, 'attempts': 5,
2259 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002260 threads.append( t )
2261 t.start()
2262
2263 for t in threads:
2264 t.join()
2265 try:
2266 hosts.append( json.loads( t.result ) )
2267 except ( ValueError, TypeError ):
2268 main.log.exception( "Error parsing hosts results" )
2269 main.log.error( repr( t.result ) )
Jon Hall3afe4c92015-12-14 19:30:38 -08002270 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002271 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002272 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002273 if hosts[ controller ]:
2274 for host in hosts[ controller ]:
2275 if host is None or host.get( 'ipAddresses', [] ) == []:
2276 main.log.error(
2277 "Error with host ipAddresses on controller" +
2278 controllerStr + ": " + str( host ) )
2279 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002280 ports = []
2281 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002282 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002283 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002284 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002285 args=[ main.CLIs[i].ports, [ None ] ],
2286 kwargs= { 'sleep': 5, 'attempts': 5,
2287 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002288 threads.append( t )
2289 t.start()
2290
2291 for t in threads:
2292 t.join()
2293 ports.append( t.result )
2294 links = []
2295 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002296 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002297 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002298 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002299 args=[ main.CLIs[i].links, [ None ] ],
2300 kwargs= { 'sleep': 5, 'attempts': 5,
2301 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002302 threads.append( t )
2303 t.start()
2304
2305 for t in threads:
2306 t.join()
2307 links.append( t.result )
2308 clusters = []
2309 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002310 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002311 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002312 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002313 args=[ main.CLIs[i].clusters, [ None ] ],
2314 kwargs= { 'sleep': 5, 'attempts': 5,
2315 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002316 threads.append( t )
2317 t.start()
2318
2319 for t in threads:
2320 t.join()
2321 clusters.append( t.result )
2322
2323 elapsed = time.time() - startTime
2324 cliTime = time.time() - cliStart
2325 print "Elapsed time: " + str( elapsed )
2326 print "CLI time: " + str( cliTime )
2327
Jon Hall6e709752016-02-01 13:38:46 -08002328 if all( e is None for e in devices ) and\
2329 all( e is None for e in hosts ) and\
2330 all( e is None for e in ports ) and\
2331 all( e is None for e in links ) and\
2332 all( e is None for e in clusters ):
2333 topoFailMsg = "Could not get topology from ONOS"
2334 main.log.error( topoFailMsg )
2335 continue # Try again, No use trying to compare
2336
Jon Hall5cf14d52015-07-16 12:15:19 -07002337 mnSwitches = main.Mininet1.getSwitches()
2338 mnLinks = main.Mininet1.getLinks()
2339 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07002340 for controller in range( len( main.activeNodes ) ):
2341 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002342 if devices[ controller ] and ports[ controller ] and\
2343 "Error" not in devices[ controller ] and\
2344 "Error" not in ports[ controller ]:
2345
Jon Hallc6793552016-01-19 14:18:37 -08002346 try:
2347 currentDevicesResult = main.Mininet1.compareSwitches(
2348 mnSwitches,
2349 json.loads( devices[ controller ] ),
2350 json.loads( ports[ controller ] ) )
2351 except ( TypeError, ValueError ) as e:
2352 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2353 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002354 else:
2355 currentDevicesResult = main.FALSE
2356 utilities.assert_equals( expect=main.TRUE,
2357 actual=currentDevicesResult,
2358 onpass="ONOS" + controllerStr +
2359 " Switches view is correct",
2360 onfail="ONOS" + controllerStr +
2361 " Switches view is incorrect" )
2362
2363 if links[ controller ] and "Error" not in links[ controller ]:
2364 currentLinksResult = main.Mininet1.compareLinks(
2365 mnSwitches, mnLinks,
2366 json.loads( links[ controller ] ) )
2367 else:
2368 currentLinksResult = main.FALSE
2369 utilities.assert_equals( expect=main.TRUE,
2370 actual=currentLinksResult,
2371 onpass="ONOS" + controllerStr +
2372 " links view is correct",
2373 onfail="ONOS" + controllerStr +
2374 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002375 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002376 currentHostsResult = main.Mininet1.compareHosts(
2377 mnHosts,
2378 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002379 elif hosts[ controller ] == []:
2380 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002381 else:
2382 currentHostsResult = main.FALSE
2383 utilities.assert_equals( expect=main.TRUE,
2384 actual=currentHostsResult,
2385 onpass="ONOS" + controllerStr +
2386 " hosts exist in Mininet",
2387 onfail="ONOS" + controllerStr +
2388 " hosts don't match Mininet" )
2389 # CHECKING HOST ATTACHMENT POINTS
2390 hostAttachment = True
Jon Halla440e872016-03-31 15:15:50 -07002391 zeroHosts = False
Jon Hall5cf14d52015-07-16 12:15:19 -07002392 # FIXME: topo-HA/obelisk specific mappings:
2393 # key is mac and value is dpid
2394 mappings = {}
2395 for i in range( 1, 29 ): # hosts 1 through 28
2396 # set up correct variables:
2397 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2398 if i == 1:
2399 deviceId = "1000".zfill(16)
2400 elif i == 2:
2401 deviceId = "2000".zfill(16)
2402 elif i == 3:
2403 deviceId = "3000".zfill(16)
2404 elif i == 4:
2405 deviceId = "3004".zfill(16)
2406 elif i == 5:
2407 deviceId = "5000".zfill(16)
2408 elif i == 6:
2409 deviceId = "6000".zfill(16)
2410 elif i == 7:
2411 deviceId = "6007".zfill(16)
2412 elif i >= 8 and i <= 17:
2413 dpid = '3' + str( i ).zfill( 3 )
2414 deviceId = dpid.zfill(16)
2415 elif i >= 18 and i <= 27:
2416 dpid = '6' + str( i ).zfill( 3 )
2417 deviceId = dpid.zfill(16)
2418 elif i == 28:
2419 deviceId = "2800".zfill(16)
2420 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002421 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002422 if hosts[ controller ] == []:
2423 main.log.warn( "There are no hosts discovered" )
Jon Halla440e872016-03-31 15:15:50 -07002424 zeroHosts = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002425 else:
2426 for host in hosts[ controller ]:
2427 mac = None
2428 location = None
2429 device = None
2430 port = None
2431 try:
2432 mac = host.get( 'mac' )
2433 assert mac, "mac field could not be found for this host object"
2434
2435 location = host.get( 'location' )
2436 assert location, "location field could not be found for this host object"
2437
2438 # Trim the protocol identifier off deviceId
2439 device = str( location.get( 'elementId' ) ).split(':')[1]
2440 assert device, "elementId field could not be found for this host location object"
2441
2442 port = location.get( 'port' )
2443 assert port, "port field could not be found for this host location object"
2444
2445 # Now check if this matches where they should be
2446 if mac and device and port:
2447 if str( port ) != "1":
2448 main.log.error( "The attachment port is incorrect for " +
2449 "host " + str( mac ) +
2450 ". Expected: 1 Actual: " + str( port) )
2451 hostAttachment = False
2452 if device != mappings[ str( mac ) ]:
2453 main.log.error( "The attachment device is incorrect for " +
2454 "host " + str( mac ) +
2455 ". Expected: " + mappings[ str( mac ) ] +
2456 " Actual: " + device )
2457 hostAttachment = False
2458 else:
2459 hostAttachment = False
2460 except AssertionError:
2461 main.log.exception( "Json object not as expected" )
2462 main.log.error( repr( host ) )
2463 hostAttachment = False
2464 else:
2465 main.log.error( "No hosts json output or \"Error\"" +
2466 " in output. hosts = " +
2467 repr( hosts[ controller ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002468 if zeroHosts is False:
Jon Hall5cf14d52015-07-16 12:15:19 -07002469 # TODO: Find a way to know if there should be hosts in a
2470 # given point of the test
2471 hostAttachment = True
2472
2473 # END CHECKING HOST ATTACHMENT POINTS
2474 devicesResults = devicesResults and currentDevicesResult
2475 linksResults = linksResults and currentLinksResult
2476 hostsResults = hostsResults and currentHostsResult
2477 hostAttachmentResults = hostAttachmentResults and\
2478 hostAttachment
2479 topoResult = ( devicesResults and linksResults
2480 and hostsResults and ipResult and
2481 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002482 utilities.assert_equals( expect=True,
2483 actual=topoResult,
2484 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002485 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002486 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002487
2488 # Compare json objects for hosts and dataplane clusters
2489
2490 # hosts
2491 main.step( "Hosts view is consistent across all ONOS nodes" )
2492 consistentHostsResult = main.TRUE
2493 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002494 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002495 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002496 if hosts[ controller ] == hosts[ 0 ]:
2497 continue
2498 else: # hosts not consistent
2499 main.log.error( "hosts from ONOS" + controllerStr +
2500 " is inconsistent with ONOS1" )
2501 main.log.warn( repr( hosts[ controller ] ) )
2502 consistentHostsResult = main.FALSE
2503
2504 else:
2505 main.log.error( "Error in getting ONOS hosts from ONOS" +
2506 controllerStr )
2507 consistentHostsResult = main.FALSE
2508 main.log.warn( "ONOS" + controllerStr +
2509 " hosts response: " +
2510 repr( hosts[ controller ] ) )
2511 utilities.assert_equals(
2512 expect=main.TRUE,
2513 actual=consistentHostsResult,
2514 onpass="Hosts view is consistent across all ONOS nodes",
2515 onfail="ONOS nodes have different views of hosts" )
2516
2517 main.step( "Hosts information is correct" )
2518 hostsResults = hostsResults and ipResult
2519 utilities.assert_equals(
2520 expect=main.TRUE,
2521 actual=hostsResults,
2522 onpass="Host information is correct",
2523 onfail="Host information is incorrect" )
2524
2525 main.step( "Host attachment points to the network" )
2526 utilities.assert_equals(
2527 expect=True,
2528 actual=hostAttachmentResults,
2529 onpass="Hosts are correctly attached to the network",
2530 onfail="ONOS did not correctly attach hosts to the network" )
2531
2532 # Strongly connected clusters of devices
2533 main.step( "Clusters view is consistent across all ONOS nodes" )
2534 consistentClustersResult = main.TRUE
2535 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07002536 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002537 if "Error" not in clusters[ controller ]:
2538 if clusters[ controller ] == clusters[ 0 ]:
2539 continue
2540 else: # clusters not consistent
2541 main.log.error( "clusters from ONOS" +
2542 controllerStr +
2543 " is inconsistent with ONOS1" )
2544 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002545 else:
2546 main.log.error( "Error in getting dataplane clusters " +
2547 "from ONOS" + controllerStr )
2548 consistentClustersResult = main.FALSE
2549 main.log.warn( "ONOS" + controllerStr +
2550 " clusters response: " +
2551 repr( clusters[ controller ] ) )
2552 utilities.assert_equals(
2553 expect=main.TRUE,
2554 actual=consistentClustersResult,
2555 onpass="Clusters view is consistent across all ONOS nodes",
2556 onfail="ONOS nodes have different views of clusters" )
2557
2558 main.step( "There is only one SCC" )
2559 # there should always only be one cluster
2560 try:
2561 numClusters = len( json.loads( clusters[ 0 ] ) )
2562 except ( ValueError, TypeError ):
2563 main.log.exception( "Error parsing clusters[0]: " +
2564 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002565 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002566 clusterResults = main.FALSE
2567 if numClusters == 1:
2568 clusterResults = main.TRUE
2569 utilities.assert_equals(
2570 expect=1,
2571 actual=numClusters,
2572 onpass="ONOS shows 1 SCC",
2573 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2574
2575 topoResult = ( devicesResults and linksResults
2576 and hostsResults and consistentHostsResult
2577 and consistentClustersResult and clusterResults
2578 and ipResult and hostAttachmentResults )
2579
2580 topoResult = topoResult and int( count <= 2 )
2581 note = "note it takes about " + str( int( cliTime ) ) + \
2582 " seconds for the test to make all the cli calls to fetch " +\
2583 "the topology from each ONOS instance"
2584 main.log.info(
2585 "Very crass estimate for topology discovery/convergence( " +
2586 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2587 str( count ) + " tries" )
2588
2589 main.step( "Device information is correct" )
2590 utilities.assert_equals(
2591 expect=main.TRUE,
2592 actual=devicesResults,
2593 onpass="Device information is correct",
2594 onfail="Device information is incorrect" )
2595
2596 main.step( "Links are correct" )
2597 utilities.assert_equals(
2598 expect=main.TRUE,
2599 actual=linksResults,
2600 onpass="Link are correct",
2601 onfail="Links are incorrect" )
2602
Jon Halla440e872016-03-31 15:15:50 -07002603 main.step( "Hosts are correct" )
2604 utilities.assert_equals(
2605 expect=main.TRUE,
2606 actual=hostsResults,
2607 onpass="Hosts are correct",
2608 onfail="Hosts are incorrect" )
2609
Jon Hall5cf14d52015-07-16 12:15:19 -07002610 # FIXME: move this to an ONOS state case
2611 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002612 nodeResults = utilities.retry( main.HA.nodesCheck,
2613 False,
2614 args=[main.activeNodes],
2615 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002616
Jon Hall41d39f12016-04-11 22:54:35 -07002617 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002618 onpass="Nodes check successful",
2619 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002620 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002621 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002622 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002623 main.CLIs[i].name,
2624 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002625
2626 def CASE9( self, main ):
2627 """
2628 Link s3-s28 down
2629 """
2630 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002631 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002632 assert main, "main not defined"
2633 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002634 assert main.CLIs, "main.CLIs not defined"
2635 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002636 # NOTE: You should probably run a topology check after this
2637
2638 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2639
2640 description = "Turn off a link to ensure that Link Discovery " +\
2641 "is working properly"
2642 main.case( description )
2643
2644 main.step( "Kill Link between s3 and s28" )
2645 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2646 main.log.info( "Waiting " + str( linkSleep ) +
2647 " seconds for link down to be discovered" )
2648 time.sleep( linkSleep )
2649 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2650 onpass="Link down successful",
2651 onfail="Failed to bring link down" )
2652 # TODO do some sort of check here
2653
2654 def CASE10( self, main ):
2655 """
2656 Link s3-s28 up
2657 """
2658 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002659 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002660 assert main, "main not defined"
2661 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002662 assert main.CLIs, "main.CLIs not defined"
2663 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002664 # NOTE: You should probably run a topology check after this
2665
2666 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2667
2668 description = "Restore a link to ensure that Link Discovery is " + \
2669 "working properly"
2670 main.case( description )
2671
2672 main.step( "Bring link between s3 and s28 back up" )
2673 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2674 main.log.info( "Waiting " + str( linkSleep ) +
2675 " seconds for link up to be discovered" )
2676 time.sleep( linkSleep )
2677 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2678 onpass="Link up successful",
2679 onfail="Failed to bring link up" )
2680 # TODO do some sort of check here
2681
2682 def CASE11( self, main ):
2683 """
2684 Switch Down
2685 """
2686 # NOTE: You should probably run a topology check after this
2687 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002688 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002689 assert main, "main not defined"
2690 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002691 assert main.CLIs, "main.CLIs not defined"
2692 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002693
2694 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2695
2696 description = "Killing a switch to ensure it is discovered correctly"
Jon Halla440e872016-03-31 15:15:50 -07002697 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002698 main.case( description )
2699 switch = main.params[ 'kill' ][ 'switch' ]
2700 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2701
2702 # TODO: Make this switch parameterizable
2703 main.step( "Kill " + switch )
2704 main.log.info( "Deleting " + switch )
2705 main.Mininet1.delSwitch( switch )
2706 main.log.info( "Waiting " + str( switchSleep ) +
2707 " seconds for switch down to be discovered" )
2708 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002709 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002710 # Peek at the deleted switch
2711 main.log.warn( str( device ) )
2712 result = main.FALSE
2713 if device and device[ 'available' ] is False:
2714 result = main.TRUE
2715 utilities.assert_equals( expect=main.TRUE, actual=result,
2716 onpass="Kill switch successful",
2717 onfail="Failed to kill switch?" )
2718
2719 def CASE12( self, main ):
2720 """
2721 Switch Up
2722 """
2723 # NOTE: You should probably run a topology check after this
2724 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002725 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002726 assert main, "main not defined"
2727 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002728 assert main.CLIs, "main.CLIs not defined"
2729 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002730 assert ONOS1Port, "ONOS1Port not defined"
2731 assert ONOS2Port, "ONOS2Port not defined"
2732 assert ONOS3Port, "ONOS3Port not defined"
2733 assert ONOS4Port, "ONOS4Port not defined"
2734 assert ONOS5Port, "ONOS5Port not defined"
2735 assert ONOS6Port, "ONOS6Port not defined"
2736 assert ONOS7Port, "ONOS7Port not defined"
2737
2738 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2739 switch = main.params[ 'kill' ][ 'switch' ]
2740 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2741 links = main.params[ 'kill' ][ 'links' ].split()
Jon Halla440e872016-03-31 15:15:50 -07002742 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002743 description = "Adding a switch to ensure it is discovered correctly"
2744 main.case( description )
2745
2746 main.step( "Add back " + switch )
2747 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2748 for peer in links:
2749 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07002750 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002751 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2752 main.log.info( "Waiting " + str( switchSleep ) +
2753 " seconds for switch up to be discovered" )
2754 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002755 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002756 # Peek at the deleted switch
2757 main.log.warn( str( device ) )
2758 result = main.FALSE
2759 if device and device[ 'available' ]:
2760 result = main.TRUE
2761 utilities.assert_equals( expect=main.TRUE, actual=result,
2762 onpass="add switch successful",
2763 onfail="Failed to add switch?" )
2764
2765 def CASE13( self, main ):
2766 """
2767 Clean up
2768 """
2769 import os
2770 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002771 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002772 assert main, "main not defined"
2773 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002774 assert main.CLIs, "main.CLIs not defined"
2775 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002776
2777 # printing colors to terminal
2778 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2779 'blue': '\033[94m', 'green': '\033[92m',
2780 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2781 main.case( "Test Cleanup" )
2782 main.step( "Killing tcpdumps" )
2783 main.Mininet2.stopTcpdump()
2784
2785 testname = main.TEST
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002786 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002787 main.step( "Copying MN pcap and ONOS log files to test station" )
2788 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2789 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002790 # NOTE: MN Pcap file is being saved to logdir.
2791 # We scp this file as MN and TestON aren't necessarily the same vm
2792
2793 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002794 # TODO: Load these from params
2795 # NOTE: must end in /
2796 logFolder = "/opt/onos/log/"
2797 logFiles = [ "karaf.log", "karaf.log.1" ]
2798 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002799 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002800 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002801 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002802 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2803 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002804 # std*.log's
2805 # NOTE: must end in /
2806 logFolder = "/opt/onos/var/"
2807 logFiles = [ "stderr.log", "stdout.log" ]
2808 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002809 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002810 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002811 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002812 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2813 logFolder + f, dstName )
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002814 else:
2815 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002816
2817 main.step( "Stopping Mininet" )
2818 mnResult = main.Mininet1.stopNet()
2819 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2820 onpass="Mininet stopped",
2821 onfail="MN cleanup NOT successful" )
2822
2823 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002824 for node in main.nodes:
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002825 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2826 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002827
2828 try:
2829 timerLog = open( main.logdir + "/Timers.csv", 'w')
2830 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2831 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2832 timerLog.close()
2833 except NameError, e:
2834 main.log.exception(e)
2835
2836 def CASE14( self, main ):
2837 """
2838 start election app on all onos nodes
2839 """
Jon Halle1a3b752015-07-22 13:02:46 -07002840 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002841 assert main, "main not defined"
2842 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002843 assert main.CLIs, "main.CLIs not defined"
2844 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002845
2846 main.case("Start Leadership Election app")
2847 main.step( "Install leadership election app" )
Jon Halla440e872016-03-31 15:15:50 -07002848 onosCli = main.CLIs[ main.activeNodes[0] ]
2849 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002850 utilities.assert_equals(
2851 expect=main.TRUE,
2852 actual=appResult,
2853 onpass="Election app installed",
2854 onfail="Something went wrong with installing Leadership election" )
2855
2856 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002857 for i in main.activeNodes:
2858 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002859 time.sleep(5)
2860 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2861 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002862 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002863 expect=True,
2864 actual=sameResult,
2865 onpass="All nodes see the same leaderboards",
2866 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002867
Jon Hall25463a82016-04-13 14:03:52 -07002868 if sameResult:
2869 leader = leaders[ 0 ][ 0 ]
2870 if main.nodes[main.activeNodes[0]].ip_address in leader:
2871 correctLeader = True
2872 else:
2873 correctLeader = False
2874 main.step( "First node was elected leader" )
2875 utilities.assert_equals(
2876 expect=True,
2877 actual=correctLeader,
2878 onpass="Correct leader was elected",
2879 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002880
2881 def CASE15( self, main ):
2882 """
2883 Check that Leadership Election is still functional
acsmars9475b1c2015-08-28 18:02:08 -07002884 15.1 Run election on each node
2885 15.2 Check that each node has the same leaders and candidates
2886 15.3 Find current leader and withdraw
2887 15.4 Check that a new node was elected leader
2888 15.5 Check that that new leader was the candidate of old leader
2889 15.6 Run for election on old leader
2890 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2891 15.8 Make sure that the old leader was added to the candidate list
2892
2893 old and new variable prefixes refer to data from before vs after
2894 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002895 """
2896 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002897 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002898 assert main, "main not defined"
2899 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002900 assert main.CLIs, "main.CLIs not defined"
2901 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002902
Jon Hall5cf14d52015-07-16 12:15:19 -07002903 description = "Check that Leadership Election is still functional"
2904 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002905 # NOTE: Need to re-run after restarts since being a canidate is not persistant
acsmars9475b1c2015-08-28 18:02:08 -07002906
Jon Halla440e872016-03-31 15:15:50 -07002907 oldLeaders = [] # list of lists of each nodes' candidates before
2908 newLeaders = [] # list of lists of each nodes' candidates after
acsmars9475b1c2015-08-28 18:02:08 -07002909 oldLeader = '' # the old leader from oldLeaders, None if not same
2910 newLeader = '' # the new leaders fron newLoeaders, None if not same
2911 oldLeaderCLI = None # the CLI of the old leader used for re-electing
acsmars71adceb2015-08-31 15:09:26 -07002912 expectNoLeader = False # True when there is only one leader
2913 if main.numCtrls == 1:
2914 expectNoLeader = True
acsmars9475b1c2015-08-28 18:02:08 -07002915
Jon Hall5cf14d52015-07-16 12:15:19 -07002916 main.step( "Run for election on each node" )
acsmars9475b1c2015-08-28 18:02:08 -07002917 electionResult = main.TRUE
2918
Jon Halla440e872016-03-31 15:15:50 -07002919 for i in main.activeNodes: # run test election on each node
2920 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars9475b1c2015-08-28 18:02:08 -07002921 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002922 utilities.assert_equals(
2923 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002924 actual=electionResult,
2925 onpass="All nodes successfully ran for leadership",
2926 onfail="At least one node failed to run for leadership" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002927
acsmars3a72bde2015-09-02 14:16:22 -07002928 if electionResult == main.FALSE:
2929 main.log.error(
2930 "Skipping Test Case because Election Test App isn't loaded" )
2931 main.skipCase()
2932
acsmars9475b1c2015-08-28 18:02:08 -07002933 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002934 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002935 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002936 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002937 if sameResult:
2938 oldLeader = oldLeaders[ 0 ][ 0 ]
2939 main.log.warn( oldLeader )
acsmars9475b1c2015-08-28 18:02:08 -07002940 else:
Jon Halla440e872016-03-31 15:15:50 -07002941 oldLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002942 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002943 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002944 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002945 onpass="Leaderboards are consistent for the election topic",
acsmars9475b1c2015-08-28 18:02:08 -07002946 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002947
2948 main.step( "Find current leader and withdraw" )
acsmars9475b1c2015-08-28 18:02:08 -07002949 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002950 # do some sanity checking on leader before using it
acsmars9475b1c2015-08-28 18:02:08 -07002951 if oldLeader is None:
2952 main.log.error( "Leadership isn't consistent." )
2953 withdrawResult = main.FALSE
2954 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002955 for i in main.activeNodes:
acsmars9475b1c2015-08-28 18:02:08 -07002956 if oldLeader == main.nodes[ i ].ip_address:
2957 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002958 break
2959 else: # FOR/ELSE statement
2960 main.log.error( "Leader election, could not find current leader" )
2961 if oldLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002962 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002963 utilities.assert_equals(
2964 expect=main.TRUE,
2965 actual=withdrawResult,
2966 onpass="Node was withdrawn from election",
2967 onfail="Node was not withdrawn from election" )
2968
acsmars9475b1c2015-08-28 18:02:08 -07002969 main.step( "Check that a new node was elected leader" )
acsmars9475b1c2015-08-28 18:02:08 -07002970 failMessage = "Nodes have different leaders"
acsmars9475b1c2015-08-28 18:02:08 -07002971 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002972 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002973 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002974 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002975 if newLeaders[ 0 ][ 0 ] == 'none':
2976 main.log.error( "No leader was elected on at least 1 node" )
2977 if not expectNoLeader:
2978 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002979 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002980
acsmars9475b1c2015-08-28 18:02:08 -07002981 # Check that the new leader is not the older leader, which was withdrawn
2982 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002983 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08002984 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars9475b1c2015-08-28 18:02:08 -07002985 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002986 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002987 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07002988 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002989 onpass="Leadership election passed",
2990 onfail="Something went wrong with Leadership election" )
2991
Jon Halla440e872016-03-31 15:15:50 -07002992 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08002993 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars9475b1c2015-08-28 18:02:08 -07002994 correctCandidateResult = main.TRUE
acsmars71adceb2015-08-31 15:09:26 -07002995 if expectNoLeader:
2996 if newLeader == 'none':
2997 main.log.info( "No leader expected. None found. Pass" )
2998 correctCandidateResult = main.TRUE
2999 else:
3000 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3001 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003002 elif len( oldLeaders[0] ) >= 3:
3003 if newLeader == oldLeaders[ 0 ][ 2 ]:
3004 # correct leader was elected
3005 correctCandidateResult = main.TRUE
3006 else:
3007 correctCandidateResult = main.FALSE
3008 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3009 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003010 else:
3011 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003012 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003013 correctCandidateResult = main.FALSE
acsmars9475b1c2015-08-28 18:02:08 -07003014 utilities.assert_equals(
3015 expect=main.TRUE,
3016 actual=correctCandidateResult,
3017 onpass="Correct Candidate Elected",
3018 onfail="Incorrect Candidate Elected" )
3019
Jon Hall5cf14d52015-07-16 12:15:19 -07003020 main.step( "Run for election on old leader( just so everyone " +
3021 "is in the hat )" )
acsmars9475b1c2015-08-28 18:02:08 -07003022 if oldLeaderCLI is not None:
3023 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003024 else:
acsmars9475b1c2015-08-28 18:02:08 -07003025 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003026 runResult = main.FALSE
3027 utilities.assert_equals(
3028 expect=main.TRUE,
3029 actual=runResult,
3030 onpass="App re-ran for election",
3031 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003032
acsmars9475b1c2015-08-28 18:02:08 -07003033 main.step(
3034 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003035 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003036 # Get new leaders and candidates
3037 reRunLeaders = []
3038 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003039 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003040
acsmars9475b1c2015-08-28 18:02:08 -07003041 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07003042 if not reRunLeaders[0]:
3043 positionResult = main.FALSE
3044 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07003045 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3046 str( reRunLeaders[ 0 ] ) ) )
acsmars9475b1c2015-08-28 18:02:08 -07003047 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003048 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003049 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07003050 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003051 onpass="Old leader successfully re-ran for election",
3052 onfail="Something went wrong with Leadership election after " +
3053 "the old leader re-ran for election" )
3054
3055 def CASE16( self, main ):
3056 """
3057 Install Distributed Primitives app
3058 """
3059 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003060 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003061 assert main, "main not defined"
3062 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003063 assert main.CLIs, "main.CLIs not defined"
3064 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003065
3066 # Variables for the distributed primitives tests
3067 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003068 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003069 global onosSet
3070 global onosSetName
3071 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003072 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003073 onosSet = set([])
3074 onosSetName = "TestON-set"
3075
3076 description = "Install Primitives app"
3077 main.case( description )
3078 main.step( "Install Primitives app" )
3079 appName = "org.onosproject.distributedprimitives"
Jon Halla440e872016-03-31 15:15:50 -07003080 node = main.activeNodes[0]
3081 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003082 utilities.assert_equals( expect=main.TRUE,
3083 actual=appResults,
3084 onpass="Primitives app activated",
3085 onfail="Primitives app not activated" )
3086 time.sleep( 5 ) # To allow all nodes to activate
3087
3088 def CASE17( self, main ):
3089 """
3090 Check for basic functionality with distributed primitives
3091 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003092 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003093 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003094 assert main, "main not defined"
3095 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003096 assert main.CLIs, "main.CLIs not defined"
3097 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003098 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003099 assert onosSetName, "onosSetName not defined"
3100 # NOTE: assert fails if value is 0/None/Empty/False
3101 try:
3102 pCounterValue
3103 except NameError:
3104 main.log.error( "pCounterValue not defined, setting to 0" )
3105 pCounterValue = 0
3106 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003107 onosSet
3108 except NameError:
3109 main.log.error( "onosSet not defined, setting to empty Set" )
3110 onosSet = set([])
3111 # Variables for the distributed primitives tests. These are local only
3112 addValue = "a"
3113 addAllValue = "a b c d e f"
3114 retainValue = "c d e f"
3115
3116 description = "Check for basic functionality with distributed " +\
3117 "primitives"
3118 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003119 main.caseExplanation = "Test the methods of the distributed " +\
3120 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003121 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003122 # Partitioned counters
3123 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003124 pCounters = []
3125 threads = []
3126 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003127 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003128 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3129 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003130 args=[ pCounterName ] )
3131 pCounterValue += 1
3132 addedPValues.append( pCounterValue )
3133 threads.append( t )
3134 t.start()
3135
3136 for t in threads:
3137 t.join()
3138 pCounters.append( t.result )
3139 # Check that counter incremented numController times
3140 pCounterResults = True
3141 for i in addedPValues:
3142 tmpResult = i in pCounters
3143 pCounterResults = pCounterResults and tmpResult
3144 if not tmpResult:
3145 main.log.error( str( i ) + " is not in partitioned "
3146 "counter incremented results" )
3147 utilities.assert_equals( expect=True,
3148 actual=pCounterResults,
3149 onpass="Default counter incremented",
3150 onfail="Error incrementing default" +
3151 " counter" )
3152
Jon Halle1a3b752015-07-22 13:02:46 -07003153 main.step( "Get then Increment a default counter on each node" )
3154 pCounters = []
3155 threads = []
3156 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003157 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003158 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3159 name="counterGetAndAdd-" + str( i ),
3160 args=[ pCounterName ] )
3161 addedPValues.append( pCounterValue )
3162 pCounterValue += 1
3163 threads.append( t )
3164 t.start()
3165
3166 for t in threads:
3167 t.join()
3168 pCounters.append( t.result )
3169 # Check that counter incremented numController times
3170 pCounterResults = True
3171 for i in addedPValues:
3172 tmpResult = i in pCounters
3173 pCounterResults = pCounterResults and tmpResult
3174 if not tmpResult:
3175 main.log.error( str( i ) + " is not in partitioned "
3176 "counter incremented results" )
3177 utilities.assert_equals( expect=True,
3178 actual=pCounterResults,
3179 onpass="Default counter incremented",
3180 onfail="Error incrementing default" +
3181 " counter" )
3182
3183 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003184 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003185 utilities.assert_equals( expect=main.TRUE,
3186 actual=incrementCheck,
3187 onpass="Added counters are correct",
3188 onfail="Added counters are incorrect" )
3189
3190 main.step( "Add -8 to then get a default counter on each node" )
3191 pCounters = []
3192 threads = []
3193 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003194 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003195 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3196 name="counterIncrement-" + str( i ),
3197 args=[ pCounterName ],
3198 kwargs={ "delta": -8 } )
3199 pCounterValue += -8
3200 addedPValues.append( pCounterValue )
3201 threads.append( t )
3202 t.start()
3203
3204 for t in threads:
3205 t.join()
3206 pCounters.append( t.result )
3207 # Check that counter incremented numController times
3208 pCounterResults = True
3209 for i in addedPValues:
3210 tmpResult = i in pCounters
3211 pCounterResults = pCounterResults and tmpResult
3212 if not tmpResult:
3213 main.log.error( str( i ) + " is not in partitioned "
3214 "counter incremented results" )
3215 utilities.assert_equals( expect=True,
3216 actual=pCounterResults,
3217 onpass="Default counter incremented",
3218 onfail="Error incrementing default" +
3219 " counter" )
3220
3221 main.step( "Add 5 to then get a default counter on each node" )
3222 pCounters = []
3223 threads = []
3224 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003225 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003226 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3227 name="counterIncrement-" + str( i ),
3228 args=[ pCounterName ],
3229 kwargs={ "delta": 5 } )
3230 pCounterValue += 5
3231 addedPValues.append( pCounterValue )
3232 threads.append( t )
3233 t.start()
3234
3235 for t in threads:
3236 t.join()
3237 pCounters.append( t.result )
3238 # Check that counter incremented numController times
3239 pCounterResults = True
3240 for i in addedPValues:
3241 tmpResult = i in pCounters
3242 pCounterResults = pCounterResults and tmpResult
3243 if not tmpResult:
3244 main.log.error( str( i ) + " is not in partitioned "
3245 "counter incremented results" )
3246 utilities.assert_equals( expect=True,
3247 actual=pCounterResults,
3248 onpass="Default counter incremented",
3249 onfail="Error incrementing default" +
3250 " counter" )
3251
3252 main.step( "Get then add 5 to a default counter on each node" )
3253 pCounters = []
3254 threads = []
3255 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003256 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003257 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3258 name="counterIncrement-" + str( i ),
3259 args=[ pCounterName ],
3260 kwargs={ "delta": 5 } )
3261 addedPValues.append( pCounterValue )
3262 pCounterValue += 5
3263 threads.append( t )
3264 t.start()
3265
3266 for t in threads:
3267 t.join()
3268 pCounters.append( t.result )
3269 # Check that counter incremented numController times
3270 pCounterResults = True
3271 for i in addedPValues:
3272 tmpResult = i in pCounters
3273 pCounterResults = pCounterResults and tmpResult
3274 if not tmpResult:
3275 main.log.error( str( i ) + " is not in partitioned "
3276 "counter incremented results" )
3277 utilities.assert_equals( expect=True,
3278 actual=pCounterResults,
3279 onpass="Default counter incremented",
3280 onfail="Error incrementing default" +
3281 " counter" )
3282
3283 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003284 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003285 utilities.assert_equals( expect=main.TRUE,
3286 actual=incrementCheck,
3287 onpass="Added counters are correct",
3288 onfail="Added counters are incorrect" )
3289
Jon Hall5cf14d52015-07-16 12:15:19 -07003290 # DISTRIBUTED SETS
3291 main.step( "Distributed Set get" )
3292 size = len( onosSet )
3293 getResponses = []
3294 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003295 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003296 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003297 name="setTestGet-" + str( i ),
3298 args=[ onosSetName ] )
3299 threads.append( t )
3300 t.start()
3301 for t in threads:
3302 t.join()
3303 getResponses.append( t.result )
3304
3305 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003306 for i in range( len( main.activeNodes ) ):
3307 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003308 if isinstance( getResponses[ i ], list):
3309 current = set( getResponses[ i ] )
3310 if len( current ) == len( getResponses[ i ] ):
3311 # no repeats
3312 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003313 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003314 " has incorrect view" +
3315 " of set " + onosSetName + ":\n" +
3316 str( getResponses[ i ] ) )
3317 main.log.debug( "Expected: " + str( onosSet ) )
3318 main.log.debug( "Actual: " + str( current ) )
3319 getResults = main.FALSE
3320 else:
3321 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003322 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003323 " has repeat elements in" +
3324 " set " + onosSetName + ":\n" +
3325 str( getResponses[ i ] ) )
3326 getResults = main.FALSE
3327 elif getResponses[ i ] == main.ERROR:
3328 getResults = main.FALSE
3329 utilities.assert_equals( expect=main.TRUE,
3330 actual=getResults,
3331 onpass="Set elements are correct",
3332 onfail="Set elements are incorrect" )
3333
3334 main.step( "Distributed Set size" )
3335 sizeResponses = []
3336 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003337 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003338 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003339 name="setTestSize-" + str( i ),
3340 args=[ onosSetName ] )
3341 threads.append( t )
3342 t.start()
3343 for t in threads:
3344 t.join()
3345 sizeResponses.append( t.result )
3346
3347 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003348 for i in range( len( main.activeNodes ) ):
3349 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003350 if size != sizeResponses[ i ]:
3351 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003352 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003353 " expected a size of " + str( size ) +
3354 " for set " + onosSetName +
3355 " but got " + str( sizeResponses[ i ] ) )
3356 utilities.assert_equals( expect=main.TRUE,
3357 actual=sizeResults,
3358 onpass="Set sizes are correct",
3359 onfail="Set sizes are incorrect" )
3360
3361 main.step( "Distributed Set add()" )
3362 onosSet.add( addValue )
3363 addResponses = []
3364 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003365 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003366 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003367 name="setTestAdd-" + str( i ),
3368 args=[ onosSetName, addValue ] )
3369 threads.append( t )
3370 t.start()
3371 for t in threads:
3372 t.join()
3373 addResponses.append( t.result )
3374
3375 # main.TRUE = successfully changed the set
3376 # main.FALSE = action resulted in no change in set
3377 # main.ERROR - Some error in executing the function
3378 addResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003379 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003380 if addResponses[ i ] == main.TRUE:
3381 # All is well
3382 pass
3383 elif addResponses[ i ] == main.FALSE:
3384 # Already in set, probably fine
3385 pass
3386 elif addResponses[ i ] == main.ERROR:
3387 # Error in execution
3388 addResults = main.FALSE
3389 else:
3390 # unexpected result
3391 addResults = main.FALSE
3392 if addResults != main.TRUE:
3393 main.log.error( "Error executing set add" )
3394
3395 # Check if set is still correct
3396 size = len( onosSet )
3397 getResponses = []
3398 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003399 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003400 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003401 name="setTestGet-" + str( i ),
3402 args=[ onosSetName ] )
3403 threads.append( t )
3404 t.start()
3405 for t in threads:
3406 t.join()
3407 getResponses.append( t.result )
3408 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003409 for i in range( len( main.activeNodes ) ):
3410 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003411 if isinstance( getResponses[ i ], list):
3412 current = set( getResponses[ i ] )
3413 if len( current ) == len( getResponses[ i ] ):
3414 # no repeats
3415 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003416 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003417 " of set " + onosSetName + ":\n" +
3418 str( getResponses[ i ] ) )
3419 main.log.debug( "Expected: " + str( onosSet ) )
3420 main.log.debug( "Actual: " + str( current ) )
3421 getResults = main.FALSE
3422 else:
3423 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003424 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003425 " set " + onosSetName + ":\n" +
3426 str( getResponses[ i ] ) )
3427 getResults = main.FALSE
3428 elif getResponses[ i ] == main.ERROR:
3429 getResults = main.FALSE
3430 sizeResponses = []
3431 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003432 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003433 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003434 name="setTestSize-" + str( i ),
3435 args=[ onosSetName ] )
3436 threads.append( t )
3437 t.start()
3438 for t in threads:
3439 t.join()
3440 sizeResponses.append( t.result )
3441 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003442 for i in range( len( main.activeNodes ) ):
3443 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003444 if size != sizeResponses[ i ]:
3445 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003446 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003447 " expected a size of " + str( size ) +
3448 " for set " + onosSetName +
3449 " but got " + str( sizeResponses[ i ] ) )
3450 addResults = addResults and getResults and sizeResults
3451 utilities.assert_equals( expect=main.TRUE,
3452 actual=addResults,
3453 onpass="Set add correct",
3454 onfail="Set add was incorrect" )
3455
3456 main.step( "Distributed Set addAll()" )
3457 onosSet.update( addAllValue.split() )
3458 addResponses = []
3459 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003460 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003461 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003462 name="setTestAddAll-" + str( i ),
3463 args=[ onosSetName, addAllValue ] )
3464 threads.append( t )
3465 t.start()
3466 for t in threads:
3467 t.join()
3468 addResponses.append( t.result )
3469
3470 # main.TRUE = successfully changed the set
3471 # main.FALSE = action resulted in no change in set
3472 # main.ERROR - Some error in executing the function
3473 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003474 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003475 if addResponses[ i ] == main.TRUE:
3476 # All is well
3477 pass
3478 elif addResponses[ i ] == main.FALSE:
3479 # Already in set, probably fine
3480 pass
3481 elif addResponses[ i ] == main.ERROR:
3482 # Error in execution
3483 addAllResults = main.FALSE
3484 else:
3485 # unexpected result
3486 addAllResults = main.FALSE
3487 if addAllResults != main.TRUE:
3488 main.log.error( "Error executing set addAll" )
3489
3490 # Check if set is still correct
3491 size = len( onosSet )
3492 getResponses = []
3493 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003494 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003495 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003496 name="setTestGet-" + str( i ),
3497 args=[ onosSetName ] )
3498 threads.append( t )
3499 t.start()
3500 for t in threads:
3501 t.join()
3502 getResponses.append( t.result )
3503 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003504 for i in range( len( main.activeNodes ) ):
3505 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003506 if isinstance( getResponses[ i ], list):
3507 current = set( getResponses[ i ] )
3508 if len( current ) == len( getResponses[ i ] ):
3509 # no repeats
3510 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003511 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003512 " has incorrect view" +
3513 " of set " + onosSetName + ":\n" +
3514 str( getResponses[ i ] ) )
3515 main.log.debug( "Expected: " + str( onosSet ) )
3516 main.log.debug( "Actual: " + str( current ) )
3517 getResults = main.FALSE
3518 else:
3519 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003520 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003521 " has repeat elements in" +
3522 " set " + onosSetName + ":\n" +
3523 str( getResponses[ i ] ) )
3524 getResults = main.FALSE
3525 elif getResponses[ i ] == main.ERROR:
3526 getResults = main.FALSE
3527 sizeResponses = []
3528 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003529 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003530 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003531 name="setTestSize-" + str( i ),
3532 args=[ onosSetName ] )
3533 threads.append( t )
3534 t.start()
3535 for t in threads:
3536 t.join()
3537 sizeResponses.append( t.result )
3538 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003539 for i in range( len( main.activeNodes ) ):
3540 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003541 if size != sizeResponses[ i ]:
3542 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003543 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003544 " expected a size of " + str( size ) +
3545 " for set " + onosSetName +
3546 " but got " + str( sizeResponses[ i ] ) )
3547 addAllResults = addAllResults and getResults and sizeResults
3548 utilities.assert_equals( expect=main.TRUE,
3549 actual=addAllResults,
3550 onpass="Set addAll correct",
3551 onfail="Set addAll was incorrect" )
3552
3553 main.step( "Distributed Set contains()" )
3554 containsResponses = []
3555 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003556 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003557 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003558 name="setContains-" + str( i ),
3559 args=[ onosSetName ],
3560 kwargs={ "values": addValue } )
3561 threads.append( t )
3562 t.start()
3563 for t in threads:
3564 t.join()
3565 # NOTE: This is the tuple
3566 containsResponses.append( t.result )
3567
3568 containsResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003569 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003570 if containsResponses[ i ] == main.ERROR:
3571 containsResults = main.FALSE
3572 else:
3573 containsResults = containsResults and\
3574 containsResponses[ i ][ 1 ]
3575 utilities.assert_equals( expect=main.TRUE,
3576 actual=containsResults,
3577 onpass="Set contains is functional",
3578 onfail="Set contains failed" )
3579
3580 main.step( "Distributed Set containsAll()" )
3581 containsAllResponses = []
3582 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003583 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003584 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003585 name="setContainsAll-" + str( i ),
3586 args=[ onosSetName ],
3587 kwargs={ "values": addAllValue } )
3588 threads.append( t )
3589 t.start()
3590 for t in threads:
3591 t.join()
3592 # NOTE: This is the tuple
3593 containsAllResponses.append( t.result )
3594
3595 containsAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003596 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003597 if containsResponses[ i ] == main.ERROR:
3598 containsResults = main.FALSE
3599 else:
3600 containsResults = containsResults and\
3601 containsResponses[ i ][ 1 ]
3602 utilities.assert_equals( expect=main.TRUE,
3603 actual=containsAllResults,
3604 onpass="Set containsAll is functional",
3605 onfail="Set containsAll failed" )
3606
3607 main.step( "Distributed Set remove()" )
3608 onosSet.remove( addValue )
3609 removeResponses = []
3610 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003611 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003612 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003613 name="setTestRemove-" + str( i ),
3614 args=[ onosSetName, addValue ] )
3615 threads.append( t )
3616 t.start()
3617 for t in threads:
3618 t.join()
3619 removeResponses.append( t.result )
3620
3621 # main.TRUE = successfully changed the set
3622 # main.FALSE = action resulted in no change in set
3623 # main.ERROR - Some error in executing the function
3624 removeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003625 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003626 if removeResponses[ i ] == main.TRUE:
3627 # All is well
3628 pass
3629 elif removeResponses[ i ] == main.FALSE:
3630 # not in set, probably fine
3631 pass
3632 elif removeResponses[ i ] == main.ERROR:
3633 # Error in execution
3634 removeResults = main.FALSE
3635 else:
3636 # unexpected result
3637 removeResults = main.FALSE
3638 if removeResults != main.TRUE:
3639 main.log.error( "Error executing set remove" )
3640
3641 # Check if set is still correct
3642 size = len( onosSet )
3643 getResponses = []
3644 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003645 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003646 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003647 name="setTestGet-" + str( i ),
3648 args=[ onosSetName ] )
3649 threads.append( t )
3650 t.start()
3651 for t in threads:
3652 t.join()
3653 getResponses.append( t.result )
3654 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003655 for i in range( len( main.activeNodes ) ):
3656 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003657 if isinstance( getResponses[ i ], list):
3658 current = set( getResponses[ i ] )
3659 if len( current ) == len( getResponses[ i ] ):
3660 # no repeats
3661 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003662 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003663 " has incorrect view" +
3664 " of set " + onosSetName + ":\n" +
3665 str( getResponses[ i ] ) )
3666 main.log.debug( "Expected: " + str( onosSet ) )
3667 main.log.debug( "Actual: " + str( current ) )
3668 getResults = main.FALSE
3669 else:
3670 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003671 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003672 " has repeat elements in" +
3673 " set " + onosSetName + ":\n" +
3674 str( getResponses[ i ] ) )
3675 getResults = main.FALSE
3676 elif getResponses[ i ] == main.ERROR:
3677 getResults = main.FALSE
3678 sizeResponses = []
3679 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003680 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003681 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003682 name="setTestSize-" + str( i ),
3683 args=[ onosSetName ] )
3684 threads.append( t )
3685 t.start()
3686 for t in threads:
3687 t.join()
3688 sizeResponses.append( t.result )
3689 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003690 for i in range( len( main.activeNodes ) ):
3691 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003692 if size != sizeResponses[ i ]:
3693 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003694 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003695 " expected a size of " + str( size ) +
3696 " for set " + onosSetName +
3697 " but got " + str( sizeResponses[ i ] ) )
3698 removeResults = removeResults and getResults and sizeResults
3699 utilities.assert_equals( expect=main.TRUE,
3700 actual=removeResults,
3701 onpass="Set remove correct",
3702 onfail="Set remove was incorrect" )
3703
3704 main.step( "Distributed Set removeAll()" )
3705 onosSet.difference_update( addAllValue.split() )
3706 removeAllResponses = []
3707 threads = []
3708 try:
Jon Halla440e872016-03-31 15:15:50 -07003709 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003710 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003711 name="setTestRemoveAll-" + str( i ),
3712 args=[ onosSetName, addAllValue ] )
3713 threads.append( t )
3714 t.start()
3715 for t in threads:
3716 t.join()
3717 removeAllResponses.append( t.result )
3718 except Exception, e:
3719 main.log.exception(e)
3720
3721 # main.TRUE = successfully changed the set
3722 # main.FALSE = action resulted in no change in set
3723 # main.ERROR - Some error in executing the function
3724 removeAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003725 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003726 if removeAllResponses[ i ] == main.TRUE:
3727 # All is well
3728 pass
3729 elif removeAllResponses[ i ] == main.FALSE:
3730 # not in set, probably fine
3731 pass
3732 elif removeAllResponses[ i ] == main.ERROR:
3733 # Error in execution
3734 removeAllResults = main.FALSE
3735 else:
3736 # unexpected result
3737 removeAllResults = main.FALSE
3738 if removeAllResults != main.TRUE:
3739 main.log.error( "Error executing set removeAll" )
3740
3741 # Check if set is still correct
3742 size = len( onosSet )
3743 getResponses = []
3744 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003745 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003746 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003747 name="setTestGet-" + str( i ),
3748 args=[ onosSetName ] )
3749 threads.append( t )
3750 t.start()
3751 for t in threads:
3752 t.join()
3753 getResponses.append( t.result )
3754 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003755 for i in range( len( main.activeNodes ) ):
3756 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003757 if isinstance( getResponses[ i ], list):
3758 current = set( getResponses[ i ] )
3759 if len( current ) == len( getResponses[ i ] ):
3760 # no repeats
3761 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003762 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003763 " has incorrect view" +
3764 " of set " + onosSetName + ":\n" +
3765 str( getResponses[ i ] ) )
3766 main.log.debug( "Expected: " + str( onosSet ) )
3767 main.log.debug( "Actual: " + str( current ) )
3768 getResults = main.FALSE
3769 else:
3770 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003771 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003772 " has repeat elements in" +
3773 " set " + onosSetName + ":\n" +
3774 str( getResponses[ i ] ) )
3775 getResults = main.FALSE
3776 elif getResponses[ i ] == main.ERROR:
3777 getResults = main.FALSE
3778 sizeResponses = []
3779 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003780 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003781 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003782 name="setTestSize-" + str( i ),
3783 args=[ onosSetName ] )
3784 threads.append( t )
3785 t.start()
3786 for t in threads:
3787 t.join()
3788 sizeResponses.append( t.result )
3789 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003790 for i in range( len( main.activeNodes ) ):
3791 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003792 if size != sizeResponses[ i ]:
3793 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003794 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003795 " expected a size of " + str( size ) +
3796 " for set " + onosSetName +
3797 " but got " + str( sizeResponses[ i ] ) )
3798 removeAllResults = removeAllResults and getResults and sizeResults
3799 utilities.assert_equals( expect=main.TRUE,
3800 actual=removeAllResults,
3801 onpass="Set removeAll correct",
3802 onfail="Set removeAll was incorrect" )
3803
3804 main.step( "Distributed Set addAll()" )
3805 onosSet.update( addAllValue.split() )
3806 addResponses = []
3807 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003808 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003809 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003810 name="setTestAddAll-" + str( i ),
3811 args=[ onosSetName, addAllValue ] )
3812 threads.append( t )
3813 t.start()
3814 for t in threads:
3815 t.join()
3816 addResponses.append( t.result )
3817
3818 # main.TRUE = successfully changed the set
3819 # main.FALSE = action resulted in no change in set
3820 # main.ERROR - Some error in executing the function
3821 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003822 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003823 if addResponses[ i ] == main.TRUE:
3824 # All is well
3825 pass
3826 elif addResponses[ i ] == main.FALSE:
3827 # Already in set, probably fine
3828 pass
3829 elif addResponses[ i ] == main.ERROR:
3830 # Error in execution
3831 addAllResults = main.FALSE
3832 else:
3833 # unexpected result
3834 addAllResults = main.FALSE
3835 if addAllResults != main.TRUE:
3836 main.log.error( "Error executing set addAll" )
3837
3838 # Check if set is still correct
3839 size = len( onosSet )
3840 getResponses = []
3841 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003842 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003843 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003844 name="setTestGet-" + str( i ),
3845 args=[ onosSetName ] )
3846 threads.append( t )
3847 t.start()
3848 for t in threads:
3849 t.join()
3850 getResponses.append( t.result )
3851 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003852 for i in range( len( main.activeNodes ) ):
3853 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003854 if isinstance( getResponses[ i ], list):
3855 current = set( getResponses[ i ] )
3856 if len( current ) == len( getResponses[ i ] ):
3857 # no repeats
3858 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003859 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003860 " has incorrect view" +
3861 " of set " + onosSetName + ":\n" +
3862 str( getResponses[ i ] ) )
3863 main.log.debug( "Expected: " + str( onosSet ) )
3864 main.log.debug( "Actual: " + str( current ) )
3865 getResults = main.FALSE
3866 else:
3867 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003868 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003869 " has repeat elements in" +
3870 " set " + onosSetName + ":\n" +
3871 str( getResponses[ i ] ) )
3872 getResults = main.FALSE
3873 elif getResponses[ i ] == main.ERROR:
3874 getResults = main.FALSE
3875 sizeResponses = []
3876 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003877 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003878 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003879 name="setTestSize-" + str( i ),
3880 args=[ onosSetName ] )
3881 threads.append( t )
3882 t.start()
3883 for t in threads:
3884 t.join()
3885 sizeResponses.append( t.result )
3886 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003887 for i in range( len( main.activeNodes ) ):
3888 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003889 if size != sizeResponses[ i ]:
3890 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003891 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003892 " expected a size of " + str( size ) +
3893 " for set " + onosSetName +
3894 " but got " + str( sizeResponses[ i ] ) )
3895 addAllResults = addAllResults and getResults and sizeResults
3896 utilities.assert_equals( expect=main.TRUE,
3897 actual=addAllResults,
3898 onpass="Set addAll correct",
3899 onfail="Set addAll was incorrect" )
3900
3901 main.step( "Distributed Set clear()" )
3902 onosSet.clear()
3903 clearResponses = []
3904 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003905 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003906 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003907 name="setTestClear-" + str( i ),
3908 args=[ onosSetName, " "], # Values doesn't matter
3909 kwargs={ "clear": True } )
3910 threads.append( t )
3911 t.start()
3912 for t in threads:
3913 t.join()
3914 clearResponses.append( t.result )
3915
3916 # main.TRUE = successfully changed the set
3917 # main.FALSE = action resulted in no change in set
3918 # main.ERROR - Some error in executing the function
3919 clearResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003920 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003921 if clearResponses[ i ] == main.TRUE:
3922 # All is well
3923 pass
3924 elif clearResponses[ i ] == main.FALSE:
3925 # Nothing set, probably fine
3926 pass
3927 elif clearResponses[ i ] == main.ERROR:
3928 # Error in execution
3929 clearResults = main.FALSE
3930 else:
3931 # unexpected result
3932 clearResults = main.FALSE
3933 if clearResults != main.TRUE:
3934 main.log.error( "Error executing set clear" )
3935
3936 # Check if set is still correct
3937 size = len( onosSet )
3938 getResponses = []
3939 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003940 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003941 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003942 name="setTestGet-" + str( i ),
3943 args=[ onosSetName ] )
3944 threads.append( t )
3945 t.start()
3946 for t in threads:
3947 t.join()
3948 getResponses.append( t.result )
3949 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003950 for i in range( len( main.activeNodes ) ):
3951 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003952 if isinstance( getResponses[ i ], list):
3953 current = set( getResponses[ i ] )
3954 if len( current ) == len( getResponses[ i ] ):
3955 # no repeats
3956 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003957 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003958 " has incorrect view" +
3959 " of set " + onosSetName + ":\n" +
3960 str( getResponses[ i ] ) )
3961 main.log.debug( "Expected: " + str( onosSet ) )
3962 main.log.debug( "Actual: " + str( current ) )
3963 getResults = main.FALSE
3964 else:
3965 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003966 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003967 " has repeat elements in" +
3968 " set " + onosSetName + ":\n" +
3969 str( getResponses[ i ] ) )
3970 getResults = main.FALSE
3971 elif getResponses[ i ] == main.ERROR:
3972 getResults = main.FALSE
3973 sizeResponses = []
3974 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003975 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003976 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003977 name="setTestSize-" + str( i ),
3978 args=[ onosSetName ] )
3979 threads.append( t )
3980 t.start()
3981 for t in threads:
3982 t.join()
3983 sizeResponses.append( t.result )
3984 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003985 for i in range( len( main.activeNodes ) ):
3986 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003987 if size != sizeResponses[ i ]:
3988 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003989 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003990 " expected a size of " + str( size ) +
3991 " for set " + onosSetName +
3992 " but got " + str( sizeResponses[ i ] ) )
3993 clearResults = clearResults and getResults and sizeResults
3994 utilities.assert_equals( expect=main.TRUE,
3995 actual=clearResults,
3996 onpass="Set clear correct",
3997 onfail="Set clear was incorrect" )
3998
3999 main.step( "Distributed Set addAll()" )
4000 onosSet.update( addAllValue.split() )
4001 addResponses = []
4002 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004003 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004004 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004005 name="setTestAddAll-" + str( i ),
4006 args=[ onosSetName, addAllValue ] )
4007 threads.append( t )
4008 t.start()
4009 for t in threads:
4010 t.join()
4011 addResponses.append( t.result )
4012
4013 # main.TRUE = successfully changed the set
4014 # main.FALSE = action resulted in no change in set
4015 # main.ERROR - Some error in executing the function
4016 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004017 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004018 if addResponses[ i ] == main.TRUE:
4019 # All is well
4020 pass
4021 elif addResponses[ i ] == main.FALSE:
4022 # Already in set, probably fine
4023 pass
4024 elif addResponses[ i ] == main.ERROR:
4025 # Error in execution
4026 addAllResults = main.FALSE
4027 else:
4028 # unexpected result
4029 addAllResults = main.FALSE
4030 if addAllResults != main.TRUE:
4031 main.log.error( "Error executing set addAll" )
4032
4033 # Check if set is still correct
4034 size = len( onosSet )
4035 getResponses = []
4036 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004037 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004038 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004039 name="setTestGet-" + str( i ),
4040 args=[ onosSetName ] )
4041 threads.append( t )
4042 t.start()
4043 for t in threads:
4044 t.join()
4045 getResponses.append( t.result )
4046 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004047 for i in range( len( main.activeNodes ) ):
4048 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004049 if isinstance( getResponses[ i ], list):
4050 current = set( getResponses[ i ] )
4051 if len( current ) == len( getResponses[ i ] ):
4052 # no repeats
4053 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004054 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004055 " has incorrect view" +
4056 " of set " + onosSetName + ":\n" +
4057 str( getResponses[ i ] ) )
4058 main.log.debug( "Expected: " + str( onosSet ) )
4059 main.log.debug( "Actual: " + str( current ) )
4060 getResults = main.FALSE
4061 else:
4062 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004063 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004064 " has repeat elements in" +
4065 " set " + onosSetName + ":\n" +
4066 str( getResponses[ i ] ) )
4067 getResults = main.FALSE
4068 elif getResponses[ i ] == main.ERROR:
4069 getResults = main.FALSE
4070 sizeResponses = []
4071 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004072 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004073 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004074 name="setTestSize-" + str( i ),
4075 args=[ onosSetName ] )
4076 threads.append( t )
4077 t.start()
4078 for t in threads:
4079 t.join()
4080 sizeResponses.append( t.result )
4081 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004082 for i in range( len( main.activeNodes ) ):
4083 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004084 if size != sizeResponses[ i ]:
4085 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004086 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004087 " expected a size of " + str( size ) +
4088 " for set " + onosSetName +
4089 " but got " + str( sizeResponses[ i ] ) )
4090 addAllResults = addAllResults and getResults and sizeResults
4091 utilities.assert_equals( expect=main.TRUE,
4092 actual=addAllResults,
4093 onpass="Set addAll correct",
4094 onfail="Set addAll was incorrect" )
4095
4096 main.step( "Distributed Set retain()" )
4097 onosSet.intersection_update( retainValue.split() )
4098 retainResponses = []
4099 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004100 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004101 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004102 name="setTestRetain-" + str( i ),
4103 args=[ onosSetName, retainValue ],
4104 kwargs={ "retain": True } )
4105 threads.append( t )
4106 t.start()
4107 for t in threads:
4108 t.join()
4109 retainResponses.append( t.result )
4110
4111 # main.TRUE = successfully changed the set
4112 # main.FALSE = action resulted in no change in set
4113 # main.ERROR - Some error in executing the function
4114 retainResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004115 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004116 if retainResponses[ i ] == main.TRUE:
4117 # All is well
4118 pass
4119 elif retainResponses[ i ] == main.FALSE:
4120 # Already in set, probably fine
4121 pass
4122 elif retainResponses[ i ] == main.ERROR:
4123 # Error in execution
4124 retainResults = main.FALSE
4125 else:
4126 # unexpected result
4127 retainResults = main.FALSE
4128 if retainResults != main.TRUE:
4129 main.log.error( "Error executing set retain" )
4130
4131 # Check if set is still correct
4132 size = len( onosSet )
4133 getResponses = []
4134 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004135 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004136 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004137 name="setTestGet-" + str( i ),
4138 args=[ onosSetName ] )
4139 threads.append( t )
4140 t.start()
4141 for t in threads:
4142 t.join()
4143 getResponses.append( t.result )
4144 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004145 for i in range( len( main.activeNodes ) ):
4146 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004147 if isinstance( getResponses[ i ], list):
4148 current = set( getResponses[ i ] )
4149 if len( current ) == len( getResponses[ i ] ):
4150 # no repeats
4151 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004152 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004153 " has incorrect view" +
4154 " of set " + onosSetName + ":\n" +
4155 str( getResponses[ i ] ) )
4156 main.log.debug( "Expected: " + str( onosSet ) )
4157 main.log.debug( "Actual: " + str( current ) )
4158 getResults = main.FALSE
4159 else:
4160 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004161 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004162 " has repeat elements in" +
4163 " set " + onosSetName + ":\n" +
4164 str( getResponses[ i ] ) )
4165 getResults = main.FALSE
4166 elif getResponses[ i ] == main.ERROR:
4167 getResults = main.FALSE
4168 sizeResponses = []
4169 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004170 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004171 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004172 name="setTestSize-" + str( i ),
4173 args=[ onosSetName ] )
4174 threads.append( t )
4175 t.start()
4176 for t in threads:
4177 t.join()
4178 sizeResponses.append( t.result )
4179 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004180 for i in range( len( main.activeNodes ) ):
4181 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004182 if size != sizeResponses[ i ]:
4183 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004184 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004185 str( size ) + " for set " + onosSetName +
4186 " but got " + str( sizeResponses[ i ] ) )
4187 retainResults = retainResults and getResults and sizeResults
4188 utilities.assert_equals( expect=main.TRUE,
4189 actual=retainResults,
4190 onpass="Set retain correct",
4191 onfail="Set retain was incorrect" )
4192
Jon Hall2a5002c2015-08-21 16:49:11 -07004193 # Transactional maps
4194 main.step( "Partitioned Transactional maps put" )
4195 tMapValue = "Testing"
4196 numKeys = 100
4197 putResult = True
Jon Halla440e872016-03-31 15:15:50 -07004198 node = main.activeNodes[0]
4199 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004200 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004201 for i in putResponses:
4202 if putResponses[ i ][ 'value' ] != tMapValue:
4203 putResult = False
4204 else:
4205 putResult = False
4206 if not putResult:
4207 main.log.debug( "Put response values: " + str( putResponses ) )
4208 utilities.assert_equals( expect=True,
4209 actual=putResult,
4210 onpass="Partitioned Transactional Map put successful",
4211 onfail="Partitioned Transactional Map put values are incorrect" )
4212
4213 main.step( "Partitioned Transactional maps get" )
4214 getCheck = True
4215 for n in range( 1, numKeys + 1 ):
4216 getResponses = []
4217 threads = []
4218 valueCheck = True
Jon Halla440e872016-03-31 15:15:50 -07004219 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004220 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4221 name="TMap-get-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08004222 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004223 threads.append( t )
4224 t.start()
4225 for t in threads:
4226 t.join()
4227 getResponses.append( t.result )
4228 for node in getResponses:
4229 if node != tMapValue:
4230 valueCheck = False
4231 if not valueCheck:
4232 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4233 main.log.warn( getResponses )
4234 getCheck = getCheck and valueCheck
4235 utilities.assert_equals( expect=True,
4236 actual=getCheck,
4237 onpass="Partitioned Transactional Map get values were correct",
4238 onfail="Partitioned Transactional Map values incorrect" )