blob: 69f094d12addd47fe9256fb4a9c0915dd096b162 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAclusterRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080051 import time
Jon Halla440e872016-03-31 15:15:50 -070052 import json
Jon Hall5cf14d52015-07-16 12:15:19 -070053 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
54 "initialization" )
55 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070056 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070057 "installing ONOS, starting Mininet and ONOS" +\
58 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070059
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
71 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79 # These are for csv plotting in jenkins
80 global labels
81 global data
82 labels = []
83 data = []
84
85 # FIXME: just get controller port from params?
86 # TODO: do we really need all these?
87 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
88 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
89 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
90 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
91 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
92 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
93 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
94
Jon Halle1a3b752015-07-22 13:02:46 -070095 try:
Jon Hall53c5e662016-04-13 16:06:56 -070096 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070097 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070098 except Exception as e:
99 main.log.exception( e )
100 main.cleanup()
101 main.exit()
102
103 main.CLIs = []
104 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700105 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700106 for i in range( 1, main.numCtrls + 1 ):
107 try:
108 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
109 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
110 ipList.append( main.nodes[ -1 ].ip_address )
111 except AttributeError:
112 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700113
114 main.step( "Create cell file" )
115 cellAppString = main.params[ 'ENV' ][ 'appString' ]
116 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
117 main.Mininet1.ip_address,
118 cellAppString, ipList )
119 main.step( "Applying cell variable to environment" )
120 cellResult = main.ONOSbench.setCell( cellName )
121 verifyResult = main.ONOSbench.verifyCell()
122
123 # FIXME:this is short term fix
124 main.log.info( "Removing raft logs" )
125 main.ONOSbench.onosRemoveRaftLogs()
126
127 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700128 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700129 main.ONOSbench.onosUninstall( node.ip_address )
130
131 # Make sure ONOS is DEAD
132 main.log.info( "Killing any ONOS processes" )
133 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700134 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700135 killed = main.ONOSbench.onosKill( node.ip_address )
136 killResults = killResults and killed
137
138 cleanInstallResult = main.TRUE
139 gitPullResult = main.TRUE
140
141 main.step( "Starting Mininet" )
142 # scp topo file to mininet
143 # TODO: move to params?
144 topoName = "obelisk.py"
145 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700146 main.ONOSbench.scp( main.Mininet1,
147 filePath + topoName,
148 main.Mininet1.home,
149 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700150 mnResult = main.Mininet1.startNet( )
151 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
152 onpass="Mininet Started",
153 onfail="Error starting Mininet" )
154
155 main.step( "Git checkout and pull " + gitBranch )
156 if PULLCODE:
157 main.ONOSbench.gitCheckout( gitBranch )
158 gitPullResult = main.ONOSbench.gitPull()
159 # values of 1 or 3 are good
160 utilities.assert_lesser( expect=0, actual=gitPullResult,
161 onpass="Git pull successful",
162 onfail="Git pull failed" )
163 main.ONOSbench.getVersion( report=True )
164
165 main.step( "Using mvn clean install" )
166 cleanInstallResult = main.TRUE
167 if PULLCODE and gitPullResult == main.TRUE:
168 cleanInstallResult = main.ONOSbench.cleanInstall()
169 else:
170 main.log.warn( "Did not pull new code so skipping mvn " +
171 "clean install" )
172 utilities.assert_equals( expect=main.TRUE,
173 actual=cleanInstallResult,
174 onpass="MCI successful",
175 onfail="MCI failed" )
176 # GRAPHS
177 # NOTE: important params here:
178 # job = name of Jenkins job
179 # Plot Name = Plot-HA, only can be used if multiple plots
180 # index = The number of the graph under plot name
181 job = "HAclusterRestart"
182 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700183 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700184 graphs = '<ac:structured-macro ac:name="html">\n'
185 graphs += '<ac:plain-text-body><![CDATA[\n'
186 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800187 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700188 '&width=500&height=300"' +\
189 'noborder="0" width="500" height="300" scrolling="yes" ' +\
190 'seamless="seamless"></iframe>\n'
191 graphs += ']]></ac:plain-text-body>\n'
192 graphs += '</ac:structured-macro>\n'
193 main.log.wiki(graphs)
194
195 main.step( "Creating ONOS package" )
196 packageResult = main.ONOSbench.onosPackage()
197 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
198 onpass="ONOS package successful",
199 onfail="ONOS package failed" )
200
201 main.step( "Installing ONOS package" )
202 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700203 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700204 tmpResult = main.ONOSbench.onosInstall( options="-f",
205 node=node.ip_address )
206 onosInstallResult = onosInstallResult and tmpResult
207 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
208 onpass="ONOS install successful",
209 onfail="ONOS install failed" )
210
211 main.step( "Checking if ONOS is up yet" )
212 for i in range( 2 ):
213 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700214 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700215 started = main.ONOSbench.isup( node.ip_address )
216 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800217 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700218 onosIsupResult = onosIsupResult and started
219 if onosIsupResult == main.TRUE:
220 break
221 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
222 onpass="ONOS startup successful",
223 onfail="ONOS startup failed" )
224
225 main.log.step( "Starting ONOS CLI sessions" )
226 cliResults = main.TRUE
227 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700228 for i in range( main.numCtrls ):
229 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700231 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700232 threads.append( t )
233 t.start()
234
235 for t in threads:
236 t.join()
237 cliResults = cliResults and t.result
238 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
239 onpass="ONOS cli startup successful",
240 onfail="ONOS cli startup failed" )
241
Jon Halla440e872016-03-31 15:15:50 -0700242 # Create a list of active nodes for use when some nodes are stopped
243 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
244
Jon Hall5cf14d52015-07-16 12:15:19 -0700245 if main.params[ 'tcpdump' ].lower() == "true":
246 main.step( "Start Packet Capture MN" )
247 main.Mininet2.startTcpdump(
248 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
249 + "-MN.pcap",
250 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
251 port=main.params[ 'MNtcpdump' ][ 'port' ] )
252
Jon Halla440e872016-03-31 15:15:50 -0700253 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700254 nodeResults = utilities.retry( main.HA.nodesCheck,
255 False,
256 args=[main.activeNodes],
257 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700258
Jon Hall41d39f12016-04-11 22:54:35 -0700259 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700260 onpass="Nodes check successful",
261 onfail="Nodes check NOT successful" )
262
263 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700264 for i in main.activeNodes:
265 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700266 main.log.debug( "{} components not ACTIVE: \n{}".format(
267 cli.name,
268 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700269 main.log.error( "Failed to start ONOS, stopping test" )
270 main.cleanup()
271 main.exit()
272
Jon Hall172b7ba2016-04-07 18:12:20 -0700273 main.step( "Activate apps defined in the params file" )
274 # get data from the params
275 apps = main.params.get( 'apps' )
276 if apps:
277 apps = apps.split(',')
278 main.log.warn( apps )
279 activateResult = True
280 for app in apps:
281 main.CLIs[ 0 ].app( app, "Activate" )
282 # TODO: check this worked
283 time.sleep( 10 ) # wait for apps to activate
284 for app in apps:
285 state = main.CLIs[ 0 ].appStatus( app )
286 if state == "ACTIVE":
287 activateResult = activeResult and True
288 else:
289 main.log.error( "{} is in {} state".format( app, state ) )
290 activeResult = False
291 utilities.assert_equals( expect=True,
292 actual=activateResult,
293 onpass="Successfully activated apps",
294 onfail="Failed to activate apps" )
295 else:
296 main.log.warn( "No apps were specified to be loaded after startup" )
297
298 main.step( "Set ONOS configurations" )
299 config = main.params.get( 'ONOS_Configuration' )
300 if config:
301 main.log.debug( config )
302 checkResult = main.TRUE
303 for component in config:
304 for setting in config[component]:
305 value = config[component][setting]
306 check = main.CLIs[ 0 ].setCfg( component, setting, value )
307 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
308 checkResult = check and checkResult
309 utilities.assert_equals( expect=main.TRUE,
310 actual=checkResult,
311 onpass="Successfully set config",
312 onfail="Failed to set config" )
313 else:
314 main.log.warn( "No configurations were specified to be changed after startup" )
315
Jon Hall9d2dcad2016-04-08 10:15:20 -0700316 main.step( "App Ids check" )
317 appCheck = main.TRUE
318 threads = []
319 for i in main.activeNodes:
320 t = main.Thread( target=main.CLIs[i].appToIDCheck,
321 name="appToIDCheck-" + str( i ),
322 args=[] )
323 threads.append( t )
324 t.start()
325
326 for t in threads:
327 t.join()
328 appCheck = appCheck and t.result
329 if appCheck != main.TRUE:
330 node = main.activeNodes[0]
331 main.log.warn( main.CLIs[node].apps() )
332 main.log.warn( main.CLIs[node].appIDs() )
333 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
334 onpass="App Ids seem to be correct",
335 onfail="Something is wrong with app Ids" )
336
Jon Hall5cf14d52015-07-16 12:15:19 -0700337 def CASE2( self, main ):
338 """
339 Assign devices to controllers
340 """
341 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700342 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700343 assert main, "main not defined"
344 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700345 assert main.CLIs, "main.CLIs not defined"
346 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700347 assert ONOS1Port, "ONOS1Port not defined"
348 assert ONOS2Port, "ONOS2Port not defined"
349 assert ONOS3Port, "ONOS3Port not defined"
350 assert ONOS4Port, "ONOS4Port not defined"
351 assert ONOS5Port, "ONOS5Port not defined"
352 assert ONOS6Port, "ONOS6Port not defined"
353 assert ONOS7Port, "ONOS7Port not defined"
354
355 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700356 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700357 "and check that an ONOS node becomes the " +\
358 "master of the device."
359 main.step( "Assign switches to controllers" )
360
361 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700362 for i in range( main.numCtrls ):
363 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700364 swList = []
365 for i in range( 1, 29 ):
366 swList.append( "s" + str( i ) )
367 main.Mininet1.assignSwController( sw=swList, ip=ipList )
368
369 mastershipCheck = main.TRUE
370 for i in range( 1, 29 ):
371 response = main.Mininet1.getSwController( "s" + str( i ) )
372 try:
373 main.log.info( str( response ) )
374 except Exception:
375 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700376 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700377 if re.search( "tcp:" + node.ip_address, response ):
378 mastershipCheck = mastershipCheck and main.TRUE
379 else:
380 main.log.error( "Error, node " + node.ip_address + " is " +
381 "not in the list of controllers s" +
382 str( i ) + " is connecting to." )
383 mastershipCheck = main.FALSE
384 utilities.assert_equals(
385 expect=main.TRUE,
386 actual=mastershipCheck,
387 onpass="Switch mastership assigned correctly",
388 onfail="Switches not assigned correctly to controllers" )
389
390 def CASE21( self, main ):
391 """
392 Assign mastership to controllers
393 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700394 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700395 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700396 assert main, "main not defined"
397 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700398 assert main.CLIs, "main.CLIs not defined"
399 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700400 assert ONOS1Port, "ONOS1Port not defined"
401 assert ONOS2Port, "ONOS2Port not defined"
402 assert ONOS3Port, "ONOS3Port not defined"
403 assert ONOS4Port, "ONOS4Port not defined"
404 assert ONOS5Port, "ONOS5Port not defined"
405 assert ONOS6Port, "ONOS6Port not defined"
406 assert ONOS7Port, "ONOS7Port not defined"
407
408 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700409 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700410 "device. Then manually assign" +\
411 " mastership to specific ONOS nodes using" +\
412 " 'device-role'"
413 main.step( "Assign mastership of switches to specific controllers" )
414 # Manually assign mastership to the controller we want
415 roleCall = main.TRUE
416
417 ipList = [ ]
418 deviceList = []
Jon Halla440e872016-03-31 15:15:50 -0700419 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700420 try:
421 # Assign mastership to specific controllers. This assignment was
422 # determined for a 7 node cluser, but will work with any sized
423 # cluster
424 for i in range( 1, 29 ): # switches 1 through 28
425 # set up correct variables:
426 if i == 1:
427 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700428 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700429 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700430 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700431 c = 1 % main.numCtrls
432 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700433 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700434 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700435 c = 1 % main.numCtrls
436 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700437 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700438 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700439 c = 3 % main.numCtrls
440 ip = main.nodes[ c ].ip_address # ONOS4
Jon Halla440e872016-03-31 15:15:50 -0700441 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700442 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700443 c = 2 % main.numCtrls
444 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700445 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700446 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700447 c = 2 % main.numCtrls
448 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700449 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700450 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700451 c = 5 % main.numCtrls
452 ip = main.nodes[ c ].ip_address # ONOS6
Jon Halla440e872016-03-31 15:15:50 -0700453 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700454 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700455 c = 4 % main.numCtrls
456 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700457 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700458 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700460 c = 6 % main.numCtrls
461 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700462 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700463 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700464 elif i == 28:
465 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700466 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700467 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700468 else:
469 main.log.error( "You didn't write an else statement for " +
470 "switch s" + str( i ) )
471 roleCall = main.FALSE
472 # Assign switch
473 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
474 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700475 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700476 ipList.append( ip )
477 deviceList.append( deviceId )
478 except ( AttributeError, AssertionError ):
479 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700480 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700481 utilities.assert_equals(
482 expect=main.TRUE,
483 actual=roleCall,
484 onpass="Re-assigned switch mastership to designated controller",
485 onfail="Something wrong with deviceRole calls" )
486
487 main.step( "Check mastership was correctly assigned" )
488 roleCheck = main.TRUE
489 # NOTE: This is due to the fact that device mastership change is not
490 # atomic and is actually a multi step process
491 time.sleep( 5 )
492 for i in range( len( ipList ) ):
493 ip = ipList[i]
494 deviceId = deviceList[i]
495 # Check assignment
Jon Halla440e872016-03-31 15:15:50 -0700496 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700497 if ip in master:
498 roleCheck = roleCheck and main.TRUE
499 else:
500 roleCheck = roleCheck and main.FALSE
501 main.log.error( "Error, controller " + ip + " is not" +
502 " master " + "of device " +
503 str( deviceId ) + ". Master is " +
504 repr( master ) + "." )
505 utilities.assert_equals(
506 expect=main.TRUE,
507 actual=roleCheck,
508 onpass="Switches were successfully reassigned to designated " +
509 "controller",
510 onfail="Switches were not successfully reassigned" )
511
512 def CASE3( self, main ):
513 """
514 Assign intents
515 """
516 import time
517 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700518 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700519 assert main, "main not defined"
520 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700521 assert main.CLIs, "main.CLIs not defined"
522 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700523 try:
524 labels
525 except NameError:
526 main.log.error( "labels not defined, setting to []" )
527 labels = []
528 try:
529 data
530 except NameError:
531 main.log.error( "data not defined, setting to []" )
532 data = []
533 # NOTE: we must reinstall intents until we have a persistant intent
534 # datastore!
535 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700536 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700537 "assign predetermined host-to-host intents." +\
538 " After installation, check that the intent" +\
539 " is distributed to all nodes and the state" +\
540 " is INSTALLED"
541
542 # install onos-app-fwd
543 main.step( "Install reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700544 onosCli = main.CLIs[ main.activeNodes[0] ]
545 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700546 utilities.assert_equals( expect=main.TRUE, actual=installResults,
547 onpass="Install fwd successful",
548 onfail="Install fwd failed" )
549
550 main.step( "Check app ids" )
551 appCheck = main.TRUE
552 threads = []
Jon Halla440e872016-03-31 15:15:50 -0700553 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700554 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700555 name="appToIDCheck-" + str( i ),
556 args=[] )
557 threads.append( t )
558 t.start()
559
560 for t in threads:
561 t.join()
562 appCheck = appCheck and t.result
563 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700564 main.log.warn( onosCli.apps() )
565 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700566 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
567 onpass="App Ids seem to be correct",
568 onfail="Something is wrong with app Ids" )
569
570 main.step( "Discovering Hosts( Via pingall for now )" )
571 # FIXME: Once we have a host discovery mechanism, use that instead
572 # REACTIVE FWD test
573 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700574 passMsg = "Reactive Pingall test passed"
575 time1 = time.time()
576 pingResult = main.Mininet1.pingall()
577 time2 = time.time()
578 if not pingResult:
579 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700580 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700581 passMsg += " on the second try"
582 utilities.assert_equals(
583 expect=main.TRUE,
584 actual=pingResult,
585 onpass= passMsg,
586 onfail="Reactive Pingall failed, " +
587 "one or more ping pairs failed" )
588 main.log.info( "Time for pingall: %2f seconds" %
589 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700590 # timeout for fwd flows
591 time.sleep( 11 )
592 # uninstall onos-app-fwd
593 main.step( "Uninstall reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700594 node = main.activeNodes[0]
595 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700596 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
597 onpass="Uninstall fwd successful",
598 onfail="Uninstall fwd failed" )
599
600 main.step( "Check app ids" )
601 threads = []
602 appCheck2 = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700603 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700604 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700605 name="appToIDCheck-" + str( i ),
606 args=[] )
607 threads.append( t )
608 t.start()
609
610 for t in threads:
611 t.join()
612 appCheck2 = appCheck2 and t.result
613 if appCheck2 != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700614 node = main.activeNodes[0]
615 main.log.warn( main.CLIs[node].apps() )
616 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700617 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
618 onpass="App Ids seem to be correct",
619 onfail="Something is wrong with app Ids" )
620
621 main.step( "Add host intents via cli" )
622 intentIds = []
Jon Hall6e709752016-02-01 13:38:46 -0800623 # TODO: move the host numbers to params
624 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700625 intentAddResult = True
626 hostResult = main.TRUE
627 for i in range( 8, 18 ):
628 main.log.info( "Adding host intent between h" + str( i ) +
629 " and h" + str( i + 10 ) )
630 host1 = "00:00:00:00:00:" + \
631 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
632 host2 = "00:00:00:00:00:" + \
633 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
634 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700635 host1Dict = onosCli.getHost( host1 )
636 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700637 host1Id = None
638 host2Id = None
639 if host1Dict and host2Dict:
640 host1Id = host1Dict.get( 'id', None )
641 host2Id = host2Dict.get( 'id', None )
642 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700643 nodeNum = ( i % len( main.activeNodes ) )
644 node = main.activeNodes[nodeNum]
645 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700646 if tmpId:
647 main.log.info( "Added intent with id: " + tmpId )
648 intentIds.append( tmpId )
649 else:
650 main.log.error( "addHostIntent returned: " +
651 repr( tmpId ) )
652 else:
653 main.log.error( "Error, getHost() failed for h" + str( i ) +
654 " and/or h" + str( i + 10 ) )
Jon Halla440e872016-03-31 15:15:50 -0700655 node = main.activeNodes[0]
656 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700657 main.log.warn( "Hosts output: " )
658 try:
659 main.log.warn( json.dumps( json.loads( hosts ),
660 sort_keys=True,
661 indent=4,
662 separators=( ',', ': ' ) ) )
663 except ( ValueError, TypeError ):
664 main.log.warn( repr( hosts ) )
665 hostResult = main.FALSE
666 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
667 onpass="Found a host id for each host",
668 onfail="Error looking up host ids" )
669
670 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700671 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700672 main.log.info( "Submitted intents: " + str( intentIds ) )
673 main.log.info( "Intents in ONOS: " + str( onosIds ) )
674 for intent in intentIds:
675 if intent in onosIds:
676 pass # intent submitted is in onos
677 else:
678 intentAddResult = False
679 if intentAddResult:
680 intentStop = time.time()
681 else:
682 intentStop = None
683 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700684 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700685 intentStates = []
686 installedCheck = True
687 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
688 count = 0
689 try:
690 for intent in json.loads( intents ):
691 state = intent.get( 'state', None )
692 if "INSTALLED" not in state:
693 installedCheck = False
694 intentId = intent.get( 'id', None )
695 intentStates.append( ( intentId, state ) )
696 except ( ValueError, TypeError ):
697 main.log.exception( "Error parsing intents" )
698 # add submitted intents not in the store
699 tmplist = [ i for i, s in intentStates ]
700 missingIntents = False
701 for i in intentIds:
702 if i not in tmplist:
703 intentStates.append( ( i, " - " ) )
704 missingIntents = True
705 intentStates.sort()
706 for i, s in intentStates:
707 count += 1
708 main.log.info( "%-6s%-15s%-15s" %
709 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700710 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700711 try:
712 missing = False
713 if leaders:
714 parsedLeaders = json.loads( leaders )
715 main.log.warn( json.dumps( parsedLeaders,
716 sort_keys=True,
717 indent=4,
718 separators=( ',', ': ' ) ) )
719 # check for all intent partitions
720 topics = []
721 for i in range( 14 ):
722 topics.append( "intent-partition-" + str( i ) )
723 main.log.debug( topics )
724 ONOStopics = [ j['topic'] for j in parsedLeaders ]
725 for topic in topics:
726 if topic not in ONOStopics:
727 main.log.error( "Error: " + topic +
728 " not in leaders" )
729 missing = True
730 else:
731 main.log.error( "leaders() returned None" )
732 except ( ValueError, TypeError ):
733 main.log.exception( "Error parsing leaders" )
734 main.log.error( repr( leaders ) )
735 # Check all nodes
736 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700737 for i in main.activeNodes:
738 response = main.CLIs[i].leaders( jsonFormat=False)
739 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700740 str( response ) )
741
Jon Halla440e872016-03-31 15:15:50 -0700742 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700743 try:
744 if partitions :
745 parsedPartitions = json.loads( partitions )
746 main.log.warn( json.dumps( parsedPartitions,
747 sort_keys=True,
748 indent=4,
749 separators=( ',', ': ' ) ) )
750 # TODO check for a leader in all paritions
751 # TODO check for consistency among nodes
752 else:
753 main.log.error( "partitions() returned None" )
754 except ( ValueError, TypeError ):
755 main.log.exception( "Error parsing partitions" )
756 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700757 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700758 try:
759 if pendingMap :
760 parsedPending = json.loads( pendingMap )
761 main.log.warn( json.dumps( parsedPending,
762 sort_keys=True,
763 indent=4,
764 separators=( ',', ': ' ) ) )
765 # TODO check something here?
766 else:
767 main.log.error( "pendingMap() returned None" )
768 except ( ValueError, TypeError ):
769 main.log.exception( "Error parsing pending map" )
770 main.log.error( repr( pendingMap ) )
771
772 intentAddResult = bool( intentAddResult and not missingIntents and
773 installedCheck )
774 if not intentAddResult:
775 main.log.error( "Error in pushing host intents to ONOS" )
776
777 main.step( "Intent Anti-Entropy dispersion" )
Jon Halla440e872016-03-31 15:15:50 -0700778 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700779 correct = True
780 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700781 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700782 onosIds = []
Jon Halla440e872016-03-31 15:15:50 -0700783 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700784 onosIds.append( ids )
Jon Halla440e872016-03-31 15:15:50 -0700785 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700786 str( sorted( onosIds ) ) )
787 if sorted( ids ) != sorted( intentIds ):
788 main.log.warn( "Set of intent IDs doesn't match" )
789 correct = False
790 break
791 else:
Jon Halla440e872016-03-31 15:15:50 -0700792 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700793 for intent in intents:
794 if intent[ 'state' ] != "INSTALLED":
795 main.log.warn( "Intent " + intent[ 'id' ] +
796 " is " + intent[ 'state' ] )
797 correct = False
798 break
799 if correct:
800 break
801 else:
802 time.sleep(1)
803 if not intentStop:
804 intentStop = time.time()
805 global gossipTime
806 gossipTime = intentStop - intentStart
807 main.log.info( "It took about " + str( gossipTime ) +
808 " seconds for all intents to appear in each node" )
809 append = False
810 title = "Gossip Intents"
811 count = 1
812 while append is False:
813 curTitle = title + str( count )
814 if curTitle not in labels:
815 labels.append( curTitle )
816 data.append( str( gossipTime ) )
817 append = True
818 else:
819 count += 1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700820 gossipPeriod = int( main.params['timers']['gossip'] )
Jon Halla440e872016-03-31 15:15:50 -0700821 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700822 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700823 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700824 onpass="ECM anti-entropy for intents worked within " +
825 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700826 onfail="Intent ECM anti-entropy took too long. " +
827 "Expected time:{}, Actual time:{}".format( maxGossipTime,
828 gossipTime ) )
829 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700830 intentAddResult = True
831
832 if not intentAddResult or "key" in pendingMap:
833 import time
834 installedCheck = True
835 main.log.info( "Sleeping 60 seconds to see if intents are found" )
836 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700837 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700838 main.log.info( "Submitted intents: " + str( intentIds ) )
839 main.log.info( "Intents in ONOS: " + str( onosIds ) )
840 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700841 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700842 intentStates = []
843 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
844 count = 0
845 try:
846 for intent in json.loads( intents ):
847 # Iter through intents of a node
848 state = intent.get( 'state', None )
849 if "INSTALLED" not in state:
850 installedCheck = False
851 intentId = intent.get( 'id', None )
852 intentStates.append( ( intentId, state ) )
853 except ( ValueError, TypeError ):
854 main.log.exception( "Error parsing intents" )
855 # add submitted intents not in the store
856 tmplist = [ i for i, s in intentStates ]
857 for i in intentIds:
858 if i not in tmplist:
859 intentStates.append( ( i, " - " ) )
860 intentStates.sort()
861 for i, s in intentStates:
862 count += 1
863 main.log.info( "%-6s%-15s%-15s" %
864 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700865 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700866 try:
867 missing = False
868 if leaders:
869 parsedLeaders = json.loads( leaders )
870 main.log.warn( json.dumps( parsedLeaders,
871 sort_keys=True,
872 indent=4,
873 separators=( ',', ': ' ) ) )
874 # check for all intent partitions
875 # check for election
876 topics = []
877 for i in range( 14 ):
878 topics.append( "intent-partition-" + str( i ) )
879 # FIXME: this should only be after we start the app
880 topics.append( "org.onosproject.election" )
881 main.log.debug( topics )
882 ONOStopics = [ j['topic'] for j in parsedLeaders ]
883 for topic in topics:
884 if topic not in ONOStopics:
885 main.log.error( "Error: " + topic +
886 " not in leaders" )
887 missing = True
888 else:
889 main.log.error( "leaders() returned None" )
890 except ( ValueError, TypeError ):
891 main.log.exception( "Error parsing leaders" )
892 main.log.error( repr( leaders ) )
893 # Check all nodes
894 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700895 for i in main.activeNodes:
896 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700897 response = node.leaders( jsonFormat=False)
898 main.log.warn( str( node.name ) + " leaders output: \n" +
899 str( response ) )
900
Jon Halla440e872016-03-31 15:15:50 -0700901 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700902 try:
903 if partitions :
904 parsedPartitions = json.loads( partitions )
905 main.log.warn( json.dumps( parsedPartitions,
906 sort_keys=True,
907 indent=4,
908 separators=( ',', ': ' ) ) )
909 # TODO check for a leader in all paritions
910 # TODO check for consistency among nodes
911 else:
912 main.log.error( "partitions() returned None" )
913 except ( ValueError, TypeError ):
914 main.log.exception( "Error parsing partitions" )
915 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700916 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700917 try:
918 if pendingMap :
919 parsedPending = json.loads( pendingMap )
920 main.log.warn( json.dumps( parsedPending,
921 sort_keys=True,
922 indent=4,
923 separators=( ',', ': ' ) ) )
924 # TODO check something here?
925 else:
926 main.log.error( "pendingMap() returned None" )
927 except ( ValueError, TypeError ):
928 main.log.exception( "Error parsing pending map" )
929 main.log.error( repr( pendingMap ) )
930
931 def CASE4( self, main ):
932 """
933 Ping across added host intents
934 """
935 import json
936 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700937 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700938 assert main, "main not defined"
939 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700940 assert main.CLIs, "main.CLIs not defined"
941 assert main.nodes, "main.nodes not defined"
Jon Hall6e709752016-02-01 13:38:46 -0800942 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700943 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700944 "functionality and check the state of " +\
945 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700946
Jon Hall41d39f12016-04-11 22:54:35 -0700947 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700948 main.step( "Check Intent state" )
949 installedCheck = False
950 loopCount = 0
951 while not installedCheck and loopCount < 40:
952 installedCheck = True
953 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700954 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700955 intentStates = []
956 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700957 count = 0
Jon Hall5cf14d52015-07-16 12:15:19 -0700958 # Iter through intents of a node
959 try:
960 for intent in json.loads( intents ):
961 state = intent.get( 'state', None )
962 if "INSTALLED" not in state:
963 installedCheck = False
964 intentId = intent.get( 'id', None )
965 intentStates.append( ( intentId, state ) )
966 except ( ValueError, TypeError ):
967 main.log.exception( "Error parsing intents." )
968 # Print states
969 intentStates.sort()
970 for i, s in intentStates:
971 count += 1
972 main.log.info( "%-6s%-15s%-15s" %
973 ( str( count ), str( i ), str( s ) ) )
974 if not installedCheck:
975 time.sleep( 1 )
976 loopCount += 1
977 utilities.assert_equals( expect=True, actual=installedCheck,
978 onpass="Intents are all INSTALLED",
979 onfail="Intents are not all in " +
980 "INSTALLED state" )
981
Jon Hall9d2dcad2016-04-08 10:15:20 -0700982 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700983 PingResult = main.TRUE
984 for i in range( 8, 18 ):
985 ping = main.Mininet1.pingHost( src="h" + str( i ),
986 target="h" + str( i + 10 ) )
987 PingResult = PingResult and ping
988 if ping == main.FALSE:
989 main.log.warn( "Ping failed between h" + str( i ) +
990 " and h" + str( i + 10 ) )
991 elif ping == main.TRUE:
992 main.log.info( "Ping test passed!" )
993 # Don't set PingResult or you'd override failures
994 if PingResult == main.FALSE:
995 main.log.error(
996 "Intents have not been installed correctly, pings failed." )
997 # TODO: pretty print
998 main.log.warn( "ONOS1 intents: " )
999 try:
1000 tmpIntents = onosCli.intents()
1001 main.log.warn( json.dumps( json.loads( tmpIntents ),
1002 sort_keys=True,
1003 indent=4,
1004 separators=( ',', ': ' ) ) )
1005 except ( ValueError, TypeError ):
1006 main.log.warn( repr( tmpIntents ) )
1007 utilities.assert_equals(
1008 expect=main.TRUE,
1009 actual=PingResult,
1010 onpass="Intents have been installed correctly and pings work",
1011 onfail="Intents have not been installed correctly, pings failed." )
1012
Jon Hall5cf14d52015-07-16 12:15:19 -07001013 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -07001014 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001015 topicCheck = main.TRUE
1016 try:
1017 if leaders:
1018 parsedLeaders = json.loads( leaders )
1019 main.log.warn( json.dumps( parsedLeaders,
1020 sort_keys=True,
1021 indent=4,
1022 separators=( ',', ': ' ) ) )
1023 # check for all intent partitions
1024 # check for election
1025 # TODO: Look at Devices as topics now that it uses this system
1026 topics = []
1027 for i in range( 14 ):
1028 topics.append( "intent-partition-" + str( i ) )
1029 # FIXME: this should only be after we start the app
1030 # FIXME: topics.append( "org.onosproject.election" )
1031 # Print leaders output
1032 main.log.debug( topics )
1033 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1034 for topic in topics:
1035 if topic not in ONOStopics:
1036 main.log.error( "Error: " + topic +
1037 " not in leaders" )
1038 topicCheck = main.FALSE
1039 else:
1040 main.log.error( "leaders() returned None" )
1041 topicCheck = main.FALSE
1042 except ( ValueError, TypeError ):
1043 topicCheck = main.FALSE
1044 main.log.exception( "Error parsing leaders" )
1045 main.log.error( repr( leaders ) )
1046 # TODO: Check for a leader of these topics
1047 # Check all nodes
1048 if topicCheck:
Jon Halla440e872016-03-31 15:15:50 -07001049 for i in main.activeNodes:
1050 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001051 response = node.leaders( jsonFormat=False)
1052 main.log.warn( str( node.name ) + " leaders output: \n" +
1053 str( response ) )
1054
1055 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1056 onpass="intent Partitions is in leaders",
1057 onfail="Some topics were lost " )
1058 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -07001059 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001060 try:
1061 if partitions :
1062 parsedPartitions = json.loads( partitions )
1063 main.log.warn( json.dumps( parsedPartitions,
1064 sort_keys=True,
1065 indent=4,
1066 separators=( ',', ': ' ) ) )
1067 # TODO check for a leader in all paritions
1068 # TODO check for consistency among nodes
1069 else:
1070 main.log.error( "partitions() returned None" )
1071 except ( ValueError, TypeError ):
1072 main.log.exception( "Error parsing partitions" )
1073 main.log.error( repr( partitions ) )
1074 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001075 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001076 try:
1077 if pendingMap :
1078 parsedPending = json.loads( pendingMap )
1079 main.log.warn( json.dumps( parsedPending,
1080 sort_keys=True,
1081 indent=4,
1082 separators=( ',', ': ' ) ) )
1083 # TODO check something here?
1084 else:
1085 main.log.error( "pendingMap() returned None" )
1086 except ( ValueError, TypeError ):
1087 main.log.exception( "Error parsing pending map" )
1088 main.log.error( repr( pendingMap ) )
1089
1090 if not installedCheck:
1091 main.log.info( "Waiting 60 seconds to see if the state of " +
1092 "intents change" )
1093 time.sleep( 60 )
1094 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001095 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001096 intentStates = []
1097 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1098 count = 0
1099 # Iter through intents of a node
1100 try:
1101 for intent in json.loads( intents ):
1102 state = intent.get( 'state', None )
1103 if "INSTALLED" not in state:
1104 installedCheck = False
1105 intentId = intent.get( 'id', None )
1106 intentStates.append( ( intentId, state ) )
1107 except ( ValueError, TypeError ):
1108 main.log.exception( "Error parsing intents." )
1109 intentStates.sort()
1110 for i, s in intentStates:
1111 count += 1
1112 main.log.info( "%-6s%-15s%-15s" %
1113 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001114 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001115 try:
1116 missing = False
1117 if leaders:
1118 parsedLeaders = json.loads( leaders )
1119 main.log.warn( json.dumps( parsedLeaders,
1120 sort_keys=True,
1121 indent=4,
1122 separators=( ',', ': ' ) ) )
1123 # check for all intent partitions
1124 # check for election
1125 topics = []
1126 for i in range( 14 ):
1127 topics.append( "intent-partition-" + str( i ) )
1128 # FIXME: this should only be after we start the app
1129 topics.append( "org.onosproject.election" )
1130 main.log.debug( topics )
1131 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1132 for topic in topics:
1133 if topic not in ONOStopics:
1134 main.log.error( "Error: " + topic +
1135 " not in leaders" )
1136 missing = True
1137 else:
1138 main.log.error( "leaders() returned None" )
1139 except ( ValueError, TypeError ):
1140 main.log.exception( "Error parsing leaders" )
1141 main.log.error( repr( leaders ) )
1142 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001143 for i in main.activeNodes:
1144 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001145 response = node.leaders( jsonFormat=False)
1146 main.log.warn( str( node.name ) + " leaders output: \n" +
1147 str( response ) )
1148
Jon Halla440e872016-03-31 15:15:50 -07001149 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001150 try:
1151 if partitions :
1152 parsedPartitions = json.loads( partitions )
1153 main.log.warn( json.dumps( parsedPartitions,
1154 sort_keys=True,
1155 indent=4,
1156 separators=( ',', ': ' ) ) )
1157 # TODO check for a leader in all paritions
1158 # TODO check for consistency among nodes
1159 else:
1160 main.log.error( "partitions() returned None" )
1161 except ( ValueError, TypeError ):
1162 main.log.exception( "Error parsing partitions" )
1163 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001164 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001165 try:
1166 if pendingMap :
1167 parsedPending = json.loads( pendingMap )
1168 main.log.warn( json.dumps( parsedPending,
1169 sort_keys=True,
1170 indent=4,
1171 separators=( ',', ': ' ) ) )
1172 # TODO check something here?
1173 else:
1174 main.log.error( "pendingMap() returned None" )
1175 except ( ValueError, TypeError ):
1176 main.log.exception( "Error parsing pending map" )
1177 main.log.error( repr( pendingMap ) )
1178 # Print flowrules
Jon Halla440e872016-03-31 15:15:50 -07001179 node = main.activeNodes[0]
1180 main.log.debug( main.CLIs[node].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001181 main.step( "Wait a minute then ping again" )
1182 # the wait is above
1183 PingResult = main.TRUE
1184 for i in range( 8, 18 ):
1185 ping = main.Mininet1.pingHost( src="h" + str( i ),
1186 target="h" + str( i + 10 ) )
1187 PingResult = PingResult and ping
1188 if ping == main.FALSE:
1189 main.log.warn( "Ping failed between h" + str( i ) +
1190 " and h" + str( i + 10 ) )
1191 elif ping == main.TRUE:
1192 main.log.info( "Ping test passed!" )
1193 # Don't set PingResult or you'd override failures
1194 if PingResult == main.FALSE:
1195 main.log.error(
1196 "Intents have not been installed correctly, pings failed." )
1197 # TODO: pretty print
1198 main.log.warn( "ONOS1 intents: " )
1199 try:
Jon Halla440e872016-03-31 15:15:50 -07001200 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001201 main.log.warn( json.dumps( json.loads( tmpIntents ),
1202 sort_keys=True,
1203 indent=4,
1204 separators=( ',', ': ' ) ) )
1205 except ( ValueError, TypeError ):
1206 main.log.warn( repr( tmpIntents ) )
1207 utilities.assert_equals(
1208 expect=main.TRUE,
1209 actual=PingResult,
1210 onpass="Intents have been installed correctly and pings work",
1211 onfail="Intents have not been installed correctly, pings failed." )
1212
1213 def CASE5( self, main ):
1214 """
1215 Reading state of ONOS
1216 """
1217 import json
1218 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001219 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001220 assert main, "main not defined"
1221 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001222 assert main.CLIs, "main.CLIs not defined"
1223 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001224
1225 main.case( "Setting up and gathering data for current state" )
1226 # The general idea for this test case is to pull the state of
1227 # ( intents,flows, topology,... ) from each ONOS node
1228 # We can then compare them with each other and also with past states
1229
1230 main.step( "Check that each switch has a master" )
1231 global mastershipState
1232 mastershipState = '[]'
1233
1234 # Assert that each device has a master
1235 rolesNotNull = main.TRUE
1236 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001237 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001238 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001239 name="rolesNotNull-" + str( i ),
1240 args=[] )
1241 threads.append( t )
1242 t.start()
1243
1244 for t in threads:
1245 t.join()
1246 rolesNotNull = rolesNotNull and t.result
1247 utilities.assert_equals(
1248 expect=main.TRUE,
1249 actual=rolesNotNull,
1250 onpass="Each device has a master",
1251 onfail="Some devices don't have a master assigned" )
1252
1253 main.step( "Get the Mastership of each switch from each controller" )
1254 ONOSMastership = []
1255 mastershipCheck = main.FALSE
1256 consistentMastership = True
1257 rolesResults = True
1258 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001259 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001260 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001261 name="roles-" + str( i ),
1262 args=[] )
1263 threads.append( t )
1264 t.start()
1265
1266 for t in threads:
1267 t.join()
1268 ONOSMastership.append( t.result )
1269
Jon Halla440e872016-03-31 15:15:50 -07001270 for i in range( len( ONOSMastership ) ):
1271 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001272 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001273 main.log.error( "Error in getting ONOS" + node + " roles" )
1274 main.log.warn( "ONOS" + node + " mastership response: " +
1275 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001276 rolesResults = False
1277 utilities.assert_equals(
1278 expect=True,
1279 actual=rolesResults,
1280 onpass="No error in reading roles output",
1281 onfail="Error in reading roles from ONOS" )
1282
1283 main.step( "Check for consistency in roles from each controller" )
1284 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1285 main.log.info(
1286 "Switch roles are consistent across all ONOS nodes" )
1287 else:
1288 consistentMastership = False
1289 utilities.assert_equals(
1290 expect=True,
1291 actual=consistentMastership,
1292 onpass="Switch roles are consistent across all ONOS nodes",
1293 onfail="ONOS nodes have different views of switch roles" )
1294
1295 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001296 for i in range( len( main.activeNodes ) ):
1297 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001298 try:
1299 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001300 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001301 json.dumps(
1302 json.loads( ONOSMastership[ i ] ),
1303 sort_keys=True,
1304 indent=4,
1305 separators=( ',', ': ' ) ) )
1306 except ( ValueError, TypeError ):
1307 main.log.warn( repr( ONOSMastership[ i ] ) )
1308 elif rolesResults and consistentMastership:
1309 mastershipCheck = main.TRUE
1310 mastershipState = ONOSMastership[ 0 ]
1311
1312 main.step( "Get the intents from each controller" )
1313 global intentState
1314 intentState = []
1315 ONOSIntents = []
1316 intentCheck = main.FALSE
1317 consistentIntents = True
1318 intentsResults = True
1319 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001320 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001321 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001322 name="intents-" + str( i ),
1323 args=[],
1324 kwargs={ 'jsonFormat': True } )
1325 threads.append( t )
1326 t.start()
1327
1328 for t in threads:
1329 t.join()
1330 ONOSIntents.append( t.result )
1331
Jon Halla440e872016-03-31 15:15:50 -07001332 for i in range( len( ONOSIntents ) ):
1333 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001334 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hall6e709752016-02-01 13:38:46 -08001335 main.log.error( "Error in getting ONOS" + node + " intents" )
1336 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001337 repr( ONOSIntents[ i ] ) )
1338 intentsResults = False
1339 utilities.assert_equals(
1340 expect=True,
1341 actual=intentsResults,
1342 onpass="No error in reading intents output",
1343 onfail="Error in reading intents from ONOS" )
1344
1345 main.step( "Check for consistency in Intents from each controller" )
1346 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1347 main.log.info( "Intents are consistent across all ONOS " +
1348 "nodes" )
1349 else:
1350 consistentIntents = False
1351 main.log.error( "Intents not consistent" )
1352 utilities.assert_equals(
1353 expect=True,
1354 actual=consistentIntents,
1355 onpass="Intents are consistent across all ONOS nodes",
1356 onfail="ONOS nodes have different views of intents" )
1357
1358 if intentsResults:
1359 # Try to make it easy to figure out what is happening
1360 #
1361 # Intent ONOS1 ONOS2 ...
1362 # 0x01 INSTALLED INSTALLING
1363 # ... ... ...
1364 # ... ... ...
1365 title = " Id"
Jon Halla440e872016-03-31 15:15:50 -07001366 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001367 title += " " * 10 + "ONOS" + str( n + 1 )
1368 main.log.warn( title )
1369 # get all intent keys in the cluster
1370 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001371 try:
1372 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001373 for nodeStr in ONOSIntents:
1374 node = json.loads( nodeStr )
1375 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001376 keys.append( intent.get( 'id' ) )
1377 keys = set( keys )
1378 # For each intent key, print the state on each node
1379 for key in keys:
1380 row = "%-13s" % key
1381 for nodeStr in ONOSIntents:
1382 node = json.loads( nodeStr )
1383 for intent in node:
1384 if intent.get( 'id', "Error" ) == key:
1385 row += "%-15s" % intent.get( 'state' )
1386 main.log.warn( row )
1387 # End of intent state table
1388 except ValueError as e:
1389 main.log.exception( e )
1390 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001391
1392 if intentsResults and not consistentIntents:
1393 # print the json objects
Jon Halla440e872016-03-31 15:15:50 -07001394 n = str( main.activeNodes[-1] + 1 )
1395 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001396 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1397 sort_keys=True,
1398 indent=4,
1399 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001400 for i in range( len( ONOSIntents ) ):
1401 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001402 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07001403 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001404 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1405 sort_keys=True,
1406 indent=4,
1407 separators=( ',', ': ' ) ) )
1408 else:
Jon Halla440e872016-03-31 15:15:50 -07001409 main.log.debug( "ONOS" + node + " intents match ONOS" +
1410 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001411 elif intentsResults and consistentIntents:
1412 intentCheck = main.TRUE
1413 intentState = ONOSIntents[ 0 ]
1414
1415 main.step( "Get the flows from each controller" )
1416 global flowState
1417 flowState = []
1418 ONOSFlows = []
1419 ONOSFlowsJson = []
1420 flowCheck = main.FALSE
1421 consistentFlows = True
1422 flowsResults = True
1423 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001424 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001425 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001426 name="flows-" + str( i ),
1427 args=[],
1428 kwargs={ 'jsonFormat': True } )
1429 threads.append( t )
1430 t.start()
1431
1432 # NOTE: Flows command can take some time to run
1433 time.sleep(30)
1434 for t in threads:
1435 t.join()
1436 result = t.result
1437 ONOSFlows.append( result )
1438
Jon Halla440e872016-03-31 15:15:50 -07001439 for i in range( len( ONOSFlows ) ):
1440 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001441 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1442 main.log.error( "Error in getting ONOS" + num + " flows" )
1443 main.log.warn( "ONOS" + num + " flows response: " +
1444 repr( ONOSFlows[ i ] ) )
1445 flowsResults = False
1446 ONOSFlowsJson.append( None )
1447 else:
1448 try:
1449 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1450 except ( ValueError, TypeError ):
1451 # FIXME: change this to log.error?
1452 main.log.exception( "Error in parsing ONOS" + num +
1453 " response as json." )
1454 main.log.error( repr( ONOSFlows[ i ] ) )
1455 ONOSFlowsJson.append( None )
1456 flowsResults = False
1457 utilities.assert_equals(
1458 expect=True,
1459 actual=flowsResults,
1460 onpass="No error in reading flows output",
1461 onfail="Error in reading flows from ONOS" )
1462
1463 main.step( "Check for consistency in Flows from each controller" )
1464 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1465 if all( tmp ):
1466 main.log.info( "Flow count is consistent across all ONOS nodes" )
1467 else:
1468 consistentFlows = False
1469 utilities.assert_equals(
1470 expect=True,
1471 actual=consistentFlows,
1472 onpass="The flow count is consistent across all ONOS nodes",
1473 onfail="ONOS nodes have different flow counts" )
1474
1475 if flowsResults and not consistentFlows:
Jon Halla440e872016-03-31 15:15:50 -07001476 for i in range( len( ONOSFlows ) ):
1477 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001478 try:
1479 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001480 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001481 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1482 indent=4, separators=( ',', ': ' ) ) )
1483 except ( ValueError, TypeError ):
Jon Halla440e872016-03-31 15:15:50 -07001484 main.log.warn( "ONOS" + node + " flows: " +
1485 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001486 elif flowsResults and consistentFlows:
1487 flowCheck = main.TRUE
1488 flowState = ONOSFlows[ 0 ]
1489
1490 main.step( "Get the OF Table entries" )
1491 global flows
1492 flows = []
1493 for i in range( 1, 29 ):
Jon Halla440e872016-03-31 15:15:50 -07001494 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001495 if flowCheck == main.FALSE:
1496 for table in flows:
1497 main.log.warn( table )
1498 # TODO: Compare switch flow tables with ONOS flow tables
1499
1500 main.step( "Start continuous pings" )
1501 main.Mininet2.pingLong(
1502 src=main.params[ 'PING' ][ 'source1' ],
1503 target=main.params[ 'PING' ][ 'target1' ],
1504 pingTime=500 )
1505 main.Mininet2.pingLong(
1506 src=main.params[ 'PING' ][ 'source2' ],
1507 target=main.params[ 'PING' ][ 'target2' ],
1508 pingTime=500 )
1509 main.Mininet2.pingLong(
1510 src=main.params[ 'PING' ][ 'source3' ],
1511 target=main.params[ 'PING' ][ 'target3' ],
1512 pingTime=500 )
1513 main.Mininet2.pingLong(
1514 src=main.params[ 'PING' ][ 'source4' ],
1515 target=main.params[ 'PING' ][ 'target4' ],
1516 pingTime=500 )
1517 main.Mininet2.pingLong(
1518 src=main.params[ 'PING' ][ 'source5' ],
1519 target=main.params[ 'PING' ][ 'target5' ],
1520 pingTime=500 )
1521 main.Mininet2.pingLong(
1522 src=main.params[ 'PING' ][ 'source6' ],
1523 target=main.params[ 'PING' ][ 'target6' ],
1524 pingTime=500 )
1525 main.Mininet2.pingLong(
1526 src=main.params[ 'PING' ][ 'source7' ],
1527 target=main.params[ 'PING' ][ 'target7' ],
1528 pingTime=500 )
1529 main.Mininet2.pingLong(
1530 src=main.params[ 'PING' ][ 'source8' ],
1531 target=main.params[ 'PING' ][ 'target8' ],
1532 pingTime=500 )
1533 main.Mininet2.pingLong(
1534 src=main.params[ 'PING' ][ 'source9' ],
1535 target=main.params[ 'PING' ][ 'target9' ],
1536 pingTime=500 )
1537 main.Mininet2.pingLong(
1538 src=main.params[ 'PING' ][ 'source10' ],
1539 target=main.params[ 'PING' ][ 'target10' ],
1540 pingTime=500 )
1541
1542 main.step( "Collecting topology information from ONOS" )
1543 devices = []
1544 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001545 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001546 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001547 name="devices-" + str( i ),
1548 args=[ ] )
1549 threads.append( t )
1550 t.start()
1551
1552 for t in threads:
1553 t.join()
1554 devices.append( t.result )
1555 hosts = []
1556 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001557 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001558 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001559 name="hosts-" + str( i ),
1560 args=[ ] )
1561 threads.append( t )
1562 t.start()
1563
1564 for t in threads:
1565 t.join()
1566 try:
1567 hosts.append( json.loads( t.result ) )
1568 except ( ValueError, TypeError ):
1569 # FIXME: better handling of this, print which node
1570 # Maybe use thread name?
1571 main.log.exception( "Error parsing json output of hosts" )
Jon Hall3afe4c92015-12-14 19:30:38 -08001572 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001573 hosts.append( None )
1574
1575 ports = []
1576 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001577 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001578 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001579 name="ports-" + str( i ),
1580 args=[ ] )
1581 threads.append( t )
1582 t.start()
1583
1584 for t in threads:
1585 t.join()
1586 ports.append( t.result )
1587 links = []
1588 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001589 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001590 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001591 name="links-" + str( i ),
1592 args=[ ] )
1593 threads.append( t )
1594 t.start()
1595
1596 for t in threads:
1597 t.join()
1598 links.append( t.result )
1599 clusters = []
1600 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001601 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001602 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001603 name="clusters-" + str( i ),
1604 args=[ ] )
1605 threads.append( t )
1606 t.start()
1607
1608 for t in threads:
1609 t.join()
1610 clusters.append( t.result )
1611 # Compare json objects for hosts and dataplane clusters
1612
1613 # hosts
1614 main.step( "Host view is consistent across ONOS nodes" )
1615 consistentHostsResult = main.TRUE
1616 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001617 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001618 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001619 if hosts[ controller ] == hosts[ 0 ]:
1620 continue
1621 else: # hosts not consistent
1622 main.log.error( "hosts from ONOS" +
1623 controllerStr +
1624 " is inconsistent with ONOS1" )
1625 main.log.warn( repr( hosts[ controller ] ) )
1626 consistentHostsResult = main.FALSE
1627
1628 else:
1629 main.log.error( "Error in getting ONOS hosts from ONOS" +
1630 controllerStr )
1631 consistentHostsResult = main.FALSE
1632 main.log.warn( "ONOS" + controllerStr +
1633 " hosts response: " +
1634 repr( hosts[ controller ] ) )
1635 utilities.assert_equals(
1636 expect=main.TRUE,
1637 actual=consistentHostsResult,
1638 onpass="Hosts view is consistent across all ONOS nodes",
1639 onfail="ONOS nodes have different views of hosts" )
1640
1641 main.step( "Each host has an IP address" )
1642 ipResult = main.TRUE
1643 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001644 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001645 if hosts[ controller ]:
1646 for host in hosts[ controller ]:
1647 if not host.get( 'ipAddresses', [ ] ):
Jon Hallf3d16e72015-12-16 17:45:08 -08001648 main.log.error( "Error with host ips on controller" +
Jon Hall3afe4c92015-12-14 19:30:38 -08001649 controllerStr + ": " + str( host ) )
1650 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001651 utilities.assert_equals(
1652 expect=main.TRUE,
1653 actual=ipResult,
1654 onpass="The ips of the hosts aren't empty",
1655 onfail="The ip of at least one host is missing" )
1656
1657 # Strongly connected clusters of devices
1658 main.step( "Cluster view is consistent across ONOS nodes" )
1659 consistentClustersResult = main.TRUE
1660 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07001661 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001662 if "Error" not in clusters[ controller ]:
1663 if clusters[ controller ] == clusters[ 0 ]:
1664 continue
1665 else: # clusters not consistent
1666 main.log.error( "clusters from ONOS" + controllerStr +
1667 " is inconsistent with ONOS1" )
1668 consistentClustersResult = main.FALSE
1669
1670 else:
1671 main.log.error( "Error in getting dataplane clusters " +
1672 "from ONOS" + controllerStr )
1673 consistentClustersResult = main.FALSE
1674 main.log.warn( "ONOS" + controllerStr +
1675 " clusters response: " +
1676 repr( clusters[ controller ] ) )
1677 utilities.assert_equals(
1678 expect=main.TRUE,
1679 actual=consistentClustersResult,
1680 onpass="Clusters view is consistent across all ONOS nodes",
1681 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001682 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001683 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001684
Jon Hall5cf14d52015-07-16 12:15:19 -07001685 # there should always only be one cluster
1686 main.step( "Cluster view correct across ONOS nodes" )
1687 try:
1688 numClusters = len( json.loads( clusters[ 0 ] ) )
1689 except ( ValueError, TypeError ):
1690 main.log.exception( "Error parsing clusters[0]: " +
1691 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001692 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001693 clusterResults = main.FALSE
1694 if numClusters == 1:
1695 clusterResults = main.TRUE
1696 utilities.assert_equals(
1697 expect=1,
1698 actual=numClusters,
1699 onpass="ONOS shows 1 SCC",
1700 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1701
1702 main.step( "Comparing ONOS topology to MN" )
1703 devicesResults = main.TRUE
1704 linksResults = main.TRUE
1705 hostsResults = main.TRUE
1706 mnSwitches = main.Mininet1.getSwitches()
1707 mnLinks = main.Mininet1.getLinks()
1708 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001709 for controller in main.activeNodes:
1710 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001711 if devices[ controller ] and ports[ controller ] and\
1712 "Error" not in devices[ controller ] and\
1713 "Error" not in ports[ controller ]:
Jon Halle1a3b752015-07-22 13:02:46 -07001714 currentDevicesResult = main.Mininet1.compareSwitches(
1715 mnSwitches,
1716 json.loads( devices[ controller ] ),
1717 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001718 else:
1719 currentDevicesResult = main.FALSE
1720 utilities.assert_equals( expect=main.TRUE,
1721 actual=currentDevicesResult,
1722 onpass="ONOS" + controllerStr +
1723 " Switches view is correct",
1724 onfail="ONOS" + controllerStr +
1725 " Switches view is incorrect" )
1726 if links[ controller ] and "Error" not in links[ controller ]:
1727 currentLinksResult = main.Mininet1.compareLinks(
1728 mnSwitches, mnLinks,
1729 json.loads( links[ controller ] ) )
1730 else:
1731 currentLinksResult = main.FALSE
1732 utilities.assert_equals( expect=main.TRUE,
1733 actual=currentLinksResult,
1734 onpass="ONOS" + controllerStr +
1735 " links view is correct",
1736 onfail="ONOS" + controllerStr +
1737 " links view is incorrect" )
1738
Jon Hall657cdf62015-12-17 14:40:51 -08001739 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001740 currentHostsResult = main.Mininet1.compareHosts(
1741 mnHosts,
1742 hosts[ controller ] )
1743 else:
1744 currentHostsResult = main.FALSE
1745 utilities.assert_equals( expect=main.TRUE,
1746 actual=currentHostsResult,
1747 onpass="ONOS" + controllerStr +
1748 " hosts exist in Mininet",
1749 onfail="ONOS" + controllerStr +
1750 " hosts don't match Mininet" )
1751
1752 devicesResults = devicesResults and currentDevicesResult
1753 linksResults = linksResults and currentLinksResult
1754 hostsResults = hostsResults and currentHostsResult
1755
1756 main.step( "Device information is correct" )
1757 utilities.assert_equals(
1758 expect=main.TRUE,
1759 actual=devicesResults,
1760 onpass="Device information is correct",
1761 onfail="Device information is incorrect" )
1762
1763 main.step( "Links are correct" )
1764 utilities.assert_equals(
1765 expect=main.TRUE,
1766 actual=linksResults,
1767 onpass="Link are correct",
1768 onfail="Links are incorrect" )
1769
1770 main.step( "Hosts are correct" )
1771 utilities.assert_equals(
1772 expect=main.TRUE,
1773 actual=hostsResults,
1774 onpass="Hosts are correct",
1775 onfail="Hosts are incorrect" )
1776
1777 def CASE6( self, main ):
1778 """
1779 The Failure case.
1780 """
1781 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001782 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001783 assert main, "main not defined"
1784 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001785 assert main.CLIs, "main.CLIs not defined"
1786 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001787 try:
1788 labels
1789 except NameError:
1790 main.log.error( "labels not defined, setting to []" )
1791 global labels
1792 labels = []
1793 try:
1794 data
1795 except NameError:
1796 main.log.error( "data not defined, setting to []" )
1797 global data
1798 data = []
1799 # Reset non-persistent variables
1800 try:
1801 iCounterValue = 0
1802 except NameError:
1803 main.log.error( "iCounterValue not defined, setting to 0" )
1804 iCounterValue = 0
1805
1806 main.case( "Restart entire ONOS cluster" )
1807
Jon Hall5ec6b1b2015-09-17 18:20:14 -07001808 main.step( "Checking ONOS Logs for errors" )
1809 for node in main.nodes:
1810 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1811 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1812
Jon Hall5cf14d52015-07-16 12:15:19 -07001813 main.step( "Killing ONOS nodes" )
1814 killResults = main.TRUE
1815 killTime = time.time()
Jon Halle1a3b752015-07-22 13:02:46 -07001816 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001817 killed = main.ONOSbench.onosKill( node.ip_address )
1818 killResults = killResults and killed
1819 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1820 onpass="ONOS nodes killed",
1821 onfail="ONOS kill unsuccessful" )
1822
1823 main.step( "Checking if ONOS is up yet" )
1824 for i in range( 2 ):
1825 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001826 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001827 started = main.ONOSbench.isup( node.ip_address )
1828 if not started:
1829 main.log.error( node.name + " didn't start!" )
1830 onosIsupResult = onosIsupResult and started
1831 if onosIsupResult == main.TRUE:
1832 break
1833 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1834 onpass="ONOS restarted",
1835 onfail="ONOS restart NOT successful" )
1836
1837 main.log.step( "Starting ONOS CLI sessions" )
1838 cliResults = main.TRUE
1839 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001840 for i in range( main.numCtrls ):
1841 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -07001842 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -07001843 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001844 threads.append( t )
1845 t.start()
1846
1847 for t in threads:
1848 t.join()
1849 cliResults = cliResults and t.result
1850 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1851 onpass="ONOS cli started",
1852 onfail="ONOS clis did not restart" )
1853
Jon Hall6e709752016-02-01 13:38:46 -08001854 for i in range( 10 ):
1855 ready = True
Jon Hall7ac7bc32016-05-05 10:57:02 -07001856 for i in main.activeNodes:
1857 cli = main.CLIs[i]
Jon Hall6e709752016-02-01 13:38:46 -08001858 output = cli.summary()
1859 if not output:
1860 ready = False
1861 time.sleep( 30 )
1862 utilities.assert_equals( expect=True, actual=ready,
1863 onpass="ONOS summary command succeded",
1864 onfail="ONOS summary command failed" )
1865 if not ready:
1866 main.cleanup()
1867 main.exit()
1868
Jon Hall5cf14d52015-07-16 12:15:19 -07001869 # Grab the time of restart so we chan check how long the gossip
1870 # protocol has had time to work
1871 main.restartTime = time.time() - killTime
1872 main.log.debug( "Restart time: " + str( main.restartTime ) )
1873 labels.append( "Restart" )
1874 data.append( str( main.restartTime ) )
1875
Jon Hall5cf14d52015-07-16 12:15:19 -07001876 # Rerun for election on restarted nodes
1877 runResults = main.TRUE
Jon Hall7ac7bc32016-05-05 10:57:02 -07001878 for i in main.activeNodes:
1879 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -07001880 run = cli.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001881 if run != main.TRUE:
1882 main.log.error( "Error running for election on " + cli.name )
1883 runResults = runResults and run
1884 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1885 onpass="Reran for election",
1886 onfail="Failed to rerun for election" )
1887
1888 # TODO: Make this configurable
1889 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -07001890 node = main.activeNodes[0]
1891 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1892 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1893 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001894
1895 def CASE7( self, main ):
1896 """
1897 Check state after ONOS failure
1898 """
1899 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001900 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001901 assert main, "main not defined"
1902 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001903 assert main.CLIs, "main.CLIs not defined"
1904 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001905 main.case( "Running ONOS Constant State Tests" )
1906
1907 main.step( "Check that each switch has a master" )
1908 # Assert that each device has a master
1909 rolesNotNull = main.TRUE
1910 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001911 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001912 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001913 name="rolesNotNull-" + str( i ),
1914 args=[ ] )
1915 threads.append( t )
1916 t.start()
1917
1918 for t in threads:
1919 t.join()
1920 rolesNotNull = rolesNotNull and t.result
1921 utilities.assert_equals(
1922 expect=main.TRUE,
1923 actual=rolesNotNull,
1924 onpass="Each device has a master",
1925 onfail="Some devices don't have a master assigned" )
1926
1927 main.step( "Read device roles from ONOS" )
1928 ONOSMastership = []
1929 mastershipCheck = main.FALSE
1930 consistentMastership = True
1931 rolesResults = True
1932 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001933 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001934 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001935 name="roles-" + str( i ),
1936 args=[] )
1937 threads.append( t )
1938 t.start()
1939
1940 for t in threads:
1941 t.join()
1942 ONOSMastership.append( t.result )
1943
Jon Halla440e872016-03-31 15:15:50 -07001944 for i in range( len( ONOSMastership ) ):
1945 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001946 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001947 main.log.error( "Error in getting ONOS" + node + " roles" )
1948 main.log.warn( "ONOS" + node + " mastership response: " +
1949 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001950 rolesResults = False
1951 utilities.assert_equals(
1952 expect=True,
1953 actual=rolesResults,
1954 onpass="No error in reading roles output",
1955 onfail="Error in reading roles from ONOS" )
1956
1957 main.step( "Check for consistency in roles from each controller" )
1958 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1959 main.log.info(
1960 "Switch roles are consistent across all ONOS nodes" )
1961 else:
1962 consistentMastership = False
1963 utilities.assert_equals(
1964 expect=True,
1965 actual=consistentMastership,
1966 onpass="Switch roles are consistent across all ONOS nodes",
1967 onfail="ONOS nodes have different views of switch roles" )
1968
1969 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001970 for i in range( len( ONOSMastership ) ):
1971 node = str( main.activeNodes[i] + 1 )
1972 main.log.warn( "ONOS" + node + " roles: ",
Jon Hall6e709752016-02-01 13:38:46 -08001973 json.dumps( json.loads( ONOSMastership[ i ] ),
1974 sort_keys=True,
1975 indent=4,
1976 separators=( ',', ': ' ) ) )
1977 elif rolesResults and consistentMastership:
Jon Hall5cf14d52015-07-16 12:15:19 -07001978 mastershipCheck = main.TRUE
1979
Jon Hall5cf14d52015-07-16 12:15:19 -07001980 # NOTE: we expect mastership to change on controller failure
1981
1982 main.step( "Get the intents and compare across all nodes" )
1983 ONOSIntents = []
1984 intentCheck = main.FALSE
1985 consistentIntents = True
1986 intentsResults = True
1987 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001988 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001989 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001990 name="intents-" + str( i ),
1991 args=[],
1992 kwargs={ 'jsonFormat': True } )
1993 threads.append( t )
1994 t.start()
1995
1996 for t in threads:
1997 t.join()
1998 ONOSIntents.append( t.result )
1999
Jon Halla440e872016-03-31 15:15:50 -07002000 for i in range( len( ONOSIntents) ):
2001 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002002 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07002003 main.log.error( "Error in getting ONOS" + node + " intents" )
2004 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002005 repr( ONOSIntents[ i ] ) )
2006 intentsResults = False
2007 utilities.assert_equals(
2008 expect=True,
2009 actual=intentsResults,
2010 onpass="No error in reading intents output",
2011 onfail="Error in reading intents from ONOS" )
2012
2013 main.step( "Check for consistency in Intents from each controller" )
2014 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2015 main.log.info( "Intents are consistent across all ONOS " +
2016 "nodes" )
2017 else:
2018 consistentIntents = False
2019
2020 # Try to make it easy to figure out what is happening
2021 #
2022 # Intent ONOS1 ONOS2 ...
2023 # 0x01 INSTALLED INSTALLING
2024 # ... ... ...
2025 # ... ... ...
2026 title = " ID"
Jon Halla440e872016-03-31 15:15:50 -07002027 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002028 title += " " * 10 + "ONOS" + str( n + 1 )
2029 main.log.warn( title )
2030 # get all intent keys in the cluster
2031 keys = []
2032 for nodeStr in ONOSIntents:
2033 node = json.loads( nodeStr )
2034 for intent in node:
2035 keys.append( intent.get( 'id' ) )
2036 keys = set( keys )
2037 for key in keys:
2038 row = "%-13s" % key
2039 for nodeStr in ONOSIntents:
2040 node = json.loads( nodeStr )
2041 for intent in node:
2042 if intent.get( 'id' ) == key:
2043 row += "%-15s" % intent.get( 'state' )
2044 main.log.warn( row )
2045 # End table view
2046
2047 utilities.assert_equals(
2048 expect=True,
2049 actual=consistentIntents,
2050 onpass="Intents are consistent across all ONOS nodes",
2051 onfail="ONOS nodes have different views of intents" )
2052 intentStates = []
2053 for node in ONOSIntents: # Iter through ONOS nodes
2054 nodeStates = []
2055 # Iter through intents of a node
2056 try:
2057 for intent in json.loads( node ):
2058 nodeStates.append( intent[ 'state' ] )
2059 except ( ValueError, TypeError ):
2060 main.log.exception( "Error in parsing intents" )
2061 main.log.error( repr( node ) )
2062 intentStates.append( nodeStates )
2063 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2064 main.log.info( dict( out ) )
2065
2066 if intentsResults and not consistentIntents:
Jon Halla440e872016-03-31 15:15:50 -07002067 for i in range( len( main.activeNodes ) ):
2068 node = str( main.activeNodes[i] + 1 )
2069 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002070 main.log.warn( json.dumps(
2071 json.loads( ONOSIntents[ i ] ),
2072 sort_keys=True,
2073 indent=4,
2074 separators=( ',', ': ' ) ) )
2075 elif intentsResults and consistentIntents:
2076 intentCheck = main.TRUE
2077
2078 # NOTE: Store has no durability, so intents are lost across system
2079 # restarts
2080 """
2081 main.step( "Compare current intents with intents before the failure" )
2082 # NOTE: this requires case 5 to pass for intentState to be set.
2083 # maybe we should stop the test if that fails?
2084 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002085 try:
2086 intentState
2087 except NameError:
2088 main.log.warn( "No previous intent state was saved" )
2089 else:
2090 if intentState and intentState == ONOSIntents[ 0 ]:
2091 sameIntents = main.TRUE
2092 main.log.info( "Intents are consistent with before failure" )
2093 # TODO: possibly the states have changed? we may need to figure out
2094 # what the acceptable states are
2095 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2096 sameIntents = main.TRUE
2097 try:
2098 before = json.loads( intentState )
2099 after = json.loads( ONOSIntents[ 0 ] )
2100 for intent in before:
2101 if intent not in after:
2102 sameIntents = main.FALSE
2103 main.log.debug( "Intent is not currently in ONOS " +
2104 "(at least in the same form):" )
2105 main.log.debug( json.dumps( intent ) )
2106 except ( ValueError, TypeError ):
2107 main.log.exception( "Exception printing intents" )
2108 main.log.debug( repr( ONOSIntents[0] ) )
2109 main.log.debug( repr( intentState ) )
2110 if sameIntents == main.FALSE:
2111 try:
2112 main.log.debug( "ONOS intents before: " )
2113 main.log.debug( json.dumps( json.loads( intentState ),
2114 sort_keys=True, indent=4,
2115 separators=( ',', ': ' ) ) )
2116 main.log.debug( "Current ONOS intents: " )
2117 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2118 sort_keys=True, indent=4,
2119 separators=( ',', ': ' ) ) )
2120 except ( ValueError, TypeError ):
2121 main.log.exception( "Exception printing intents" )
2122 main.log.debug( repr( ONOSIntents[0] ) )
2123 main.log.debug( repr( intentState ) )
2124 utilities.assert_equals(
2125 expect=main.TRUE,
2126 actual=sameIntents,
2127 onpass="Intents are consistent with before failure",
2128 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002129 intentCheck = intentCheck and sameIntents
2130 """
2131 main.step( "Get the OF Table entries and compare to before " +
2132 "component failure" )
2133 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002134 for i in range( 28 ):
2135 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002136 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002137 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2138 FlowTables = FlowTables and curSwitch
2139 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002140 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002141 utilities.assert_equals(
2142 expect=main.TRUE,
2143 actual=FlowTables,
2144 onpass="No changes were found in the flow tables",
2145 onfail="Changes were found in the flow tables" )
2146
2147 main.Mininet2.pingLongKill()
2148 '''
2149 # main.step( "Check the continuous pings to ensure that no packets " +
2150 # "were dropped during component failure" )
2151 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2152 main.params[ 'TESTONIP' ] )
2153 LossInPings = main.FALSE
2154 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2155 for i in range( 8, 18 ):
2156 main.log.info(
2157 "Checking for a loss in pings along flow from s" +
2158 str( i ) )
2159 LossInPings = main.Mininet2.checkForLoss(
2160 "/tmp/ping.h" +
2161 str( i ) ) or LossInPings
2162 if LossInPings == main.TRUE:
2163 main.log.info( "Loss in ping detected" )
2164 elif LossInPings == main.ERROR:
2165 main.log.info( "There are multiple mininet process running" )
2166 elif LossInPings == main.FALSE:
2167 main.log.info( "No Loss in the pings" )
2168 main.log.info( "No loss of dataplane connectivity" )
2169 # utilities.assert_equals(
2170 # expect=main.FALSE,
2171 # actual=LossInPings,
2172 # onpass="No Loss of connectivity",
2173 # onfail="Loss of dataplane connectivity detected" )
2174
2175 # NOTE: Since intents are not persisted with IntnentStore,
2176 # we expect loss in dataplane connectivity
2177 LossInPings = main.FALSE
2178 '''
2179
2180 main.step( "Leadership Election is still functional" )
2181 # Test of LeadershipElection
2182 leaderList = []
2183 leaderResult = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002184
2185 for i in main.activeNodes:
2186 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002187 leaderN = cli.electionTestLeader()
2188 leaderList.append( leaderN )
2189 if leaderN == main.FALSE:
2190 # error in response
2191 main.log.error( "Something is wrong with " +
2192 "electionTestLeader function, check the" +
2193 " error logs" )
2194 leaderResult = main.FALSE
2195 elif leaderN is None:
2196 main.log.error( cli.name +
2197 " shows no leader for the election-app." )
2198 leaderResult = main.FALSE
2199 if len( set( leaderList ) ) != 1:
2200 leaderResult = main.FALSE
2201 main.log.error(
2202 "Inconsistent view of leader for the election test app" )
2203 # TODO: print the list
2204 utilities.assert_equals(
2205 expect=main.TRUE,
2206 actual=leaderResult,
2207 onpass="Leadership election passed",
2208 onfail="Something went wrong with Leadership election" )
2209
2210 def CASE8( self, main ):
2211 """
2212 Compare topo
2213 """
2214 import json
2215 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002216 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002217 assert main, "main not defined"
2218 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002219 assert main.CLIs, "main.CLIs not defined"
2220 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002221
2222 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002223 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002224 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002225 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002226 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002227 elapsed = 0
2228 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002229 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002230 startTime = time.time()
2231 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002232 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hallba609822015-09-18 12:00:21 -07002233 devicesResults = main.TRUE
2234 linksResults = main.TRUE
2235 hostsResults = main.TRUE
2236 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002237 count += 1
2238 cliStart = time.time()
2239 devices = []
2240 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002241 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002242 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002243 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002244 args=[ main.CLIs[i].devices, [ None ] ],
2245 kwargs= { 'sleep': 5, 'attempts': 5,
2246 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002247 threads.append( t )
2248 t.start()
2249
2250 for t in threads:
2251 t.join()
2252 devices.append( t.result )
2253 hosts = []
2254 ipResult = main.TRUE
2255 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002256 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002257 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002258 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002259 args=[ main.CLIs[i].hosts, [ None ] ],
2260 kwargs= { 'sleep': 5, 'attempts': 5,
2261 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002262 threads.append( t )
2263 t.start()
2264
2265 for t in threads:
2266 t.join()
2267 try:
2268 hosts.append( json.loads( t.result ) )
2269 except ( ValueError, TypeError ):
2270 main.log.exception( "Error parsing hosts results" )
2271 main.log.error( repr( t.result ) )
Jon Hall3afe4c92015-12-14 19:30:38 -08002272 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002273 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002274 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002275 if hosts[ controller ]:
2276 for host in hosts[ controller ]:
2277 if host is None or host.get( 'ipAddresses', [] ) == []:
2278 main.log.error(
2279 "Error with host ipAddresses on controller" +
2280 controllerStr + ": " + str( host ) )
2281 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002282 ports = []
2283 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002284 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002285 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002286 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002287 args=[ main.CLIs[i].ports, [ None ] ],
2288 kwargs= { 'sleep': 5, 'attempts': 5,
2289 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002290 threads.append( t )
2291 t.start()
2292
2293 for t in threads:
2294 t.join()
2295 ports.append( t.result )
2296 links = []
2297 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002298 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002299 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002300 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002301 args=[ main.CLIs[i].links, [ None ] ],
2302 kwargs= { 'sleep': 5, 'attempts': 5,
2303 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002304 threads.append( t )
2305 t.start()
2306
2307 for t in threads:
2308 t.join()
2309 links.append( t.result )
2310 clusters = []
2311 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002312 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002313 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002314 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002315 args=[ main.CLIs[i].clusters, [ None ] ],
2316 kwargs= { 'sleep': 5, 'attempts': 5,
2317 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002318 threads.append( t )
2319 t.start()
2320
2321 for t in threads:
2322 t.join()
2323 clusters.append( t.result )
2324
2325 elapsed = time.time() - startTime
2326 cliTime = time.time() - cliStart
2327 print "Elapsed time: " + str( elapsed )
2328 print "CLI time: " + str( cliTime )
2329
Jon Hall6e709752016-02-01 13:38:46 -08002330 if all( e is None for e in devices ) and\
2331 all( e is None for e in hosts ) and\
2332 all( e is None for e in ports ) and\
2333 all( e is None for e in links ) and\
2334 all( e is None for e in clusters ):
2335 topoFailMsg = "Could not get topology from ONOS"
2336 main.log.error( topoFailMsg )
2337 continue # Try again, No use trying to compare
2338
Jon Hall5cf14d52015-07-16 12:15:19 -07002339 mnSwitches = main.Mininet1.getSwitches()
2340 mnLinks = main.Mininet1.getLinks()
2341 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07002342 for controller in range( len( main.activeNodes ) ):
2343 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002344 if devices[ controller ] and ports[ controller ] and\
2345 "Error" not in devices[ controller ] and\
2346 "Error" not in ports[ controller ]:
2347
Jon Hallc6793552016-01-19 14:18:37 -08002348 try:
2349 currentDevicesResult = main.Mininet1.compareSwitches(
2350 mnSwitches,
2351 json.loads( devices[ controller ] ),
2352 json.loads( ports[ controller ] ) )
2353 except ( TypeError, ValueError ) as e:
2354 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2355 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002356 else:
2357 currentDevicesResult = main.FALSE
2358 utilities.assert_equals( expect=main.TRUE,
2359 actual=currentDevicesResult,
2360 onpass="ONOS" + controllerStr +
2361 " Switches view is correct",
2362 onfail="ONOS" + controllerStr +
2363 " Switches view is incorrect" )
2364
2365 if links[ controller ] and "Error" not in links[ controller ]:
2366 currentLinksResult = main.Mininet1.compareLinks(
2367 mnSwitches, mnLinks,
2368 json.loads( links[ controller ] ) )
2369 else:
2370 currentLinksResult = main.FALSE
2371 utilities.assert_equals( expect=main.TRUE,
2372 actual=currentLinksResult,
2373 onpass="ONOS" + controllerStr +
2374 " links view is correct",
2375 onfail="ONOS" + controllerStr +
2376 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002377 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002378 currentHostsResult = main.Mininet1.compareHosts(
2379 mnHosts,
2380 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002381 elif hosts[ controller ] == []:
2382 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002383 else:
2384 currentHostsResult = main.FALSE
2385 utilities.assert_equals( expect=main.TRUE,
2386 actual=currentHostsResult,
2387 onpass="ONOS" + controllerStr +
2388 " hosts exist in Mininet",
2389 onfail="ONOS" + controllerStr +
2390 " hosts don't match Mininet" )
2391 # CHECKING HOST ATTACHMENT POINTS
2392 hostAttachment = True
Jon Halla440e872016-03-31 15:15:50 -07002393 zeroHosts = False
Jon Hall5cf14d52015-07-16 12:15:19 -07002394 # FIXME: topo-HA/obelisk specific mappings:
2395 # key is mac and value is dpid
2396 mappings = {}
2397 for i in range( 1, 29 ): # hosts 1 through 28
2398 # set up correct variables:
2399 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2400 if i == 1:
2401 deviceId = "1000".zfill(16)
2402 elif i == 2:
2403 deviceId = "2000".zfill(16)
2404 elif i == 3:
2405 deviceId = "3000".zfill(16)
2406 elif i == 4:
2407 deviceId = "3004".zfill(16)
2408 elif i == 5:
2409 deviceId = "5000".zfill(16)
2410 elif i == 6:
2411 deviceId = "6000".zfill(16)
2412 elif i == 7:
2413 deviceId = "6007".zfill(16)
2414 elif i >= 8 and i <= 17:
2415 dpid = '3' + str( i ).zfill( 3 )
2416 deviceId = dpid.zfill(16)
2417 elif i >= 18 and i <= 27:
2418 dpid = '6' + str( i ).zfill( 3 )
2419 deviceId = dpid.zfill(16)
2420 elif i == 28:
2421 deviceId = "2800".zfill(16)
2422 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002423 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002424 if hosts[ controller ] == []:
2425 main.log.warn( "There are no hosts discovered" )
Jon Halla440e872016-03-31 15:15:50 -07002426 zeroHosts = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002427 else:
2428 for host in hosts[ controller ]:
2429 mac = None
2430 location = None
2431 device = None
2432 port = None
2433 try:
2434 mac = host.get( 'mac' )
2435 assert mac, "mac field could not be found for this host object"
2436
2437 location = host.get( 'location' )
2438 assert location, "location field could not be found for this host object"
2439
2440 # Trim the protocol identifier off deviceId
2441 device = str( location.get( 'elementId' ) ).split(':')[1]
2442 assert device, "elementId field could not be found for this host location object"
2443
2444 port = location.get( 'port' )
2445 assert port, "port field could not be found for this host location object"
2446
2447 # Now check if this matches where they should be
2448 if mac and device and port:
2449 if str( port ) != "1":
2450 main.log.error( "The attachment port is incorrect for " +
2451 "host " + str( mac ) +
2452 ". Expected: 1 Actual: " + str( port) )
2453 hostAttachment = False
2454 if device != mappings[ str( mac ) ]:
2455 main.log.error( "The attachment device is incorrect for " +
2456 "host " + str( mac ) +
2457 ". Expected: " + mappings[ str( mac ) ] +
2458 " Actual: " + device )
2459 hostAttachment = False
2460 else:
2461 hostAttachment = False
2462 except AssertionError:
2463 main.log.exception( "Json object not as expected" )
2464 main.log.error( repr( host ) )
2465 hostAttachment = False
2466 else:
2467 main.log.error( "No hosts json output or \"Error\"" +
2468 " in output. hosts = " +
2469 repr( hosts[ controller ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002470 if zeroHosts is False:
Jon Hall5cf14d52015-07-16 12:15:19 -07002471 # TODO: Find a way to know if there should be hosts in a
2472 # given point of the test
2473 hostAttachment = True
2474
2475 # END CHECKING HOST ATTACHMENT POINTS
2476 devicesResults = devicesResults and currentDevicesResult
2477 linksResults = linksResults and currentLinksResult
2478 hostsResults = hostsResults and currentHostsResult
2479 hostAttachmentResults = hostAttachmentResults and\
2480 hostAttachment
2481 topoResult = ( devicesResults and linksResults
2482 and hostsResults and ipResult and
2483 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002484 utilities.assert_equals( expect=True,
2485 actual=topoResult,
2486 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002487 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002488 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002489
2490 # Compare json objects for hosts and dataplane clusters
2491
2492 # hosts
2493 main.step( "Hosts view is consistent across all ONOS nodes" )
2494 consistentHostsResult = main.TRUE
2495 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002496 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002497 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002498 if hosts[ controller ] == hosts[ 0 ]:
2499 continue
2500 else: # hosts not consistent
2501 main.log.error( "hosts from ONOS" + controllerStr +
2502 " is inconsistent with ONOS1" )
2503 main.log.warn( repr( hosts[ controller ] ) )
2504 consistentHostsResult = main.FALSE
2505
2506 else:
2507 main.log.error( "Error in getting ONOS hosts from ONOS" +
2508 controllerStr )
2509 consistentHostsResult = main.FALSE
2510 main.log.warn( "ONOS" + controllerStr +
2511 " hosts response: " +
2512 repr( hosts[ controller ] ) )
2513 utilities.assert_equals(
2514 expect=main.TRUE,
2515 actual=consistentHostsResult,
2516 onpass="Hosts view is consistent across all ONOS nodes",
2517 onfail="ONOS nodes have different views of hosts" )
2518
2519 main.step( "Hosts information is correct" )
2520 hostsResults = hostsResults and ipResult
2521 utilities.assert_equals(
2522 expect=main.TRUE,
2523 actual=hostsResults,
2524 onpass="Host information is correct",
2525 onfail="Host information is incorrect" )
2526
2527 main.step( "Host attachment points to the network" )
2528 utilities.assert_equals(
2529 expect=True,
2530 actual=hostAttachmentResults,
2531 onpass="Hosts are correctly attached to the network",
2532 onfail="ONOS did not correctly attach hosts to the network" )
2533
2534 # Strongly connected clusters of devices
2535 main.step( "Clusters view is consistent across all ONOS nodes" )
2536 consistentClustersResult = main.TRUE
2537 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07002538 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002539 if "Error" not in clusters[ controller ]:
2540 if clusters[ controller ] == clusters[ 0 ]:
2541 continue
2542 else: # clusters not consistent
2543 main.log.error( "clusters from ONOS" +
2544 controllerStr +
2545 " is inconsistent with ONOS1" )
2546 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002547 else:
2548 main.log.error( "Error in getting dataplane clusters " +
2549 "from ONOS" + controllerStr )
2550 consistentClustersResult = main.FALSE
2551 main.log.warn( "ONOS" + controllerStr +
2552 " clusters response: " +
2553 repr( clusters[ controller ] ) )
2554 utilities.assert_equals(
2555 expect=main.TRUE,
2556 actual=consistentClustersResult,
2557 onpass="Clusters view is consistent across all ONOS nodes",
2558 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002559 if not consistentClustersResult:
2560 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002561
2562 main.step( "There is only one SCC" )
2563 # there should always only be one cluster
2564 try:
2565 numClusters = len( json.loads( clusters[ 0 ] ) )
2566 except ( ValueError, TypeError ):
2567 main.log.exception( "Error parsing clusters[0]: " +
2568 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002569 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002570 clusterResults = main.FALSE
2571 if numClusters == 1:
2572 clusterResults = main.TRUE
2573 utilities.assert_equals(
2574 expect=1,
2575 actual=numClusters,
2576 onpass="ONOS shows 1 SCC",
2577 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2578
2579 topoResult = ( devicesResults and linksResults
2580 and hostsResults and consistentHostsResult
2581 and consistentClustersResult and clusterResults
2582 and ipResult and hostAttachmentResults )
2583
2584 topoResult = topoResult and int( count <= 2 )
2585 note = "note it takes about " + str( int( cliTime ) ) + \
2586 " seconds for the test to make all the cli calls to fetch " +\
2587 "the topology from each ONOS instance"
2588 main.log.info(
2589 "Very crass estimate for topology discovery/convergence( " +
2590 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2591 str( count ) + " tries" )
2592
2593 main.step( "Device information is correct" )
2594 utilities.assert_equals(
2595 expect=main.TRUE,
2596 actual=devicesResults,
2597 onpass="Device information is correct",
2598 onfail="Device information is incorrect" )
2599
2600 main.step( "Links are correct" )
2601 utilities.assert_equals(
2602 expect=main.TRUE,
2603 actual=linksResults,
2604 onpass="Link are correct",
2605 onfail="Links are incorrect" )
2606
Jon Halla440e872016-03-31 15:15:50 -07002607 main.step( "Hosts are correct" )
2608 utilities.assert_equals(
2609 expect=main.TRUE,
2610 actual=hostsResults,
2611 onpass="Hosts are correct",
2612 onfail="Hosts are incorrect" )
2613
Jon Hall5cf14d52015-07-16 12:15:19 -07002614 # FIXME: move this to an ONOS state case
2615 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002616 nodeResults = utilities.retry( main.HA.nodesCheck,
2617 False,
2618 args=[main.activeNodes],
2619 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002620
Jon Hall41d39f12016-04-11 22:54:35 -07002621 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002622 onpass="Nodes check successful",
2623 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002624 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002625 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002626 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002627 main.CLIs[i].name,
2628 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002629
2630 def CASE9( self, main ):
2631 """
2632 Link s3-s28 down
2633 """
2634 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002635 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002636 assert main, "main not defined"
2637 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002638 assert main.CLIs, "main.CLIs not defined"
2639 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002640 # NOTE: You should probably run a topology check after this
2641
2642 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2643
2644 description = "Turn off a link to ensure that Link Discovery " +\
2645 "is working properly"
2646 main.case( description )
2647
2648 main.step( "Kill Link between s3 and s28" )
2649 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2650 main.log.info( "Waiting " + str( linkSleep ) +
2651 " seconds for link down to be discovered" )
2652 time.sleep( linkSleep )
2653 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2654 onpass="Link down successful",
2655 onfail="Failed to bring link down" )
2656 # TODO do some sort of check here
2657
2658 def CASE10( self, main ):
2659 """
2660 Link s3-s28 up
2661 """
2662 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002663 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002664 assert main, "main not defined"
2665 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002666 assert main.CLIs, "main.CLIs not defined"
2667 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002668 # NOTE: You should probably run a topology check after this
2669
2670 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2671
2672 description = "Restore a link to ensure that Link Discovery is " + \
2673 "working properly"
2674 main.case( description )
2675
2676 main.step( "Bring link between s3 and s28 back up" )
2677 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2678 main.log.info( "Waiting " + str( linkSleep ) +
2679 " seconds for link up to be discovered" )
2680 time.sleep( linkSleep )
2681 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2682 onpass="Link up successful",
2683 onfail="Failed to bring link up" )
2684 # TODO do some sort of check here
2685
2686 def CASE11( self, main ):
2687 """
2688 Switch Down
2689 """
2690 # NOTE: You should probably run a topology check after this
2691 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002692 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002693 assert main, "main not defined"
2694 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002695 assert main.CLIs, "main.CLIs not defined"
2696 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002697
2698 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2699
2700 description = "Killing a switch to ensure it is discovered correctly"
Jon Halla440e872016-03-31 15:15:50 -07002701 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002702 main.case( description )
2703 switch = main.params[ 'kill' ][ 'switch' ]
2704 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2705
2706 # TODO: Make this switch parameterizable
2707 main.step( "Kill " + switch )
2708 main.log.info( "Deleting " + switch )
2709 main.Mininet1.delSwitch( switch )
2710 main.log.info( "Waiting " + str( switchSleep ) +
2711 " seconds for switch down to be discovered" )
2712 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002713 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002714 # Peek at the deleted switch
2715 main.log.warn( str( device ) )
2716 result = main.FALSE
2717 if device and device[ 'available' ] is False:
2718 result = main.TRUE
2719 utilities.assert_equals( expect=main.TRUE, actual=result,
2720 onpass="Kill switch successful",
2721 onfail="Failed to kill switch?" )
2722
2723 def CASE12( self, main ):
2724 """
2725 Switch Up
2726 """
2727 # NOTE: You should probably run a topology check after this
2728 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002729 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002730 assert main, "main not defined"
2731 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002732 assert main.CLIs, "main.CLIs not defined"
2733 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002734 assert ONOS1Port, "ONOS1Port not defined"
2735 assert ONOS2Port, "ONOS2Port not defined"
2736 assert ONOS3Port, "ONOS3Port not defined"
2737 assert ONOS4Port, "ONOS4Port not defined"
2738 assert ONOS5Port, "ONOS5Port not defined"
2739 assert ONOS6Port, "ONOS6Port not defined"
2740 assert ONOS7Port, "ONOS7Port not defined"
2741
2742 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2743 switch = main.params[ 'kill' ][ 'switch' ]
2744 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2745 links = main.params[ 'kill' ][ 'links' ].split()
Jon Halla440e872016-03-31 15:15:50 -07002746 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002747 description = "Adding a switch to ensure it is discovered correctly"
2748 main.case( description )
2749
2750 main.step( "Add back " + switch )
2751 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2752 for peer in links:
2753 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07002754 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002755 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2756 main.log.info( "Waiting " + str( switchSleep ) +
2757 " seconds for switch up to be discovered" )
2758 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002759 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002760 # Peek at the deleted switch
2761 main.log.warn( str( device ) )
2762 result = main.FALSE
2763 if device and device[ 'available' ]:
2764 result = main.TRUE
2765 utilities.assert_equals( expect=main.TRUE, actual=result,
2766 onpass="add switch successful",
2767 onfail="Failed to add switch?" )
2768
2769 def CASE13( self, main ):
2770 """
2771 Clean up
2772 """
2773 import os
2774 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002775 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002776 assert main, "main not defined"
2777 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002778 assert main.CLIs, "main.CLIs not defined"
2779 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002780
2781 # printing colors to terminal
2782 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2783 'blue': '\033[94m', 'green': '\033[92m',
2784 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2785 main.case( "Test Cleanup" )
2786 main.step( "Killing tcpdumps" )
2787 main.Mininet2.stopTcpdump()
2788
2789 testname = main.TEST
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002790 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002791 main.step( "Copying MN pcap and ONOS log files to test station" )
2792 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2793 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002794 # NOTE: MN Pcap file is being saved to logdir.
2795 # We scp this file as MN and TestON aren't necessarily the same vm
2796
2797 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002798 # TODO: Load these from params
2799 # NOTE: must end in /
2800 logFolder = "/opt/onos/log/"
2801 logFiles = [ "karaf.log", "karaf.log.1" ]
2802 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002803 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002804 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002805 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002806 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2807 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002808 # std*.log's
2809 # NOTE: must end in /
2810 logFolder = "/opt/onos/var/"
2811 logFiles = [ "stderr.log", "stdout.log" ]
2812 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002813 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002814 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002815 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002816 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2817 logFolder + f, dstName )
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002818 else:
2819 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002820
2821 main.step( "Stopping Mininet" )
2822 mnResult = main.Mininet1.stopNet()
2823 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2824 onpass="Mininet stopped",
2825 onfail="MN cleanup NOT successful" )
2826
2827 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002828 for node in main.nodes:
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002829 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2830 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002831
2832 try:
2833 timerLog = open( main.logdir + "/Timers.csv", 'w')
2834 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2835 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2836 timerLog.close()
2837 except NameError, e:
2838 main.log.exception(e)
2839
2840 def CASE14( self, main ):
2841 """
2842 start election app on all onos nodes
2843 """
Jon Halle1a3b752015-07-22 13:02:46 -07002844 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002845 assert main, "main not defined"
2846 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002847 assert main.CLIs, "main.CLIs not defined"
2848 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002849
2850 main.case("Start Leadership Election app")
2851 main.step( "Install leadership election app" )
Jon Halla440e872016-03-31 15:15:50 -07002852 onosCli = main.CLIs[ main.activeNodes[0] ]
2853 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002854 utilities.assert_equals(
2855 expect=main.TRUE,
2856 actual=appResult,
2857 onpass="Election app installed",
2858 onfail="Something went wrong with installing Leadership election" )
2859
2860 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002861 for i in main.activeNodes:
2862 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002863 time.sleep(5)
2864 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2865 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002866 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002867 expect=True,
2868 actual=sameResult,
2869 onpass="All nodes see the same leaderboards",
2870 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002871
Jon Hall25463a82016-04-13 14:03:52 -07002872 if sameResult:
2873 leader = leaders[ 0 ][ 0 ]
2874 if main.nodes[main.activeNodes[0]].ip_address in leader:
2875 correctLeader = True
2876 else:
2877 correctLeader = False
2878 main.step( "First node was elected leader" )
2879 utilities.assert_equals(
2880 expect=True,
2881 actual=correctLeader,
2882 onpass="Correct leader was elected",
2883 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002884
2885 def CASE15( self, main ):
2886 """
2887 Check that Leadership Election is still functional
acsmars9475b1c2015-08-28 18:02:08 -07002888 15.1 Run election on each node
2889 15.2 Check that each node has the same leaders and candidates
2890 15.3 Find current leader and withdraw
2891 15.4 Check that a new node was elected leader
2892 15.5 Check that that new leader was the candidate of old leader
2893 15.6 Run for election on old leader
2894 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2895 15.8 Make sure that the old leader was added to the candidate list
2896
2897 old and new variable prefixes refer to data from before vs after
2898 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002899 """
2900 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002901 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002902 assert main, "main not defined"
2903 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002904 assert main.CLIs, "main.CLIs not defined"
2905 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002906
Jon Hall5cf14d52015-07-16 12:15:19 -07002907 description = "Check that Leadership Election is still functional"
2908 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002909 # NOTE: Need to re-run after restarts since being a canidate is not persistant
acsmars9475b1c2015-08-28 18:02:08 -07002910
Jon Halla440e872016-03-31 15:15:50 -07002911 oldLeaders = [] # list of lists of each nodes' candidates before
2912 newLeaders = [] # list of lists of each nodes' candidates after
acsmars9475b1c2015-08-28 18:02:08 -07002913 oldLeader = '' # the old leader from oldLeaders, None if not same
2914 newLeader = '' # the new leaders fron newLoeaders, None if not same
2915 oldLeaderCLI = None # the CLI of the old leader used for re-electing
acsmars71adceb2015-08-31 15:09:26 -07002916 expectNoLeader = False # True when there is only one leader
2917 if main.numCtrls == 1:
2918 expectNoLeader = True
acsmars9475b1c2015-08-28 18:02:08 -07002919
Jon Hall5cf14d52015-07-16 12:15:19 -07002920 main.step( "Run for election on each node" )
acsmars9475b1c2015-08-28 18:02:08 -07002921 electionResult = main.TRUE
2922
Jon Halla440e872016-03-31 15:15:50 -07002923 for i in main.activeNodes: # run test election on each node
2924 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars9475b1c2015-08-28 18:02:08 -07002925 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002926 utilities.assert_equals(
2927 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002928 actual=electionResult,
2929 onpass="All nodes successfully ran for leadership",
2930 onfail="At least one node failed to run for leadership" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002931
acsmars3a72bde2015-09-02 14:16:22 -07002932 if electionResult == main.FALSE:
2933 main.log.error(
2934 "Skipping Test Case because Election Test App isn't loaded" )
2935 main.skipCase()
2936
acsmars9475b1c2015-08-28 18:02:08 -07002937 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002938 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002939 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002940 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002941 if sameResult:
2942 oldLeader = oldLeaders[ 0 ][ 0 ]
2943 main.log.warn( oldLeader )
acsmars9475b1c2015-08-28 18:02:08 -07002944 else:
Jon Halla440e872016-03-31 15:15:50 -07002945 oldLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002946 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002947 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002948 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002949 onpass="Leaderboards are consistent for the election topic",
acsmars9475b1c2015-08-28 18:02:08 -07002950 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002951
2952 main.step( "Find current leader and withdraw" )
acsmars9475b1c2015-08-28 18:02:08 -07002953 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002954 # do some sanity checking on leader before using it
acsmars9475b1c2015-08-28 18:02:08 -07002955 if oldLeader is None:
2956 main.log.error( "Leadership isn't consistent." )
2957 withdrawResult = main.FALSE
2958 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002959 for i in main.activeNodes:
acsmars9475b1c2015-08-28 18:02:08 -07002960 if oldLeader == main.nodes[ i ].ip_address:
2961 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002962 break
2963 else: # FOR/ELSE statement
2964 main.log.error( "Leader election, could not find current leader" )
2965 if oldLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002966 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002967 utilities.assert_equals(
2968 expect=main.TRUE,
2969 actual=withdrawResult,
2970 onpass="Node was withdrawn from election",
2971 onfail="Node was not withdrawn from election" )
2972
acsmars9475b1c2015-08-28 18:02:08 -07002973 main.step( "Check that a new node was elected leader" )
acsmars9475b1c2015-08-28 18:02:08 -07002974 failMessage = "Nodes have different leaders"
acsmars9475b1c2015-08-28 18:02:08 -07002975 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002976 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002977 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002978 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002979 if newLeaders[ 0 ][ 0 ] == 'none':
2980 main.log.error( "No leader was elected on at least 1 node" )
2981 if not expectNoLeader:
2982 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002983 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002984
acsmars9475b1c2015-08-28 18:02:08 -07002985 # Check that the new leader is not the older leader, which was withdrawn
2986 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002987 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08002988 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars9475b1c2015-08-28 18:02:08 -07002989 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002990 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002991 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07002992 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002993 onpass="Leadership election passed",
2994 onfail="Something went wrong with Leadership election" )
2995
Jon Halla440e872016-03-31 15:15:50 -07002996 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08002997 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars9475b1c2015-08-28 18:02:08 -07002998 correctCandidateResult = main.TRUE
acsmars71adceb2015-08-31 15:09:26 -07002999 if expectNoLeader:
3000 if newLeader == 'none':
3001 main.log.info( "No leader expected. None found. Pass" )
3002 correctCandidateResult = main.TRUE
3003 else:
3004 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3005 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003006 elif len( oldLeaders[0] ) >= 3:
3007 if newLeader == oldLeaders[ 0 ][ 2 ]:
3008 # correct leader was elected
3009 correctCandidateResult = main.TRUE
3010 else:
3011 correctCandidateResult = main.FALSE
3012 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3013 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003014 else:
3015 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003016 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003017 correctCandidateResult = main.FALSE
acsmars9475b1c2015-08-28 18:02:08 -07003018 utilities.assert_equals(
3019 expect=main.TRUE,
3020 actual=correctCandidateResult,
3021 onpass="Correct Candidate Elected",
3022 onfail="Incorrect Candidate Elected" )
3023
Jon Hall5cf14d52015-07-16 12:15:19 -07003024 main.step( "Run for election on old leader( just so everyone " +
3025 "is in the hat )" )
acsmars9475b1c2015-08-28 18:02:08 -07003026 if oldLeaderCLI is not None:
3027 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003028 else:
acsmars9475b1c2015-08-28 18:02:08 -07003029 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003030 runResult = main.FALSE
3031 utilities.assert_equals(
3032 expect=main.TRUE,
3033 actual=runResult,
3034 onpass="App re-ran for election",
3035 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003036
acsmars9475b1c2015-08-28 18:02:08 -07003037 main.step(
3038 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003039 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003040 # Get new leaders and candidates
3041 reRunLeaders = []
3042 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003043 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003044
acsmars9475b1c2015-08-28 18:02:08 -07003045 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07003046 if not reRunLeaders[0]:
3047 positionResult = main.FALSE
3048 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07003049 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3050 str( reRunLeaders[ 0 ] ) ) )
acsmars9475b1c2015-08-28 18:02:08 -07003051 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003052 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003053 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07003054 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003055 onpass="Old leader successfully re-ran for election",
3056 onfail="Something went wrong with Leadership election after " +
3057 "the old leader re-ran for election" )
3058
3059 def CASE16( self, main ):
3060 """
3061 Install Distributed Primitives app
3062 """
3063 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003064 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003065 assert main, "main not defined"
3066 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003067 assert main.CLIs, "main.CLIs not defined"
3068 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003069
3070 # Variables for the distributed primitives tests
3071 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003072 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003073 global onosSet
3074 global onosSetName
3075 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003076 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003077 onosSet = set([])
3078 onosSetName = "TestON-set"
3079
3080 description = "Install Primitives app"
3081 main.case( description )
3082 main.step( "Install Primitives app" )
3083 appName = "org.onosproject.distributedprimitives"
Jon Halla440e872016-03-31 15:15:50 -07003084 node = main.activeNodes[0]
3085 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003086 utilities.assert_equals( expect=main.TRUE,
3087 actual=appResults,
3088 onpass="Primitives app activated",
3089 onfail="Primitives app not activated" )
3090 time.sleep( 5 ) # To allow all nodes to activate
3091
3092 def CASE17( self, main ):
3093 """
3094 Check for basic functionality with distributed primitives
3095 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003096 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003097 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003098 assert main, "main not defined"
3099 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003100 assert main.CLIs, "main.CLIs not defined"
3101 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003102 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003103 assert onosSetName, "onosSetName not defined"
3104 # NOTE: assert fails if value is 0/None/Empty/False
3105 try:
3106 pCounterValue
3107 except NameError:
3108 main.log.error( "pCounterValue not defined, setting to 0" )
3109 pCounterValue = 0
3110 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003111 onosSet
3112 except NameError:
3113 main.log.error( "onosSet not defined, setting to empty Set" )
3114 onosSet = set([])
3115 # Variables for the distributed primitives tests. These are local only
3116 addValue = "a"
3117 addAllValue = "a b c d e f"
3118 retainValue = "c d e f"
3119
3120 description = "Check for basic functionality with distributed " +\
3121 "primitives"
3122 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003123 main.caseExplanation = "Test the methods of the distributed " +\
3124 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003125 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003126 # Partitioned counters
3127 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003128 pCounters = []
3129 threads = []
3130 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003131 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003132 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3133 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003134 args=[ pCounterName ] )
3135 pCounterValue += 1
3136 addedPValues.append( pCounterValue )
3137 threads.append( t )
3138 t.start()
3139
3140 for t in threads:
3141 t.join()
3142 pCounters.append( t.result )
3143 # Check that counter incremented numController times
3144 pCounterResults = True
3145 for i in addedPValues:
3146 tmpResult = i in pCounters
3147 pCounterResults = pCounterResults and tmpResult
3148 if not tmpResult:
3149 main.log.error( str( i ) + " is not in partitioned "
3150 "counter incremented results" )
3151 utilities.assert_equals( expect=True,
3152 actual=pCounterResults,
3153 onpass="Default counter incremented",
3154 onfail="Error incrementing default" +
3155 " counter" )
3156
Jon Halle1a3b752015-07-22 13:02:46 -07003157 main.step( "Get then Increment a default counter on each node" )
3158 pCounters = []
3159 threads = []
3160 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003161 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003162 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3163 name="counterGetAndAdd-" + str( i ),
3164 args=[ pCounterName ] )
3165 addedPValues.append( pCounterValue )
3166 pCounterValue += 1
3167 threads.append( t )
3168 t.start()
3169
3170 for t in threads:
3171 t.join()
3172 pCounters.append( t.result )
3173 # Check that counter incremented numController times
3174 pCounterResults = True
3175 for i in addedPValues:
3176 tmpResult = i in pCounters
3177 pCounterResults = pCounterResults and tmpResult
3178 if not tmpResult:
3179 main.log.error( str( i ) + " is not in partitioned "
3180 "counter incremented results" )
3181 utilities.assert_equals( expect=True,
3182 actual=pCounterResults,
3183 onpass="Default counter incremented",
3184 onfail="Error incrementing default" +
3185 " counter" )
3186
3187 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003188 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003189 utilities.assert_equals( expect=main.TRUE,
3190 actual=incrementCheck,
3191 onpass="Added counters are correct",
3192 onfail="Added counters are incorrect" )
3193
3194 main.step( "Add -8 to then get a default counter on each node" )
3195 pCounters = []
3196 threads = []
3197 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003198 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003199 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3200 name="counterIncrement-" + str( i ),
3201 args=[ pCounterName ],
3202 kwargs={ "delta": -8 } )
3203 pCounterValue += -8
3204 addedPValues.append( pCounterValue )
3205 threads.append( t )
3206 t.start()
3207
3208 for t in threads:
3209 t.join()
3210 pCounters.append( t.result )
3211 # Check that counter incremented numController times
3212 pCounterResults = True
3213 for i in addedPValues:
3214 tmpResult = i in pCounters
3215 pCounterResults = pCounterResults and tmpResult
3216 if not tmpResult:
3217 main.log.error( str( i ) + " is not in partitioned "
3218 "counter incremented results" )
3219 utilities.assert_equals( expect=True,
3220 actual=pCounterResults,
3221 onpass="Default counter incremented",
3222 onfail="Error incrementing default" +
3223 " counter" )
3224
3225 main.step( "Add 5 to then get a default counter on each node" )
3226 pCounters = []
3227 threads = []
3228 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003229 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003230 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3231 name="counterIncrement-" + str( i ),
3232 args=[ pCounterName ],
3233 kwargs={ "delta": 5 } )
3234 pCounterValue += 5
3235 addedPValues.append( pCounterValue )
3236 threads.append( t )
3237 t.start()
3238
3239 for t in threads:
3240 t.join()
3241 pCounters.append( t.result )
3242 # Check that counter incremented numController times
3243 pCounterResults = True
3244 for i in addedPValues:
3245 tmpResult = i in pCounters
3246 pCounterResults = pCounterResults and tmpResult
3247 if not tmpResult:
3248 main.log.error( str( i ) + " is not in partitioned "
3249 "counter incremented results" )
3250 utilities.assert_equals( expect=True,
3251 actual=pCounterResults,
3252 onpass="Default counter incremented",
3253 onfail="Error incrementing default" +
3254 " counter" )
3255
3256 main.step( "Get then add 5 to a default counter on each node" )
3257 pCounters = []
3258 threads = []
3259 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003260 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003261 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3262 name="counterIncrement-" + str( i ),
3263 args=[ pCounterName ],
3264 kwargs={ "delta": 5 } )
3265 addedPValues.append( pCounterValue )
3266 pCounterValue += 5
3267 threads.append( t )
3268 t.start()
3269
3270 for t in threads:
3271 t.join()
3272 pCounters.append( t.result )
3273 # Check that counter incremented numController times
3274 pCounterResults = True
3275 for i in addedPValues:
3276 tmpResult = i in pCounters
3277 pCounterResults = pCounterResults and tmpResult
3278 if not tmpResult:
3279 main.log.error( str( i ) + " is not in partitioned "
3280 "counter incremented results" )
3281 utilities.assert_equals( expect=True,
3282 actual=pCounterResults,
3283 onpass="Default counter incremented",
3284 onfail="Error incrementing default" +
3285 " counter" )
3286
3287 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003288 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003289 utilities.assert_equals( expect=main.TRUE,
3290 actual=incrementCheck,
3291 onpass="Added counters are correct",
3292 onfail="Added counters are incorrect" )
3293
Jon Hall5cf14d52015-07-16 12:15:19 -07003294 # DISTRIBUTED SETS
3295 main.step( "Distributed Set get" )
3296 size = len( onosSet )
3297 getResponses = []
3298 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003299 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003300 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003301 name="setTestGet-" + str( i ),
3302 args=[ onosSetName ] )
3303 threads.append( t )
3304 t.start()
3305 for t in threads:
3306 t.join()
3307 getResponses.append( t.result )
3308
3309 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003310 for i in range( len( main.activeNodes ) ):
3311 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003312 if isinstance( getResponses[ i ], list):
3313 current = set( getResponses[ i ] )
3314 if len( current ) == len( getResponses[ i ] ):
3315 # no repeats
3316 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003317 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003318 " has incorrect view" +
3319 " of set " + onosSetName + ":\n" +
3320 str( getResponses[ i ] ) )
3321 main.log.debug( "Expected: " + str( onosSet ) )
3322 main.log.debug( "Actual: " + str( current ) )
3323 getResults = main.FALSE
3324 else:
3325 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003326 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003327 " has repeat elements in" +
3328 " set " + onosSetName + ":\n" +
3329 str( getResponses[ i ] ) )
3330 getResults = main.FALSE
3331 elif getResponses[ i ] == main.ERROR:
3332 getResults = main.FALSE
3333 utilities.assert_equals( expect=main.TRUE,
3334 actual=getResults,
3335 onpass="Set elements are correct",
3336 onfail="Set elements are incorrect" )
3337
3338 main.step( "Distributed Set size" )
3339 sizeResponses = []
3340 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003341 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003342 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003343 name="setTestSize-" + str( i ),
3344 args=[ onosSetName ] )
3345 threads.append( t )
3346 t.start()
3347 for t in threads:
3348 t.join()
3349 sizeResponses.append( t.result )
3350
3351 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003352 for i in range( len( main.activeNodes ) ):
3353 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003354 if size != sizeResponses[ i ]:
3355 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003356 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003357 " expected a size of " + str( size ) +
3358 " for set " + onosSetName +
3359 " but got " + str( sizeResponses[ i ] ) )
3360 utilities.assert_equals( expect=main.TRUE,
3361 actual=sizeResults,
3362 onpass="Set sizes are correct",
3363 onfail="Set sizes are incorrect" )
3364
3365 main.step( "Distributed Set add()" )
3366 onosSet.add( addValue )
3367 addResponses = []
3368 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003369 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003370 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003371 name="setTestAdd-" + str( i ),
3372 args=[ onosSetName, addValue ] )
3373 threads.append( t )
3374 t.start()
3375 for t in threads:
3376 t.join()
3377 addResponses.append( t.result )
3378
3379 # main.TRUE = successfully changed the set
3380 # main.FALSE = action resulted in no change in set
3381 # main.ERROR - Some error in executing the function
3382 addResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003383 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003384 if addResponses[ i ] == main.TRUE:
3385 # All is well
3386 pass
3387 elif addResponses[ i ] == main.FALSE:
3388 # Already in set, probably fine
3389 pass
3390 elif addResponses[ i ] == main.ERROR:
3391 # Error in execution
3392 addResults = main.FALSE
3393 else:
3394 # unexpected result
3395 addResults = main.FALSE
3396 if addResults != main.TRUE:
3397 main.log.error( "Error executing set add" )
3398
3399 # Check if set is still correct
3400 size = len( onosSet )
3401 getResponses = []
3402 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003403 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003404 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003405 name="setTestGet-" + str( i ),
3406 args=[ onosSetName ] )
3407 threads.append( t )
3408 t.start()
3409 for t in threads:
3410 t.join()
3411 getResponses.append( t.result )
3412 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003413 for i in range( len( main.activeNodes ) ):
3414 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003415 if isinstance( getResponses[ i ], list):
3416 current = set( getResponses[ i ] )
3417 if len( current ) == len( getResponses[ i ] ):
3418 # no repeats
3419 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003420 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003421 " of set " + onosSetName + ":\n" +
3422 str( getResponses[ i ] ) )
3423 main.log.debug( "Expected: " + str( onosSet ) )
3424 main.log.debug( "Actual: " + str( current ) )
3425 getResults = main.FALSE
3426 else:
3427 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003428 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003429 " set " + onosSetName + ":\n" +
3430 str( getResponses[ i ] ) )
3431 getResults = main.FALSE
3432 elif getResponses[ i ] == main.ERROR:
3433 getResults = main.FALSE
3434 sizeResponses = []
3435 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003436 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003437 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003438 name="setTestSize-" + str( i ),
3439 args=[ onosSetName ] )
3440 threads.append( t )
3441 t.start()
3442 for t in threads:
3443 t.join()
3444 sizeResponses.append( t.result )
3445 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003446 for i in range( len( main.activeNodes ) ):
3447 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003448 if size != sizeResponses[ i ]:
3449 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003450 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003451 " expected a size of " + str( size ) +
3452 " for set " + onosSetName +
3453 " but got " + str( sizeResponses[ i ] ) )
3454 addResults = addResults and getResults and sizeResults
3455 utilities.assert_equals( expect=main.TRUE,
3456 actual=addResults,
3457 onpass="Set add correct",
3458 onfail="Set add was incorrect" )
3459
3460 main.step( "Distributed Set addAll()" )
3461 onosSet.update( addAllValue.split() )
3462 addResponses = []
3463 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003464 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003465 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003466 name="setTestAddAll-" + str( i ),
3467 args=[ onosSetName, addAllValue ] )
3468 threads.append( t )
3469 t.start()
3470 for t in threads:
3471 t.join()
3472 addResponses.append( t.result )
3473
3474 # main.TRUE = successfully changed the set
3475 # main.FALSE = action resulted in no change in set
3476 # main.ERROR - Some error in executing the function
3477 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003478 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003479 if addResponses[ i ] == main.TRUE:
3480 # All is well
3481 pass
3482 elif addResponses[ i ] == main.FALSE:
3483 # Already in set, probably fine
3484 pass
3485 elif addResponses[ i ] == main.ERROR:
3486 # Error in execution
3487 addAllResults = main.FALSE
3488 else:
3489 # unexpected result
3490 addAllResults = main.FALSE
3491 if addAllResults != main.TRUE:
3492 main.log.error( "Error executing set addAll" )
3493
3494 # Check if set is still correct
3495 size = len( onosSet )
3496 getResponses = []
3497 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003498 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003499 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003500 name="setTestGet-" + str( i ),
3501 args=[ onosSetName ] )
3502 threads.append( t )
3503 t.start()
3504 for t in threads:
3505 t.join()
3506 getResponses.append( t.result )
3507 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003508 for i in range( len( main.activeNodes ) ):
3509 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003510 if isinstance( getResponses[ i ], list):
3511 current = set( getResponses[ i ] )
3512 if len( current ) == len( getResponses[ i ] ):
3513 # no repeats
3514 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003515 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003516 " has incorrect view" +
3517 " of set " + onosSetName + ":\n" +
3518 str( getResponses[ i ] ) )
3519 main.log.debug( "Expected: " + str( onosSet ) )
3520 main.log.debug( "Actual: " + str( current ) )
3521 getResults = main.FALSE
3522 else:
3523 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003524 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003525 " has repeat elements in" +
3526 " set " + onosSetName + ":\n" +
3527 str( getResponses[ i ] ) )
3528 getResults = main.FALSE
3529 elif getResponses[ i ] == main.ERROR:
3530 getResults = main.FALSE
3531 sizeResponses = []
3532 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003533 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003534 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003535 name="setTestSize-" + str( i ),
3536 args=[ onosSetName ] )
3537 threads.append( t )
3538 t.start()
3539 for t in threads:
3540 t.join()
3541 sizeResponses.append( t.result )
3542 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003543 for i in range( len( main.activeNodes ) ):
3544 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003545 if size != sizeResponses[ i ]:
3546 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003547 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003548 " expected a size of " + str( size ) +
3549 " for set " + onosSetName +
3550 " but got " + str( sizeResponses[ i ] ) )
3551 addAllResults = addAllResults and getResults and sizeResults
3552 utilities.assert_equals( expect=main.TRUE,
3553 actual=addAllResults,
3554 onpass="Set addAll correct",
3555 onfail="Set addAll was incorrect" )
3556
3557 main.step( "Distributed Set contains()" )
3558 containsResponses = []
3559 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003560 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003561 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003562 name="setContains-" + str( i ),
3563 args=[ onosSetName ],
3564 kwargs={ "values": addValue } )
3565 threads.append( t )
3566 t.start()
3567 for t in threads:
3568 t.join()
3569 # NOTE: This is the tuple
3570 containsResponses.append( t.result )
3571
3572 containsResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003573 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003574 if containsResponses[ i ] == main.ERROR:
3575 containsResults = main.FALSE
3576 else:
3577 containsResults = containsResults and\
3578 containsResponses[ i ][ 1 ]
3579 utilities.assert_equals( expect=main.TRUE,
3580 actual=containsResults,
3581 onpass="Set contains is functional",
3582 onfail="Set contains failed" )
3583
3584 main.step( "Distributed Set containsAll()" )
3585 containsAllResponses = []
3586 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003587 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003588 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003589 name="setContainsAll-" + str( i ),
3590 args=[ onosSetName ],
3591 kwargs={ "values": addAllValue } )
3592 threads.append( t )
3593 t.start()
3594 for t in threads:
3595 t.join()
3596 # NOTE: This is the tuple
3597 containsAllResponses.append( t.result )
3598
3599 containsAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003600 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003601 if containsResponses[ i ] == main.ERROR:
3602 containsResults = main.FALSE
3603 else:
3604 containsResults = containsResults and\
3605 containsResponses[ i ][ 1 ]
3606 utilities.assert_equals( expect=main.TRUE,
3607 actual=containsAllResults,
3608 onpass="Set containsAll is functional",
3609 onfail="Set containsAll failed" )
3610
3611 main.step( "Distributed Set remove()" )
3612 onosSet.remove( addValue )
3613 removeResponses = []
3614 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003615 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003616 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003617 name="setTestRemove-" + str( i ),
3618 args=[ onosSetName, addValue ] )
3619 threads.append( t )
3620 t.start()
3621 for t in threads:
3622 t.join()
3623 removeResponses.append( t.result )
3624
3625 # main.TRUE = successfully changed the set
3626 # main.FALSE = action resulted in no change in set
3627 # main.ERROR - Some error in executing the function
3628 removeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003629 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003630 if removeResponses[ i ] == main.TRUE:
3631 # All is well
3632 pass
3633 elif removeResponses[ i ] == main.FALSE:
3634 # not in set, probably fine
3635 pass
3636 elif removeResponses[ i ] == main.ERROR:
3637 # Error in execution
3638 removeResults = main.FALSE
3639 else:
3640 # unexpected result
3641 removeResults = main.FALSE
3642 if removeResults != main.TRUE:
3643 main.log.error( "Error executing set remove" )
3644
3645 # Check if set is still correct
3646 size = len( onosSet )
3647 getResponses = []
3648 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003649 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003650 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003651 name="setTestGet-" + str( i ),
3652 args=[ onosSetName ] )
3653 threads.append( t )
3654 t.start()
3655 for t in threads:
3656 t.join()
3657 getResponses.append( t.result )
3658 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003659 for i in range( len( main.activeNodes ) ):
3660 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003661 if isinstance( getResponses[ i ], list):
3662 current = set( getResponses[ i ] )
3663 if len( current ) == len( getResponses[ i ] ):
3664 # no repeats
3665 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003666 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003667 " has incorrect view" +
3668 " of set " + onosSetName + ":\n" +
3669 str( getResponses[ i ] ) )
3670 main.log.debug( "Expected: " + str( onosSet ) )
3671 main.log.debug( "Actual: " + str( current ) )
3672 getResults = main.FALSE
3673 else:
3674 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003675 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003676 " has repeat elements in" +
3677 " set " + onosSetName + ":\n" +
3678 str( getResponses[ i ] ) )
3679 getResults = main.FALSE
3680 elif getResponses[ i ] == main.ERROR:
3681 getResults = main.FALSE
3682 sizeResponses = []
3683 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003684 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003685 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003686 name="setTestSize-" + str( i ),
3687 args=[ onosSetName ] )
3688 threads.append( t )
3689 t.start()
3690 for t in threads:
3691 t.join()
3692 sizeResponses.append( t.result )
3693 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003694 for i in range( len( main.activeNodes ) ):
3695 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003696 if size != sizeResponses[ i ]:
3697 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003698 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003699 " expected a size of " + str( size ) +
3700 " for set " + onosSetName +
3701 " but got " + str( sizeResponses[ i ] ) )
3702 removeResults = removeResults and getResults and sizeResults
3703 utilities.assert_equals( expect=main.TRUE,
3704 actual=removeResults,
3705 onpass="Set remove correct",
3706 onfail="Set remove was incorrect" )
3707
3708 main.step( "Distributed Set removeAll()" )
3709 onosSet.difference_update( addAllValue.split() )
3710 removeAllResponses = []
3711 threads = []
3712 try:
Jon Halla440e872016-03-31 15:15:50 -07003713 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003714 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003715 name="setTestRemoveAll-" + str( i ),
3716 args=[ onosSetName, addAllValue ] )
3717 threads.append( t )
3718 t.start()
3719 for t in threads:
3720 t.join()
3721 removeAllResponses.append( t.result )
3722 except Exception, e:
3723 main.log.exception(e)
3724
3725 # main.TRUE = successfully changed the set
3726 # main.FALSE = action resulted in no change in set
3727 # main.ERROR - Some error in executing the function
3728 removeAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003729 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003730 if removeAllResponses[ i ] == main.TRUE:
3731 # All is well
3732 pass
3733 elif removeAllResponses[ i ] == main.FALSE:
3734 # not in set, probably fine
3735 pass
3736 elif removeAllResponses[ i ] == main.ERROR:
3737 # Error in execution
3738 removeAllResults = main.FALSE
3739 else:
3740 # unexpected result
3741 removeAllResults = main.FALSE
3742 if removeAllResults != main.TRUE:
3743 main.log.error( "Error executing set removeAll" )
3744
3745 # Check if set is still correct
3746 size = len( onosSet )
3747 getResponses = []
3748 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003749 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003750 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003751 name="setTestGet-" + str( i ),
3752 args=[ onosSetName ] )
3753 threads.append( t )
3754 t.start()
3755 for t in threads:
3756 t.join()
3757 getResponses.append( t.result )
3758 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003759 for i in range( len( main.activeNodes ) ):
3760 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003761 if isinstance( getResponses[ i ], list):
3762 current = set( getResponses[ i ] )
3763 if len( current ) == len( getResponses[ i ] ):
3764 # no repeats
3765 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003766 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003767 " has incorrect view" +
3768 " of set " + onosSetName + ":\n" +
3769 str( getResponses[ i ] ) )
3770 main.log.debug( "Expected: " + str( onosSet ) )
3771 main.log.debug( "Actual: " + str( current ) )
3772 getResults = main.FALSE
3773 else:
3774 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003775 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003776 " has repeat elements in" +
3777 " set " + onosSetName + ":\n" +
3778 str( getResponses[ i ] ) )
3779 getResults = main.FALSE
3780 elif getResponses[ i ] == main.ERROR:
3781 getResults = main.FALSE
3782 sizeResponses = []
3783 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003784 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003785 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003786 name="setTestSize-" + str( i ),
3787 args=[ onosSetName ] )
3788 threads.append( t )
3789 t.start()
3790 for t in threads:
3791 t.join()
3792 sizeResponses.append( t.result )
3793 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003794 for i in range( len( main.activeNodes ) ):
3795 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003796 if size != sizeResponses[ i ]:
3797 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003798 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003799 " expected a size of " + str( size ) +
3800 " for set " + onosSetName +
3801 " but got " + str( sizeResponses[ i ] ) )
3802 removeAllResults = removeAllResults and getResults and sizeResults
3803 utilities.assert_equals( expect=main.TRUE,
3804 actual=removeAllResults,
3805 onpass="Set removeAll correct",
3806 onfail="Set removeAll was incorrect" )
3807
3808 main.step( "Distributed Set addAll()" )
3809 onosSet.update( addAllValue.split() )
3810 addResponses = []
3811 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003812 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003813 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003814 name="setTestAddAll-" + str( i ),
3815 args=[ onosSetName, addAllValue ] )
3816 threads.append( t )
3817 t.start()
3818 for t in threads:
3819 t.join()
3820 addResponses.append( t.result )
3821
3822 # main.TRUE = successfully changed the set
3823 # main.FALSE = action resulted in no change in set
3824 # main.ERROR - Some error in executing the function
3825 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003826 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003827 if addResponses[ i ] == main.TRUE:
3828 # All is well
3829 pass
3830 elif addResponses[ i ] == main.FALSE:
3831 # Already in set, probably fine
3832 pass
3833 elif addResponses[ i ] == main.ERROR:
3834 # Error in execution
3835 addAllResults = main.FALSE
3836 else:
3837 # unexpected result
3838 addAllResults = main.FALSE
3839 if addAllResults != main.TRUE:
3840 main.log.error( "Error executing set addAll" )
3841
3842 # Check if set is still correct
3843 size = len( onosSet )
3844 getResponses = []
3845 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003846 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003847 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003848 name="setTestGet-" + str( i ),
3849 args=[ onosSetName ] )
3850 threads.append( t )
3851 t.start()
3852 for t in threads:
3853 t.join()
3854 getResponses.append( t.result )
3855 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003856 for i in range( len( main.activeNodes ) ):
3857 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003858 if isinstance( getResponses[ i ], list):
3859 current = set( getResponses[ i ] )
3860 if len( current ) == len( getResponses[ i ] ):
3861 # no repeats
3862 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003863 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003864 " has incorrect view" +
3865 " of set " + onosSetName + ":\n" +
3866 str( getResponses[ i ] ) )
3867 main.log.debug( "Expected: " + str( onosSet ) )
3868 main.log.debug( "Actual: " + str( current ) )
3869 getResults = main.FALSE
3870 else:
3871 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003872 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003873 " has repeat elements in" +
3874 " set " + onosSetName + ":\n" +
3875 str( getResponses[ i ] ) )
3876 getResults = main.FALSE
3877 elif getResponses[ i ] == main.ERROR:
3878 getResults = main.FALSE
3879 sizeResponses = []
3880 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003881 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003882 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003883 name="setTestSize-" + str( i ),
3884 args=[ onosSetName ] )
3885 threads.append( t )
3886 t.start()
3887 for t in threads:
3888 t.join()
3889 sizeResponses.append( t.result )
3890 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003891 for i in range( len( main.activeNodes ) ):
3892 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003893 if size != sizeResponses[ i ]:
3894 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003895 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003896 " expected a size of " + str( size ) +
3897 " for set " + onosSetName +
3898 " but got " + str( sizeResponses[ i ] ) )
3899 addAllResults = addAllResults and getResults and sizeResults
3900 utilities.assert_equals( expect=main.TRUE,
3901 actual=addAllResults,
3902 onpass="Set addAll correct",
3903 onfail="Set addAll was incorrect" )
3904
3905 main.step( "Distributed Set clear()" )
3906 onosSet.clear()
3907 clearResponses = []
3908 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003909 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003910 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003911 name="setTestClear-" + str( i ),
3912 args=[ onosSetName, " "], # Values doesn't matter
3913 kwargs={ "clear": True } )
3914 threads.append( t )
3915 t.start()
3916 for t in threads:
3917 t.join()
3918 clearResponses.append( t.result )
3919
3920 # main.TRUE = successfully changed the set
3921 # main.FALSE = action resulted in no change in set
3922 # main.ERROR - Some error in executing the function
3923 clearResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003924 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003925 if clearResponses[ i ] == main.TRUE:
3926 # All is well
3927 pass
3928 elif clearResponses[ i ] == main.FALSE:
3929 # Nothing set, probably fine
3930 pass
3931 elif clearResponses[ i ] == main.ERROR:
3932 # Error in execution
3933 clearResults = main.FALSE
3934 else:
3935 # unexpected result
3936 clearResults = main.FALSE
3937 if clearResults != main.TRUE:
3938 main.log.error( "Error executing set clear" )
3939
3940 # Check if set is still correct
3941 size = len( onosSet )
3942 getResponses = []
3943 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003944 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003945 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003946 name="setTestGet-" + str( i ),
3947 args=[ onosSetName ] )
3948 threads.append( t )
3949 t.start()
3950 for t in threads:
3951 t.join()
3952 getResponses.append( t.result )
3953 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003954 for i in range( len( main.activeNodes ) ):
3955 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003956 if isinstance( getResponses[ i ], list):
3957 current = set( getResponses[ i ] )
3958 if len( current ) == len( getResponses[ i ] ):
3959 # no repeats
3960 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003961 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003962 " has incorrect view" +
3963 " of set " + onosSetName + ":\n" +
3964 str( getResponses[ i ] ) )
3965 main.log.debug( "Expected: " + str( onosSet ) )
3966 main.log.debug( "Actual: " + str( current ) )
3967 getResults = main.FALSE
3968 else:
3969 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003970 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003971 " has repeat elements in" +
3972 " set " + onosSetName + ":\n" +
3973 str( getResponses[ i ] ) )
3974 getResults = main.FALSE
3975 elif getResponses[ i ] == main.ERROR:
3976 getResults = main.FALSE
3977 sizeResponses = []
3978 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003979 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003980 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003981 name="setTestSize-" + str( i ),
3982 args=[ onosSetName ] )
3983 threads.append( t )
3984 t.start()
3985 for t in threads:
3986 t.join()
3987 sizeResponses.append( t.result )
3988 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003989 for i in range( len( main.activeNodes ) ):
3990 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003991 if size != sizeResponses[ i ]:
3992 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003993 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003994 " expected a size of " + str( size ) +
3995 " for set " + onosSetName +
3996 " but got " + str( sizeResponses[ i ] ) )
3997 clearResults = clearResults and getResults and sizeResults
3998 utilities.assert_equals( expect=main.TRUE,
3999 actual=clearResults,
4000 onpass="Set clear correct",
4001 onfail="Set clear was incorrect" )
4002
4003 main.step( "Distributed Set addAll()" )
4004 onosSet.update( addAllValue.split() )
4005 addResponses = []
4006 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004007 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004008 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004009 name="setTestAddAll-" + str( i ),
4010 args=[ onosSetName, addAllValue ] )
4011 threads.append( t )
4012 t.start()
4013 for t in threads:
4014 t.join()
4015 addResponses.append( t.result )
4016
4017 # main.TRUE = successfully changed the set
4018 # main.FALSE = action resulted in no change in set
4019 # main.ERROR - Some error in executing the function
4020 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004021 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004022 if addResponses[ i ] == main.TRUE:
4023 # All is well
4024 pass
4025 elif addResponses[ i ] == main.FALSE:
4026 # Already in set, probably fine
4027 pass
4028 elif addResponses[ i ] == main.ERROR:
4029 # Error in execution
4030 addAllResults = main.FALSE
4031 else:
4032 # unexpected result
4033 addAllResults = main.FALSE
4034 if addAllResults != main.TRUE:
4035 main.log.error( "Error executing set addAll" )
4036
4037 # Check if set is still correct
4038 size = len( onosSet )
4039 getResponses = []
4040 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004041 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004042 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004043 name="setTestGet-" + str( i ),
4044 args=[ onosSetName ] )
4045 threads.append( t )
4046 t.start()
4047 for t in threads:
4048 t.join()
4049 getResponses.append( t.result )
4050 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004051 for i in range( len( main.activeNodes ) ):
4052 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004053 if isinstance( getResponses[ i ], list):
4054 current = set( getResponses[ i ] )
4055 if len( current ) == len( getResponses[ i ] ):
4056 # no repeats
4057 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004058 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004059 " has incorrect view" +
4060 " of set " + onosSetName + ":\n" +
4061 str( getResponses[ i ] ) )
4062 main.log.debug( "Expected: " + str( onosSet ) )
4063 main.log.debug( "Actual: " + str( current ) )
4064 getResults = main.FALSE
4065 else:
4066 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004067 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004068 " has repeat elements in" +
4069 " set " + onosSetName + ":\n" +
4070 str( getResponses[ i ] ) )
4071 getResults = main.FALSE
4072 elif getResponses[ i ] == main.ERROR:
4073 getResults = main.FALSE
4074 sizeResponses = []
4075 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004076 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004077 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004078 name="setTestSize-" + str( i ),
4079 args=[ onosSetName ] )
4080 threads.append( t )
4081 t.start()
4082 for t in threads:
4083 t.join()
4084 sizeResponses.append( t.result )
4085 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004086 for i in range( len( main.activeNodes ) ):
4087 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004088 if size != sizeResponses[ i ]:
4089 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004090 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004091 " expected a size of " + str( size ) +
4092 " for set " + onosSetName +
4093 " but got " + str( sizeResponses[ i ] ) )
4094 addAllResults = addAllResults and getResults and sizeResults
4095 utilities.assert_equals( expect=main.TRUE,
4096 actual=addAllResults,
4097 onpass="Set addAll correct",
4098 onfail="Set addAll was incorrect" )
4099
4100 main.step( "Distributed Set retain()" )
4101 onosSet.intersection_update( retainValue.split() )
4102 retainResponses = []
4103 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004104 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004105 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004106 name="setTestRetain-" + str( i ),
4107 args=[ onosSetName, retainValue ],
4108 kwargs={ "retain": True } )
4109 threads.append( t )
4110 t.start()
4111 for t in threads:
4112 t.join()
4113 retainResponses.append( t.result )
4114
4115 # main.TRUE = successfully changed the set
4116 # main.FALSE = action resulted in no change in set
4117 # main.ERROR - Some error in executing the function
4118 retainResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004119 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004120 if retainResponses[ i ] == main.TRUE:
4121 # All is well
4122 pass
4123 elif retainResponses[ i ] == main.FALSE:
4124 # Already in set, probably fine
4125 pass
4126 elif retainResponses[ i ] == main.ERROR:
4127 # Error in execution
4128 retainResults = main.FALSE
4129 else:
4130 # unexpected result
4131 retainResults = main.FALSE
4132 if retainResults != main.TRUE:
4133 main.log.error( "Error executing set retain" )
4134
4135 # Check if set is still correct
4136 size = len( onosSet )
4137 getResponses = []
4138 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004139 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004140 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004141 name="setTestGet-" + str( i ),
4142 args=[ onosSetName ] )
4143 threads.append( t )
4144 t.start()
4145 for t in threads:
4146 t.join()
4147 getResponses.append( t.result )
4148 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004149 for i in range( len( main.activeNodes ) ):
4150 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004151 if isinstance( getResponses[ i ], list):
4152 current = set( getResponses[ i ] )
4153 if len( current ) == len( getResponses[ i ] ):
4154 # no repeats
4155 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004156 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004157 " has incorrect view" +
4158 " of set " + onosSetName + ":\n" +
4159 str( getResponses[ i ] ) )
4160 main.log.debug( "Expected: " + str( onosSet ) )
4161 main.log.debug( "Actual: " + str( current ) )
4162 getResults = main.FALSE
4163 else:
4164 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004165 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004166 " has repeat elements in" +
4167 " set " + onosSetName + ":\n" +
4168 str( getResponses[ i ] ) )
4169 getResults = main.FALSE
4170 elif getResponses[ i ] == main.ERROR:
4171 getResults = main.FALSE
4172 sizeResponses = []
4173 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004174 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004175 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004176 name="setTestSize-" + str( i ),
4177 args=[ onosSetName ] )
4178 threads.append( t )
4179 t.start()
4180 for t in threads:
4181 t.join()
4182 sizeResponses.append( t.result )
4183 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004184 for i in range( len( main.activeNodes ) ):
4185 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004186 if size != sizeResponses[ i ]:
4187 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004188 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004189 str( size ) + " for set " + onosSetName +
4190 " but got " + str( sizeResponses[ i ] ) )
4191 retainResults = retainResults and getResults and sizeResults
4192 utilities.assert_equals( expect=main.TRUE,
4193 actual=retainResults,
4194 onpass="Set retain correct",
4195 onfail="Set retain was incorrect" )
4196
Jon Hall2a5002c2015-08-21 16:49:11 -07004197 # Transactional maps
4198 main.step( "Partitioned Transactional maps put" )
4199 tMapValue = "Testing"
4200 numKeys = 100
4201 putResult = True
Jon Halla440e872016-03-31 15:15:50 -07004202 node = main.activeNodes[0]
4203 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004204 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004205 for i in putResponses:
4206 if putResponses[ i ][ 'value' ] != tMapValue:
4207 putResult = False
4208 else:
4209 putResult = False
4210 if not putResult:
4211 main.log.debug( "Put response values: " + str( putResponses ) )
4212 utilities.assert_equals( expect=True,
4213 actual=putResult,
4214 onpass="Partitioned Transactional Map put successful",
4215 onfail="Partitioned Transactional Map put values are incorrect" )
4216
4217 main.step( "Partitioned Transactional maps get" )
Jon Hall9bfadd22016-05-11 14:48:07 -07004218 # FIXME: is this sleep needed?
4219 time.sleep( 5 )
4220
Jon Hall2a5002c2015-08-21 16:49:11 -07004221 getCheck = True
4222 for n in range( 1, numKeys + 1 ):
4223 getResponses = []
4224 threads = []
4225 valueCheck = True
Jon Halla440e872016-03-31 15:15:50 -07004226 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004227 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4228 name="TMap-get-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08004229 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004230 threads.append( t )
4231 t.start()
4232 for t in threads:
4233 t.join()
4234 getResponses.append( t.result )
4235 for node in getResponses:
4236 if node != tMapValue:
4237 valueCheck = False
4238 if not valueCheck:
4239 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4240 main.log.warn( getResponses )
4241 getCheck = getCheck and valueCheck
4242 utilities.assert_equals( expect=True,
4243 actual=getCheck,
4244 onpass="Partitioned Transactional Map get values were correct",
4245 onfail="Partitioned Transactional Map values incorrect" )