blob: 8b5add6cbe69c5fb3477f7f7d8e05bb5f5b09f11 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAclusterRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080051 import time
Jon Halla440e872016-03-31 15:15:50 -070052 import json
Jon Hall5cf14d52015-07-16 12:15:19 -070053 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
54 "initialization" )
55 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070056 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070057 "installing ONOS, starting Mininet and ONOS" +\
58 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070059
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
71 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79 # These are for csv plotting in jenkins
80 global labels
81 global data
82 labels = []
83 data = []
84
85 # FIXME: just get controller port from params?
86 # TODO: do we really need all these?
87 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
88 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
89 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
90 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
91 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
92 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
93 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
94
Jon Halle1a3b752015-07-22 13:02:46 -070095 try:
Jon Hall53c5e662016-04-13 16:06:56 -070096 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070097 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070098 except Exception as e:
99 main.log.exception( e )
100 main.cleanup()
101 main.exit()
102
103 main.CLIs = []
104 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700105 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700106 for i in range( 1, main.numCtrls + 1 ):
107 try:
108 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
109 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
110 ipList.append( main.nodes[ -1 ].ip_address )
111 except AttributeError:
112 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700113
114 main.step( "Create cell file" )
115 cellAppString = main.params[ 'ENV' ][ 'appString' ]
116 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
117 main.Mininet1.ip_address,
118 cellAppString, ipList )
119 main.step( "Applying cell variable to environment" )
120 cellResult = main.ONOSbench.setCell( cellName )
121 verifyResult = main.ONOSbench.verifyCell()
122
123 # FIXME:this is short term fix
124 main.log.info( "Removing raft logs" )
125 main.ONOSbench.onosRemoveRaftLogs()
126
127 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700128 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700129 main.ONOSbench.onosUninstall( node.ip_address )
130
131 # Make sure ONOS is DEAD
132 main.log.info( "Killing any ONOS processes" )
133 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700134 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700135 killed = main.ONOSbench.onosKill( node.ip_address )
136 killResults = killResults and killed
137
138 cleanInstallResult = main.TRUE
139 gitPullResult = main.TRUE
140
141 main.step( "Starting Mininet" )
142 # scp topo file to mininet
143 # TODO: move to params?
144 topoName = "obelisk.py"
145 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700146 main.ONOSbench.scp( main.Mininet1,
147 filePath + topoName,
148 main.Mininet1.home,
149 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700150 mnResult = main.Mininet1.startNet( )
151 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
152 onpass="Mininet Started",
153 onfail="Error starting Mininet" )
154
155 main.step( "Git checkout and pull " + gitBranch )
156 if PULLCODE:
157 main.ONOSbench.gitCheckout( gitBranch )
158 gitPullResult = main.ONOSbench.gitPull()
159 # values of 1 or 3 are good
160 utilities.assert_lesser( expect=0, actual=gitPullResult,
161 onpass="Git pull successful",
162 onfail="Git pull failed" )
163 main.ONOSbench.getVersion( report=True )
164
165 main.step( "Using mvn clean install" )
166 cleanInstallResult = main.TRUE
167 if PULLCODE and gitPullResult == main.TRUE:
168 cleanInstallResult = main.ONOSbench.cleanInstall()
169 else:
170 main.log.warn( "Did not pull new code so skipping mvn " +
171 "clean install" )
172 utilities.assert_equals( expect=main.TRUE,
173 actual=cleanInstallResult,
174 onpass="MCI successful",
175 onfail="MCI failed" )
176 # GRAPHS
177 # NOTE: important params here:
178 # job = name of Jenkins job
179 # Plot Name = Plot-HA, only can be used if multiple plots
180 # index = The number of the graph under plot name
181 job = "HAclusterRestart"
182 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700183 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700184 graphs = '<ac:structured-macro ac:name="html">\n'
185 graphs += '<ac:plain-text-body><![CDATA[\n'
186 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800187 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700188 '&width=500&height=300"' +\
189 'noborder="0" width="500" height="300" scrolling="yes" ' +\
190 'seamless="seamless"></iframe>\n'
191 graphs += ']]></ac:plain-text-body>\n'
192 graphs += '</ac:structured-macro>\n'
193 main.log.wiki(graphs)
194
195 main.step( "Creating ONOS package" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700196 packageResult = main.ONOSbench.buckBuild()
Jon Hall5cf14d52015-07-16 12:15:19 -0700197 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
198 onpass="ONOS package successful",
199 onfail="ONOS package failed" )
200
201 main.step( "Installing ONOS package" )
202 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700203 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700204 tmpResult = main.ONOSbench.onosInstall( options="-f",
205 node=node.ip_address )
206 onosInstallResult = onosInstallResult and tmpResult
207 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
208 onpass="ONOS install successful",
209 onfail="ONOS install failed" )
210
211 main.step( "Checking if ONOS is up yet" )
212 for i in range( 2 ):
213 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700214 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700215 started = main.ONOSbench.isup( node.ip_address )
216 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800217 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700218 onosIsupResult = onosIsupResult and started
219 if onosIsupResult == main.TRUE:
220 break
221 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
222 onpass="ONOS startup successful",
223 onfail="ONOS startup failed" )
224
Jon Hall6509dbf2016-06-21 17:01:17 -0700225 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700226 cliResults = main.TRUE
227 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700228 for i in range( main.numCtrls ):
229 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700231 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700232 threads.append( t )
233 t.start()
234
235 for t in threads:
236 t.join()
237 cliResults = cliResults and t.result
238 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
239 onpass="ONOS cli startup successful",
240 onfail="ONOS cli startup failed" )
241
Jon Halla440e872016-03-31 15:15:50 -0700242 # Create a list of active nodes for use when some nodes are stopped
243 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
244
Jon Hall5cf14d52015-07-16 12:15:19 -0700245 if main.params[ 'tcpdump' ].lower() == "true":
246 main.step( "Start Packet Capture MN" )
247 main.Mininet2.startTcpdump(
248 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
249 + "-MN.pcap",
250 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
251 port=main.params[ 'MNtcpdump' ][ 'port' ] )
252
Jon Halla440e872016-03-31 15:15:50 -0700253 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700254 nodeResults = utilities.retry( main.HA.nodesCheck,
255 False,
256 args=[main.activeNodes],
257 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700258
Jon Hall41d39f12016-04-11 22:54:35 -0700259 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700260 onpass="Nodes check successful",
261 onfail="Nodes check NOT successful" )
262
263 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700264 for i in main.activeNodes:
265 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700266 main.log.debug( "{} components not ACTIVE: \n{}".format(
267 cli.name,
268 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700269 main.log.error( "Failed to start ONOS, stopping test" )
270 main.cleanup()
271 main.exit()
272
Jon Hall172b7ba2016-04-07 18:12:20 -0700273 main.step( "Activate apps defined in the params file" )
274 # get data from the params
275 apps = main.params.get( 'apps' )
276 if apps:
277 apps = apps.split(',')
278 main.log.warn( apps )
279 activateResult = True
280 for app in apps:
281 main.CLIs[ 0 ].app( app, "Activate" )
282 # TODO: check this worked
283 time.sleep( 10 ) # wait for apps to activate
284 for app in apps:
285 state = main.CLIs[ 0 ].appStatus( app )
286 if state == "ACTIVE":
287 activateResult = activeResult and True
288 else:
289 main.log.error( "{} is in {} state".format( app, state ) )
290 activeResult = False
291 utilities.assert_equals( expect=True,
292 actual=activateResult,
293 onpass="Successfully activated apps",
294 onfail="Failed to activate apps" )
295 else:
296 main.log.warn( "No apps were specified to be loaded after startup" )
297
298 main.step( "Set ONOS configurations" )
299 config = main.params.get( 'ONOS_Configuration' )
300 if config:
301 main.log.debug( config )
302 checkResult = main.TRUE
303 for component in config:
304 for setting in config[component]:
305 value = config[component][setting]
306 check = main.CLIs[ 0 ].setCfg( component, setting, value )
307 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
308 checkResult = check and checkResult
309 utilities.assert_equals( expect=main.TRUE,
310 actual=checkResult,
311 onpass="Successfully set config",
312 onfail="Failed to set config" )
313 else:
314 main.log.warn( "No configurations were specified to be changed after startup" )
315
Jon Hall9d2dcad2016-04-08 10:15:20 -0700316 main.step( "App Ids check" )
317 appCheck = main.TRUE
318 threads = []
319 for i in main.activeNodes:
320 t = main.Thread( target=main.CLIs[i].appToIDCheck,
321 name="appToIDCheck-" + str( i ),
322 args=[] )
323 threads.append( t )
324 t.start()
325
326 for t in threads:
327 t.join()
328 appCheck = appCheck and t.result
329 if appCheck != main.TRUE:
330 node = main.activeNodes[0]
331 main.log.warn( main.CLIs[node].apps() )
332 main.log.warn( main.CLIs[node].appIDs() )
333 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
334 onpass="App Ids seem to be correct",
335 onfail="Something is wrong with app Ids" )
336
Jon Hall5cf14d52015-07-16 12:15:19 -0700337 def CASE2( self, main ):
338 """
339 Assign devices to controllers
340 """
341 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700342 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700343 assert main, "main not defined"
344 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700345 assert main.CLIs, "main.CLIs not defined"
346 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700347 assert ONOS1Port, "ONOS1Port not defined"
348 assert ONOS2Port, "ONOS2Port not defined"
349 assert ONOS3Port, "ONOS3Port not defined"
350 assert ONOS4Port, "ONOS4Port not defined"
351 assert ONOS5Port, "ONOS5Port not defined"
352 assert ONOS6Port, "ONOS6Port not defined"
353 assert ONOS7Port, "ONOS7Port not defined"
354
355 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700356 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700357 "and check that an ONOS node becomes the " +\
358 "master of the device."
359 main.step( "Assign switches to controllers" )
360
361 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700362 for i in range( main.numCtrls ):
363 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700364 swList = []
365 for i in range( 1, 29 ):
366 swList.append( "s" + str( i ) )
367 main.Mininet1.assignSwController( sw=swList, ip=ipList )
368
369 mastershipCheck = main.TRUE
370 for i in range( 1, 29 ):
371 response = main.Mininet1.getSwController( "s" + str( i ) )
372 try:
373 main.log.info( str( response ) )
374 except Exception:
375 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700376 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700377 if re.search( "tcp:" + node.ip_address, response ):
378 mastershipCheck = mastershipCheck and main.TRUE
379 else:
380 main.log.error( "Error, node " + node.ip_address + " is " +
381 "not in the list of controllers s" +
382 str( i ) + " is connecting to." )
383 mastershipCheck = main.FALSE
384 utilities.assert_equals(
385 expect=main.TRUE,
386 actual=mastershipCheck,
387 onpass="Switch mastership assigned correctly",
388 onfail="Switches not assigned correctly to controllers" )
389
390 def CASE21( self, main ):
391 """
392 Assign mastership to controllers
393 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700394 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700395 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700396 assert main, "main not defined"
397 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700398 assert main.CLIs, "main.CLIs not defined"
399 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700400 assert ONOS1Port, "ONOS1Port not defined"
401 assert ONOS2Port, "ONOS2Port not defined"
402 assert ONOS3Port, "ONOS3Port not defined"
403 assert ONOS4Port, "ONOS4Port not defined"
404 assert ONOS5Port, "ONOS5Port not defined"
405 assert ONOS6Port, "ONOS6Port not defined"
406 assert ONOS7Port, "ONOS7Port not defined"
407
408 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700409 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700410 "device. Then manually assign" +\
411 " mastership to specific ONOS nodes using" +\
412 " 'device-role'"
413 main.step( "Assign mastership of switches to specific controllers" )
414 # Manually assign mastership to the controller we want
415 roleCall = main.TRUE
416
417 ipList = [ ]
418 deviceList = []
Jon Halla440e872016-03-31 15:15:50 -0700419 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700420 try:
421 # Assign mastership to specific controllers. This assignment was
422 # determined for a 7 node cluser, but will work with any sized
423 # cluster
424 for i in range( 1, 29 ): # switches 1 through 28
425 # set up correct variables:
426 if i == 1:
427 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700428 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700429 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700430 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700431 c = 1 % main.numCtrls
432 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700433 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700434 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700435 c = 1 % main.numCtrls
436 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700437 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700438 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700439 c = 3 % main.numCtrls
440 ip = main.nodes[ c ].ip_address # ONOS4
Jon Halla440e872016-03-31 15:15:50 -0700441 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700442 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700443 c = 2 % main.numCtrls
444 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700445 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700446 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700447 c = 2 % main.numCtrls
448 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700449 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700450 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700451 c = 5 % main.numCtrls
452 ip = main.nodes[ c ].ip_address # ONOS6
Jon Halla440e872016-03-31 15:15:50 -0700453 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700454 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700455 c = 4 % main.numCtrls
456 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700457 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700458 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700460 c = 6 % main.numCtrls
461 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700462 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700463 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700464 elif i == 28:
465 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700466 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700467 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700468 else:
469 main.log.error( "You didn't write an else statement for " +
470 "switch s" + str( i ) )
471 roleCall = main.FALSE
472 # Assign switch
473 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
474 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700475 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700476 ipList.append( ip )
477 deviceList.append( deviceId )
478 except ( AttributeError, AssertionError ):
479 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700480 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700481 utilities.assert_equals(
482 expect=main.TRUE,
483 actual=roleCall,
484 onpass="Re-assigned switch mastership to designated controller",
485 onfail="Something wrong with deviceRole calls" )
486
487 main.step( "Check mastership was correctly assigned" )
488 roleCheck = main.TRUE
489 # NOTE: This is due to the fact that device mastership change is not
490 # atomic and is actually a multi step process
491 time.sleep( 5 )
492 for i in range( len( ipList ) ):
493 ip = ipList[i]
494 deviceId = deviceList[i]
495 # Check assignment
Jon Halla440e872016-03-31 15:15:50 -0700496 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700497 if ip in master:
498 roleCheck = roleCheck and main.TRUE
499 else:
500 roleCheck = roleCheck and main.FALSE
501 main.log.error( "Error, controller " + ip + " is not" +
502 " master " + "of device " +
503 str( deviceId ) + ". Master is " +
504 repr( master ) + "." )
505 utilities.assert_equals(
506 expect=main.TRUE,
507 actual=roleCheck,
508 onpass="Switches were successfully reassigned to designated " +
509 "controller",
510 onfail="Switches were not successfully reassigned" )
511
512 def CASE3( self, main ):
513 """
514 Assign intents
515 """
516 import time
517 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700518 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700519 assert main, "main not defined"
520 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700521 assert main.CLIs, "main.CLIs not defined"
522 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700523 try:
524 labels
525 except NameError:
526 main.log.error( "labels not defined, setting to []" )
527 labels = []
528 try:
529 data
530 except NameError:
531 main.log.error( "data not defined, setting to []" )
532 data = []
533 # NOTE: we must reinstall intents until we have a persistant intent
534 # datastore!
535 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700536 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700537 "assign predetermined host-to-host intents." +\
538 " After installation, check that the intent" +\
539 " is distributed to all nodes and the state" +\
540 " is INSTALLED"
541
542 # install onos-app-fwd
543 main.step( "Install reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700544 onosCli = main.CLIs[ main.activeNodes[0] ]
545 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700546 utilities.assert_equals( expect=main.TRUE, actual=installResults,
547 onpass="Install fwd successful",
548 onfail="Install fwd failed" )
549
550 main.step( "Check app ids" )
551 appCheck = main.TRUE
552 threads = []
Jon Halla440e872016-03-31 15:15:50 -0700553 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700554 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700555 name="appToIDCheck-" + str( i ),
556 args=[] )
557 threads.append( t )
558 t.start()
559
560 for t in threads:
561 t.join()
562 appCheck = appCheck and t.result
563 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700564 main.log.warn( onosCli.apps() )
565 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700566 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
567 onpass="App Ids seem to be correct",
568 onfail="Something is wrong with app Ids" )
569
570 main.step( "Discovering Hosts( Via pingall for now )" )
571 # FIXME: Once we have a host discovery mechanism, use that instead
572 # REACTIVE FWD test
573 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700574 passMsg = "Reactive Pingall test passed"
575 time1 = time.time()
576 pingResult = main.Mininet1.pingall()
577 time2 = time.time()
578 if not pingResult:
579 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700580 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700581 passMsg += " on the second try"
582 utilities.assert_equals(
583 expect=main.TRUE,
584 actual=pingResult,
585 onpass= passMsg,
586 onfail="Reactive Pingall failed, " +
587 "one or more ping pairs failed" )
588 main.log.info( "Time for pingall: %2f seconds" %
589 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700590 # timeout for fwd flows
591 time.sleep( 11 )
592 # uninstall onos-app-fwd
593 main.step( "Uninstall reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700594 node = main.activeNodes[0]
595 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700596 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
597 onpass="Uninstall fwd successful",
598 onfail="Uninstall fwd failed" )
599
600 main.step( "Check app ids" )
601 threads = []
602 appCheck2 = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700603 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700604 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700605 name="appToIDCheck-" + str( i ),
606 args=[] )
607 threads.append( t )
608 t.start()
609
610 for t in threads:
611 t.join()
612 appCheck2 = appCheck2 and t.result
613 if appCheck2 != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700614 node = main.activeNodes[0]
615 main.log.warn( main.CLIs[node].apps() )
616 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700617 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
618 onpass="App Ids seem to be correct",
619 onfail="Something is wrong with app Ids" )
620
621 main.step( "Add host intents via cli" )
622 intentIds = []
Jon Hall6e709752016-02-01 13:38:46 -0800623 # TODO: move the host numbers to params
624 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700625 intentAddResult = True
626 hostResult = main.TRUE
627 for i in range( 8, 18 ):
628 main.log.info( "Adding host intent between h" + str( i ) +
629 " and h" + str( i + 10 ) )
630 host1 = "00:00:00:00:00:" + \
631 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
632 host2 = "00:00:00:00:00:" + \
633 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
634 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700635 host1Dict = onosCli.getHost( host1 )
636 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700637 host1Id = None
638 host2Id = None
639 if host1Dict and host2Dict:
640 host1Id = host1Dict.get( 'id', None )
641 host2Id = host2Dict.get( 'id', None )
642 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700643 nodeNum = ( i % len( main.activeNodes ) )
644 node = main.activeNodes[nodeNum]
645 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700646 if tmpId:
647 main.log.info( "Added intent with id: " + tmpId )
648 intentIds.append( tmpId )
649 else:
650 main.log.error( "addHostIntent returned: " +
651 repr( tmpId ) )
652 else:
653 main.log.error( "Error, getHost() failed for h" + str( i ) +
654 " and/or h" + str( i + 10 ) )
Jon Halla440e872016-03-31 15:15:50 -0700655 node = main.activeNodes[0]
656 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700657 main.log.warn( "Hosts output: " )
658 try:
659 main.log.warn( json.dumps( json.loads( hosts ),
660 sort_keys=True,
661 indent=4,
662 separators=( ',', ': ' ) ) )
663 except ( ValueError, TypeError ):
664 main.log.warn( repr( hosts ) )
665 hostResult = main.FALSE
666 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
667 onpass="Found a host id for each host",
668 onfail="Error looking up host ids" )
669
670 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700671 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700672 main.log.info( "Submitted intents: " + str( intentIds ) )
673 main.log.info( "Intents in ONOS: " + str( onosIds ) )
674 for intent in intentIds:
675 if intent in onosIds:
676 pass # intent submitted is in onos
677 else:
678 intentAddResult = False
679 if intentAddResult:
680 intentStop = time.time()
681 else:
682 intentStop = None
683 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700684 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700685 intentStates = []
686 installedCheck = True
687 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
688 count = 0
689 try:
690 for intent in json.loads( intents ):
691 state = intent.get( 'state', None )
692 if "INSTALLED" not in state:
693 installedCheck = False
694 intentId = intent.get( 'id', None )
695 intentStates.append( ( intentId, state ) )
696 except ( ValueError, TypeError ):
697 main.log.exception( "Error parsing intents" )
698 # add submitted intents not in the store
699 tmplist = [ i for i, s in intentStates ]
700 missingIntents = False
701 for i in intentIds:
702 if i not in tmplist:
703 intentStates.append( ( i, " - " ) )
704 missingIntents = True
705 intentStates.sort()
706 for i, s in intentStates:
707 count += 1
708 main.log.info( "%-6s%-15s%-15s" %
709 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700710 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700711 try:
712 missing = False
713 if leaders:
714 parsedLeaders = json.loads( leaders )
715 main.log.warn( json.dumps( parsedLeaders,
716 sort_keys=True,
717 indent=4,
718 separators=( ',', ': ' ) ) )
719 # check for all intent partitions
720 topics = []
721 for i in range( 14 ):
722 topics.append( "intent-partition-" + str( i ) )
723 main.log.debug( topics )
724 ONOStopics = [ j['topic'] for j in parsedLeaders ]
725 for topic in topics:
726 if topic not in ONOStopics:
727 main.log.error( "Error: " + topic +
728 " not in leaders" )
729 missing = True
730 else:
731 main.log.error( "leaders() returned None" )
732 except ( ValueError, TypeError ):
733 main.log.exception( "Error parsing leaders" )
734 main.log.error( repr( leaders ) )
735 # Check all nodes
736 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700737 for i in main.activeNodes:
738 response = main.CLIs[i].leaders( jsonFormat=False)
739 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700740 str( response ) )
741
Jon Halla440e872016-03-31 15:15:50 -0700742 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700743 try:
744 if partitions :
745 parsedPartitions = json.loads( partitions )
746 main.log.warn( json.dumps( parsedPartitions,
747 sort_keys=True,
748 indent=4,
749 separators=( ',', ': ' ) ) )
750 # TODO check for a leader in all paritions
751 # TODO check for consistency among nodes
752 else:
753 main.log.error( "partitions() returned None" )
754 except ( ValueError, TypeError ):
755 main.log.exception( "Error parsing partitions" )
756 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700757 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700758 try:
759 if pendingMap :
760 parsedPending = json.loads( pendingMap )
761 main.log.warn( json.dumps( parsedPending,
762 sort_keys=True,
763 indent=4,
764 separators=( ',', ': ' ) ) )
765 # TODO check something here?
766 else:
767 main.log.error( "pendingMap() returned None" )
768 except ( ValueError, TypeError ):
769 main.log.exception( "Error parsing pending map" )
770 main.log.error( repr( pendingMap ) )
771
772 intentAddResult = bool( intentAddResult and not missingIntents and
773 installedCheck )
774 if not intentAddResult:
775 main.log.error( "Error in pushing host intents to ONOS" )
776
777 main.step( "Intent Anti-Entropy dispersion" )
Jon Halla440e872016-03-31 15:15:50 -0700778 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700779 correct = True
780 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700781 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700782 onosIds = []
Jon Halla440e872016-03-31 15:15:50 -0700783 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700784 onosIds.append( ids )
Jon Halla440e872016-03-31 15:15:50 -0700785 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700786 str( sorted( onosIds ) ) )
787 if sorted( ids ) != sorted( intentIds ):
788 main.log.warn( "Set of intent IDs doesn't match" )
789 correct = False
790 break
791 else:
Jon Halla440e872016-03-31 15:15:50 -0700792 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700793 for intent in intents:
794 if intent[ 'state' ] != "INSTALLED":
795 main.log.warn( "Intent " + intent[ 'id' ] +
796 " is " + intent[ 'state' ] )
797 correct = False
798 break
799 if correct:
800 break
801 else:
802 time.sleep(1)
803 if not intentStop:
804 intentStop = time.time()
805 global gossipTime
806 gossipTime = intentStop - intentStart
807 main.log.info( "It took about " + str( gossipTime ) +
808 " seconds for all intents to appear in each node" )
809 append = False
810 title = "Gossip Intents"
811 count = 1
812 while append is False:
813 curTitle = title + str( count )
814 if curTitle not in labels:
815 labels.append( curTitle )
816 data.append( str( gossipTime ) )
817 append = True
818 else:
819 count += 1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700820 gossipPeriod = int( main.params['timers']['gossip'] )
Jon Halla440e872016-03-31 15:15:50 -0700821 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700822 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700823 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700824 onpass="ECM anti-entropy for intents worked within " +
825 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700826 onfail="Intent ECM anti-entropy took too long. " +
827 "Expected time:{}, Actual time:{}".format( maxGossipTime,
828 gossipTime ) )
829 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700830 intentAddResult = True
831
832 if not intentAddResult or "key" in pendingMap:
833 import time
834 installedCheck = True
835 main.log.info( "Sleeping 60 seconds to see if intents are found" )
836 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700837 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700838 main.log.info( "Submitted intents: " + str( intentIds ) )
839 main.log.info( "Intents in ONOS: " + str( onosIds ) )
840 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700841 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700842 intentStates = []
843 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
844 count = 0
845 try:
846 for intent in json.loads( intents ):
847 # Iter through intents of a node
848 state = intent.get( 'state', None )
849 if "INSTALLED" not in state:
850 installedCheck = False
851 intentId = intent.get( 'id', None )
852 intentStates.append( ( intentId, state ) )
853 except ( ValueError, TypeError ):
854 main.log.exception( "Error parsing intents" )
855 # add submitted intents not in the store
856 tmplist = [ i for i, s in intentStates ]
857 for i in intentIds:
858 if i not in tmplist:
859 intentStates.append( ( i, " - " ) )
860 intentStates.sort()
861 for i, s in intentStates:
862 count += 1
863 main.log.info( "%-6s%-15s%-15s" %
864 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700865 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700866 try:
867 missing = False
868 if leaders:
869 parsedLeaders = json.loads( leaders )
870 main.log.warn( json.dumps( parsedLeaders,
871 sort_keys=True,
872 indent=4,
873 separators=( ',', ': ' ) ) )
874 # check for all intent partitions
875 # check for election
876 topics = []
877 for i in range( 14 ):
878 topics.append( "intent-partition-" + str( i ) )
879 # FIXME: this should only be after we start the app
880 topics.append( "org.onosproject.election" )
881 main.log.debug( topics )
882 ONOStopics = [ j['topic'] for j in parsedLeaders ]
883 for topic in topics:
884 if topic not in ONOStopics:
885 main.log.error( "Error: " + topic +
886 " not in leaders" )
887 missing = True
888 else:
889 main.log.error( "leaders() returned None" )
890 except ( ValueError, TypeError ):
891 main.log.exception( "Error parsing leaders" )
892 main.log.error( repr( leaders ) )
893 # Check all nodes
894 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700895 for i in main.activeNodes:
896 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700897 response = node.leaders( jsonFormat=False)
898 main.log.warn( str( node.name ) + " leaders output: \n" +
899 str( response ) )
900
Jon Halla440e872016-03-31 15:15:50 -0700901 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700902 try:
903 if partitions :
904 parsedPartitions = json.loads( partitions )
905 main.log.warn( json.dumps( parsedPartitions,
906 sort_keys=True,
907 indent=4,
908 separators=( ',', ': ' ) ) )
909 # TODO check for a leader in all paritions
910 # TODO check for consistency among nodes
911 else:
912 main.log.error( "partitions() returned None" )
913 except ( ValueError, TypeError ):
914 main.log.exception( "Error parsing partitions" )
915 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700916 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700917 try:
918 if pendingMap :
919 parsedPending = json.loads( pendingMap )
920 main.log.warn( json.dumps( parsedPending,
921 sort_keys=True,
922 indent=4,
923 separators=( ',', ': ' ) ) )
924 # TODO check something here?
925 else:
926 main.log.error( "pendingMap() returned None" )
927 except ( ValueError, TypeError ):
928 main.log.exception( "Error parsing pending map" )
929 main.log.error( repr( pendingMap ) )
930
931 def CASE4( self, main ):
932 """
933 Ping across added host intents
934 """
935 import json
936 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700937 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700938 assert main, "main not defined"
939 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700940 assert main.CLIs, "main.CLIs not defined"
941 assert main.nodes, "main.nodes not defined"
Jon Hall6e709752016-02-01 13:38:46 -0800942 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700943 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700944 "functionality and check the state of " +\
945 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700946
Jon Hall41d39f12016-04-11 22:54:35 -0700947 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700948 main.step( "Check Intent state" )
949 installedCheck = False
950 loopCount = 0
951 while not installedCheck and loopCount < 40:
952 installedCheck = True
953 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700954 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700955 intentStates = []
956 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700957 count = 0
Jon Hall5cf14d52015-07-16 12:15:19 -0700958 # Iter through intents of a node
959 try:
960 for intent in json.loads( intents ):
961 state = intent.get( 'state', None )
962 if "INSTALLED" not in state:
963 installedCheck = False
964 intentId = intent.get( 'id', None )
965 intentStates.append( ( intentId, state ) )
966 except ( ValueError, TypeError ):
967 main.log.exception( "Error parsing intents." )
968 # Print states
969 intentStates.sort()
970 for i, s in intentStates:
971 count += 1
972 main.log.info( "%-6s%-15s%-15s" %
973 ( str( count ), str( i ), str( s ) ) )
974 if not installedCheck:
975 time.sleep( 1 )
976 loopCount += 1
977 utilities.assert_equals( expect=True, actual=installedCheck,
978 onpass="Intents are all INSTALLED",
979 onfail="Intents are not all in " +
980 "INSTALLED state" )
981
Jon Hall9d2dcad2016-04-08 10:15:20 -0700982 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700983 PingResult = main.TRUE
984 for i in range( 8, 18 ):
985 ping = main.Mininet1.pingHost( src="h" + str( i ),
986 target="h" + str( i + 10 ) )
987 PingResult = PingResult and ping
988 if ping == main.FALSE:
989 main.log.warn( "Ping failed between h" + str( i ) +
990 " and h" + str( i + 10 ) )
991 elif ping == main.TRUE:
992 main.log.info( "Ping test passed!" )
993 # Don't set PingResult or you'd override failures
994 if PingResult == main.FALSE:
995 main.log.error(
996 "Intents have not been installed correctly, pings failed." )
997 # TODO: pretty print
998 main.log.warn( "ONOS1 intents: " )
999 try:
1000 tmpIntents = onosCli.intents()
1001 main.log.warn( json.dumps( json.loads( tmpIntents ),
1002 sort_keys=True,
1003 indent=4,
1004 separators=( ',', ': ' ) ) )
1005 except ( ValueError, TypeError ):
1006 main.log.warn( repr( tmpIntents ) )
1007 utilities.assert_equals(
1008 expect=main.TRUE,
1009 actual=PingResult,
1010 onpass="Intents have been installed correctly and pings work",
1011 onfail="Intents have not been installed correctly, pings failed." )
1012
Jon Hall5cf14d52015-07-16 12:15:19 -07001013 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -07001014 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001015 topicCheck = main.TRUE
1016 try:
1017 if leaders:
1018 parsedLeaders = json.loads( leaders )
1019 main.log.warn( json.dumps( parsedLeaders,
1020 sort_keys=True,
1021 indent=4,
1022 separators=( ',', ': ' ) ) )
1023 # check for all intent partitions
1024 # check for election
1025 # TODO: Look at Devices as topics now that it uses this system
1026 topics = []
1027 for i in range( 14 ):
1028 topics.append( "intent-partition-" + str( i ) )
1029 # FIXME: this should only be after we start the app
1030 # FIXME: topics.append( "org.onosproject.election" )
1031 # Print leaders output
1032 main.log.debug( topics )
1033 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1034 for topic in topics:
1035 if topic not in ONOStopics:
1036 main.log.error( "Error: " + topic +
1037 " not in leaders" )
1038 topicCheck = main.FALSE
1039 else:
1040 main.log.error( "leaders() returned None" )
1041 topicCheck = main.FALSE
1042 except ( ValueError, TypeError ):
1043 topicCheck = main.FALSE
1044 main.log.exception( "Error parsing leaders" )
1045 main.log.error( repr( leaders ) )
1046 # TODO: Check for a leader of these topics
1047 # Check all nodes
1048 if topicCheck:
Jon Halla440e872016-03-31 15:15:50 -07001049 for i in main.activeNodes:
1050 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001051 response = node.leaders( jsonFormat=False)
1052 main.log.warn( str( node.name ) + " leaders output: \n" +
1053 str( response ) )
1054
1055 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1056 onpass="intent Partitions is in leaders",
1057 onfail="Some topics were lost " )
1058 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -07001059 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001060 try:
1061 if partitions :
1062 parsedPartitions = json.loads( partitions )
1063 main.log.warn( json.dumps( parsedPartitions,
1064 sort_keys=True,
1065 indent=4,
1066 separators=( ',', ': ' ) ) )
1067 # TODO check for a leader in all paritions
1068 # TODO check for consistency among nodes
1069 else:
1070 main.log.error( "partitions() returned None" )
1071 except ( ValueError, TypeError ):
1072 main.log.exception( "Error parsing partitions" )
1073 main.log.error( repr( partitions ) )
1074 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001075 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001076 try:
1077 if pendingMap :
1078 parsedPending = json.loads( pendingMap )
1079 main.log.warn( json.dumps( parsedPending,
1080 sort_keys=True,
1081 indent=4,
1082 separators=( ',', ': ' ) ) )
1083 # TODO check something here?
1084 else:
1085 main.log.error( "pendingMap() returned None" )
1086 except ( ValueError, TypeError ):
1087 main.log.exception( "Error parsing pending map" )
1088 main.log.error( repr( pendingMap ) )
1089
1090 if not installedCheck:
1091 main.log.info( "Waiting 60 seconds to see if the state of " +
1092 "intents change" )
1093 time.sleep( 60 )
1094 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001095 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001096 intentStates = []
1097 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1098 count = 0
1099 # Iter through intents of a node
1100 try:
1101 for intent in json.loads( intents ):
1102 state = intent.get( 'state', None )
1103 if "INSTALLED" not in state:
1104 installedCheck = False
1105 intentId = intent.get( 'id', None )
1106 intentStates.append( ( intentId, state ) )
1107 except ( ValueError, TypeError ):
1108 main.log.exception( "Error parsing intents." )
1109 intentStates.sort()
1110 for i, s in intentStates:
1111 count += 1
1112 main.log.info( "%-6s%-15s%-15s" %
1113 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001114 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001115 try:
1116 missing = False
1117 if leaders:
1118 parsedLeaders = json.loads( leaders )
1119 main.log.warn( json.dumps( parsedLeaders,
1120 sort_keys=True,
1121 indent=4,
1122 separators=( ',', ': ' ) ) )
1123 # check for all intent partitions
1124 # check for election
1125 topics = []
1126 for i in range( 14 ):
1127 topics.append( "intent-partition-" + str( i ) )
1128 # FIXME: this should only be after we start the app
1129 topics.append( "org.onosproject.election" )
1130 main.log.debug( topics )
1131 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1132 for topic in topics:
1133 if topic not in ONOStopics:
1134 main.log.error( "Error: " + topic +
1135 " not in leaders" )
1136 missing = True
1137 else:
1138 main.log.error( "leaders() returned None" )
1139 except ( ValueError, TypeError ):
1140 main.log.exception( "Error parsing leaders" )
1141 main.log.error( repr( leaders ) )
1142 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001143 for i in main.activeNodes:
1144 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001145 response = node.leaders( jsonFormat=False)
1146 main.log.warn( str( node.name ) + " leaders output: \n" +
1147 str( response ) )
1148
Jon Halla440e872016-03-31 15:15:50 -07001149 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001150 try:
1151 if partitions :
1152 parsedPartitions = json.loads( partitions )
1153 main.log.warn( json.dumps( parsedPartitions,
1154 sort_keys=True,
1155 indent=4,
1156 separators=( ',', ': ' ) ) )
1157 # TODO check for a leader in all paritions
1158 # TODO check for consistency among nodes
1159 else:
1160 main.log.error( "partitions() returned None" )
1161 except ( ValueError, TypeError ):
1162 main.log.exception( "Error parsing partitions" )
1163 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001164 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001165 try:
1166 if pendingMap :
1167 parsedPending = json.loads( pendingMap )
1168 main.log.warn( json.dumps( parsedPending,
1169 sort_keys=True,
1170 indent=4,
1171 separators=( ',', ': ' ) ) )
1172 # TODO check something here?
1173 else:
1174 main.log.error( "pendingMap() returned None" )
1175 except ( ValueError, TypeError ):
1176 main.log.exception( "Error parsing pending map" )
1177 main.log.error( repr( pendingMap ) )
1178 # Print flowrules
Jon Halla440e872016-03-31 15:15:50 -07001179 node = main.activeNodes[0]
1180 main.log.debug( main.CLIs[node].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001181 main.step( "Wait a minute then ping again" )
1182 # the wait is above
1183 PingResult = main.TRUE
1184 for i in range( 8, 18 ):
1185 ping = main.Mininet1.pingHost( src="h" + str( i ),
1186 target="h" + str( i + 10 ) )
1187 PingResult = PingResult and ping
1188 if ping == main.FALSE:
1189 main.log.warn( "Ping failed between h" + str( i ) +
1190 " and h" + str( i + 10 ) )
1191 elif ping == main.TRUE:
1192 main.log.info( "Ping test passed!" )
1193 # Don't set PingResult or you'd override failures
1194 if PingResult == main.FALSE:
1195 main.log.error(
1196 "Intents have not been installed correctly, pings failed." )
1197 # TODO: pretty print
1198 main.log.warn( "ONOS1 intents: " )
1199 try:
Jon Halla440e872016-03-31 15:15:50 -07001200 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001201 main.log.warn( json.dumps( json.loads( tmpIntents ),
1202 sort_keys=True,
1203 indent=4,
1204 separators=( ',', ': ' ) ) )
1205 except ( ValueError, TypeError ):
1206 main.log.warn( repr( tmpIntents ) )
1207 utilities.assert_equals(
1208 expect=main.TRUE,
1209 actual=PingResult,
1210 onpass="Intents have been installed correctly and pings work",
1211 onfail="Intents have not been installed correctly, pings failed." )
1212
1213 def CASE5( self, main ):
1214 """
1215 Reading state of ONOS
1216 """
1217 import json
1218 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001219 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001220 assert main, "main not defined"
1221 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001222 assert main.CLIs, "main.CLIs not defined"
1223 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001224
1225 main.case( "Setting up and gathering data for current state" )
1226 # The general idea for this test case is to pull the state of
1227 # ( intents,flows, topology,... ) from each ONOS node
1228 # We can then compare them with each other and also with past states
1229
1230 main.step( "Check that each switch has a master" )
1231 global mastershipState
1232 mastershipState = '[]'
1233
1234 # Assert that each device has a master
1235 rolesNotNull = main.TRUE
1236 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001237 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001238 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001239 name="rolesNotNull-" + str( i ),
1240 args=[] )
1241 threads.append( t )
1242 t.start()
1243
1244 for t in threads:
1245 t.join()
1246 rolesNotNull = rolesNotNull and t.result
1247 utilities.assert_equals(
1248 expect=main.TRUE,
1249 actual=rolesNotNull,
1250 onpass="Each device has a master",
1251 onfail="Some devices don't have a master assigned" )
1252
1253 main.step( "Get the Mastership of each switch from each controller" )
1254 ONOSMastership = []
1255 mastershipCheck = main.FALSE
1256 consistentMastership = True
1257 rolesResults = True
1258 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001259 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001260 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001261 name="roles-" + str( i ),
1262 args=[] )
1263 threads.append( t )
1264 t.start()
1265
1266 for t in threads:
1267 t.join()
1268 ONOSMastership.append( t.result )
1269
Jon Halla440e872016-03-31 15:15:50 -07001270 for i in range( len( ONOSMastership ) ):
1271 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001272 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001273 main.log.error( "Error in getting ONOS" + node + " roles" )
1274 main.log.warn( "ONOS" + node + " mastership response: " +
1275 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001276 rolesResults = False
1277 utilities.assert_equals(
1278 expect=True,
1279 actual=rolesResults,
1280 onpass="No error in reading roles output",
1281 onfail="Error in reading roles from ONOS" )
1282
1283 main.step( "Check for consistency in roles from each controller" )
1284 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1285 main.log.info(
1286 "Switch roles are consistent across all ONOS nodes" )
1287 else:
1288 consistentMastership = False
1289 utilities.assert_equals(
1290 expect=True,
1291 actual=consistentMastership,
1292 onpass="Switch roles are consistent across all ONOS nodes",
1293 onfail="ONOS nodes have different views of switch roles" )
1294
1295 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001296 for i in range( len( main.activeNodes ) ):
1297 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001298 try:
1299 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001300 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001301 json.dumps(
1302 json.loads( ONOSMastership[ i ] ),
1303 sort_keys=True,
1304 indent=4,
1305 separators=( ',', ': ' ) ) )
1306 except ( ValueError, TypeError ):
1307 main.log.warn( repr( ONOSMastership[ i ] ) )
1308 elif rolesResults and consistentMastership:
1309 mastershipCheck = main.TRUE
1310 mastershipState = ONOSMastership[ 0 ]
1311
1312 main.step( "Get the intents from each controller" )
1313 global intentState
1314 intentState = []
1315 ONOSIntents = []
1316 intentCheck = main.FALSE
1317 consistentIntents = True
1318 intentsResults = True
1319 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001320 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001321 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001322 name="intents-" + str( i ),
1323 args=[],
1324 kwargs={ 'jsonFormat': True } )
1325 threads.append( t )
1326 t.start()
1327
1328 for t in threads:
1329 t.join()
1330 ONOSIntents.append( t.result )
1331
Jon Halla440e872016-03-31 15:15:50 -07001332 for i in range( len( ONOSIntents ) ):
1333 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001334 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hall6e709752016-02-01 13:38:46 -08001335 main.log.error( "Error in getting ONOS" + node + " intents" )
1336 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001337 repr( ONOSIntents[ i ] ) )
1338 intentsResults = False
1339 utilities.assert_equals(
1340 expect=True,
1341 actual=intentsResults,
1342 onpass="No error in reading intents output",
1343 onfail="Error in reading intents from ONOS" )
1344
1345 main.step( "Check for consistency in Intents from each controller" )
1346 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1347 main.log.info( "Intents are consistent across all ONOS " +
1348 "nodes" )
1349 else:
1350 consistentIntents = False
1351 main.log.error( "Intents not consistent" )
1352 utilities.assert_equals(
1353 expect=True,
1354 actual=consistentIntents,
1355 onpass="Intents are consistent across all ONOS nodes",
1356 onfail="ONOS nodes have different views of intents" )
1357
1358 if intentsResults:
1359 # Try to make it easy to figure out what is happening
1360 #
1361 # Intent ONOS1 ONOS2 ...
1362 # 0x01 INSTALLED INSTALLING
1363 # ... ... ...
1364 # ... ... ...
1365 title = " Id"
Jon Halla440e872016-03-31 15:15:50 -07001366 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001367 title += " " * 10 + "ONOS" + str( n + 1 )
1368 main.log.warn( title )
1369 # get all intent keys in the cluster
1370 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001371 try:
1372 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001373 for nodeStr in ONOSIntents:
1374 node = json.loads( nodeStr )
1375 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001376 keys.append( intent.get( 'id' ) )
1377 keys = set( keys )
1378 # For each intent key, print the state on each node
1379 for key in keys:
1380 row = "%-13s" % key
1381 for nodeStr in ONOSIntents:
1382 node = json.loads( nodeStr )
1383 for intent in node:
1384 if intent.get( 'id', "Error" ) == key:
1385 row += "%-15s" % intent.get( 'state' )
1386 main.log.warn( row )
1387 # End of intent state table
1388 except ValueError as e:
1389 main.log.exception( e )
1390 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001391
1392 if intentsResults and not consistentIntents:
1393 # print the json objects
Jon Halla440e872016-03-31 15:15:50 -07001394 n = str( main.activeNodes[-1] + 1 )
1395 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001396 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1397 sort_keys=True,
1398 indent=4,
1399 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001400 for i in range( len( ONOSIntents ) ):
1401 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001402 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07001403 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001404 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1405 sort_keys=True,
1406 indent=4,
1407 separators=( ',', ': ' ) ) )
1408 else:
Jon Halla440e872016-03-31 15:15:50 -07001409 main.log.debug( "ONOS" + node + " intents match ONOS" +
1410 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001411 elif intentsResults and consistentIntents:
1412 intentCheck = main.TRUE
1413 intentState = ONOSIntents[ 0 ]
1414
1415 main.step( "Get the flows from each controller" )
1416 global flowState
1417 flowState = []
1418 ONOSFlows = []
1419 ONOSFlowsJson = []
1420 flowCheck = main.FALSE
1421 consistentFlows = True
1422 flowsResults = True
1423 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001424 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001425 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001426 name="flows-" + str( i ),
1427 args=[],
1428 kwargs={ 'jsonFormat': True } )
1429 threads.append( t )
1430 t.start()
1431
1432 # NOTE: Flows command can take some time to run
1433 time.sleep(30)
1434 for t in threads:
1435 t.join()
1436 result = t.result
1437 ONOSFlows.append( result )
1438
Jon Halla440e872016-03-31 15:15:50 -07001439 for i in range( len( ONOSFlows ) ):
1440 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001441 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1442 main.log.error( "Error in getting ONOS" + num + " flows" )
1443 main.log.warn( "ONOS" + num + " flows response: " +
1444 repr( ONOSFlows[ i ] ) )
1445 flowsResults = False
1446 ONOSFlowsJson.append( None )
1447 else:
1448 try:
1449 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1450 except ( ValueError, TypeError ):
1451 # FIXME: change this to log.error?
1452 main.log.exception( "Error in parsing ONOS" + num +
1453 " response as json." )
1454 main.log.error( repr( ONOSFlows[ i ] ) )
1455 ONOSFlowsJson.append( None )
1456 flowsResults = False
1457 utilities.assert_equals(
1458 expect=True,
1459 actual=flowsResults,
1460 onpass="No error in reading flows output",
1461 onfail="Error in reading flows from ONOS" )
1462
1463 main.step( "Check for consistency in Flows from each controller" )
1464 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1465 if all( tmp ):
1466 main.log.info( "Flow count is consistent across all ONOS nodes" )
1467 else:
1468 consistentFlows = False
1469 utilities.assert_equals(
1470 expect=True,
1471 actual=consistentFlows,
1472 onpass="The flow count is consistent across all ONOS nodes",
1473 onfail="ONOS nodes have different flow counts" )
1474
1475 if flowsResults and not consistentFlows:
Jon Halla440e872016-03-31 15:15:50 -07001476 for i in range( len( ONOSFlows ) ):
1477 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001478 try:
1479 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001480 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001481 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1482 indent=4, separators=( ',', ': ' ) ) )
1483 except ( ValueError, TypeError ):
Jon Halla440e872016-03-31 15:15:50 -07001484 main.log.warn( "ONOS" + node + " flows: " +
1485 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001486 elif flowsResults and consistentFlows:
1487 flowCheck = main.TRUE
1488 flowState = ONOSFlows[ 0 ]
1489
1490 main.step( "Get the OF Table entries" )
1491 global flows
1492 flows = []
1493 for i in range( 1, 29 ):
Jon Halla440e872016-03-31 15:15:50 -07001494 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001495 if flowCheck == main.FALSE:
1496 for table in flows:
1497 main.log.warn( table )
1498 # TODO: Compare switch flow tables with ONOS flow tables
1499
1500 main.step( "Start continuous pings" )
1501 main.Mininet2.pingLong(
1502 src=main.params[ 'PING' ][ 'source1' ],
1503 target=main.params[ 'PING' ][ 'target1' ],
1504 pingTime=500 )
1505 main.Mininet2.pingLong(
1506 src=main.params[ 'PING' ][ 'source2' ],
1507 target=main.params[ 'PING' ][ 'target2' ],
1508 pingTime=500 )
1509 main.Mininet2.pingLong(
1510 src=main.params[ 'PING' ][ 'source3' ],
1511 target=main.params[ 'PING' ][ 'target3' ],
1512 pingTime=500 )
1513 main.Mininet2.pingLong(
1514 src=main.params[ 'PING' ][ 'source4' ],
1515 target=main.params[ 'PING' ][ 'target4' ],
1516 pingTime=500 )
1517 main.Mininet2.pingLong(
1518 src=main.params[ 'PING' ][ 'source5' ],
1519 target=main.params[ 'PING' ][ 'target5' ],
1520 pingTime=500 )
1521 main.Mininet2.pingLong(
1522 src=main.params[ 'PING' ][ 'source6' ],
1523 target=main.params[ 'PING' ][ 'target6' ],
1524 pingTime=500 )
1525 main.Mininet2.pingLong(
1526 src=main.params[ 'PING' ][ 'source7' ],
1527 target=main.params[ 'PING' ][ 'target7' ],
1528 pingTime=500 )
1529 main.Mininet2.pingLong(
1530 src=main.params[ 'PING' ][ 'source8' ],
1531 target=main.params[ 'PING' ][ 'target8' ],
1532 pingTime=500 )
1533 main.Mininet2.pingLong(
1534 src=main.params[ 'PING' ][ 'source9' ],
1535 target=main.params[ 'PING' ][ 'target9' ],
1536 pingTime=500 )
1537 main.Mininet2.pingLong(
1538 src=main.params[ 'PING' ][ 'source10' ],
1539 target=main.params[ 'PING' ][ 'target10' ],
1540 pingTime=500 )
1541
1542 main.step( "Collecting topology information from ONOS" )
1543 devices = []
1544 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001545 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001546 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001547 name="devices-" + str( i ),
1548 args=[ ] )
1549 threads.append( t )
1550 t.start()
1551
1552 for t in threads:
1553 t.join()
1554 devices.append( t.result )
1555 hosts = []
1556 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001557 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001558 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001559 name="hosts-" + str( i ),
1560 args=[ ] )
1561 threads.append( t )
1562 t.start()
1563
1564 for t in threads:
1565 t.join()
1566 try:
1567 hosts.append( json.loads( t.result ) )
1568 except ( ValueError, TypeError ):
1569 # FIXME: better handling of this, print which node
1570 # Maybe use thread name?
1571 main.log.exception( "Error parsing json output of hosts" )
Jon Hall3afe4c92015-12-14 19:30:38 -08001572 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001573 hosts.append( None )
1574
1575 ports = []
1576 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001577 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001578 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001579 name="ports-" + str( i ),
1580 args=[ ] )
1581 threads.append( t )
1582 t.start()
1583
1584 for t in threads:
1585 t.join()
1586 ports.append( t.result )
1587 links = []
1588 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001589 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001590 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001591 name="links-" + str( i ),
1592 args=[ ] )
1593 threads.append( t )
1594 t.start()
1595
1596 for t in threads:
1597 t.join()
1598 links.append( t.result )
1599 clusters = []
1600 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001601 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001602 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001603 name="clusters-" + str( i ),
1604 args=[ ] )
1605 threads.append( t )
1606 t.start()
1607
1608 for t in threads:
1609 t.join()
1610 clusters.append( t.result )
1611 # Compare json objects for hosts and dataplane clusters
1612
1613 # hosts
1614 main.step( "Host view is consistent across ONOS nodes" )
1615 consistentHostsResult = main.TRUE
1616 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001617 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001618 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001619 if hosts[ controller ] == hosts[ 0 ]:
1620 continue
1621 else: # hosts not consistent
1622 main.log.error( "hosts from ONOS" +
1623 controllerStr +
1624 " is inconsistent with ONOS1" )
1625 main.log.warn( repr( hosts[ controller ] ) )
1626 consistentHostsResult = main.FALSE
1627
1628 else:
1629 main.log.error( "Error in getting ONOS hosts from ONOS" +
1630 controllerStr )
1631 consistentHostsResult = main.FALSE
1632 main.log.warn( "ONOS" + controllerStr +
1633 " hosts response: " +
1634 repr( hosts[ controller ] ) )
1635 utilities.assert_equals(
1636 expect=main.TRUE,
1637 actual=consistentHostsResult,
1638 onpass="Hosts view is consistent across all ONOS nodes",
1639 onfail="ONOS nodes have different views of hosts" )
1640
1641 main.step( "Each host has an IP address" )
1642 ipResult = main.TRUE
1643 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001644 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001645 if hosts[ controller ]:
1646 for host in hosts[ controller ]:
1647 if not host.get( 'ipAddresses', [ ] ):
Jon Hallf3d16e72015-12-16 17:45:08 -08001648 main.log.error( "Error with host ips on controller" +
Jon Hall3afe4c92015-12-14 19:30:38 -08001649 controllerStr + ": " + str( host ) )
1650 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001651 utilities.assert_equals(
1652 expect=main.TRUE,
1653 actual=ipResult,
1654 onpass="The ips of the hosts aren't empty",
1655 onfail="The ip of at least one host is missing" )
1656
1657 # Strongly connected clusters of devices
1658 main.step( "Cluster view is consistent across ONOS nodes" )
1659 consistentClustersResult = main.TRUE
1660 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07001661 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001662 if "Error" not in clusters[ controller ]:
1663 if clusters[ controller ] == clusters[ 0 ]:
1664 continue
1665 else: # clusters not consistent
1666 main.log.error( "clusters from ONOS" + controllerStr +
1667 " is inconsistent with ONOS1" )
1668 consistentClustersResult = main.FALSE
1669
1670 else:
1671 main.log.error( "Error in getting dataplane clusters " +
1672 "from ONOS" + controllerStr )
1673 consistentClustersResult = main.FALSE
1674 main.log.warn( "ONOS" + controllerStr +
1675 " clusters response: " +
1676 repr( clusters[ controller ] ) )
1677 utilities.assert_equals(
1678 expect=main.TRUE,
1679 actual=consistentClustersResult,
1680 onpass="Clusters view is consistent across all ONOS nodes",
1681 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001682 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001683 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001684
Jon Hall5cf14d52015-07-16 12:15:19 -07001685 # there should always only be one cluster
1686 main.step( "Cluster view correct across ONOS nodes" )
1687 try:
1688 numClusters = len( json.loads( clusters[ 0 ] ) )
1689 except ( ValueError, TypeError ):
1690 main.log.exception( "Error parsing clusters[0]: " +
1691 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001692 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001693 clusterResults = main.FALSE
1694 if numClusters == 1:
1695 clusterResults = main.TRUE
1696 utilities.assert_equals(
1697 expect=1,
1698 actual=numClusters,
1699 onpass="ONOS shows 1 SCC",
1700 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1701
1702 main.step( "Comparing ONOS topology to MN" )
1703 devicesResults = main.TRUE
1704 linksResults = main.TRUE
1705 hostsResults = main.TRUE
1706 mnSwitches = main.Mininet1.getSwitches()
1707 mnLinks = main.Mininet1.getLinks()
1708 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001709 for controller in main.activeNodes:
1710 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001711 if devices[ controller ] and ports[ controller ] and\
1712 "Error" not in devices[ controller ] and\
1713 "Error" not in ports[ controller ]:
Jon Halle1a3b752015-07-22 13:02:46 -07001714 currentDevicesResult = main.Mininet1.compareSwitches(
1715 mnSwitches,
1716 json.loads( devices[ controller ] ),
1717 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001718 else:
1719 currentDevicesResult = main.FALSE
1720 utilities.assert_equals( expect=main.TRUE,
1721 actual=currentDevicesResult,
1722 onpass="ONOS" + controllerStr +
1723 " Switches view is correct",
1724 onfail="ONOS" + controllerStr +
1725 " Switches view is incorrect" )
1726 if links[ controller ] and "Error" not in links[ controller ]:
1727 currentLinksResult = main.Mininet1.compareLinks(
1728 mnSwitches, mnLinks,
1729 json.loads( links[ controller ] ) )
1730 else:
1731 currentLinksResult = main.FALSE
1732 utilities.assert_equals( expect=main.TRUE,
1733 actual=currentLinksResult,
1734 onpass="ONOS" + controllerStr +
1735 " links view is correct",
1736 onfail="ONOS" + controllerStr +
1737 " links view is incorrect" )
1738
Jon Hall657cdf62015-12-17 14:40:51 -08001739 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001740 currentHostsResult = main.Mininet1.compareHosts(
1741 mnHosts,
1742 hosts[ controller ] )
1743 else:
1744 currentHostsResult = main.FALSE
1745 utilities.assert_equals( expect=main.TRUE,
1746 actual=currentHostsResult,
1747 onpass="ONOS" + controllerStr +
1748 " hosts exist in Mininet",
1749 onfail="ONOS" + controllerStr +
1750 " hosts don't match Mininet" )
1751
1752 devicesResults = devicesResults and currentDevicesResult
1753 linksResults = linksResults and currentLinksResult
1754 hostsResults = hostsResults and currentHostsResult
1755
1756 main.step( "Device information is correct" )
1757 utilities.assert_equals(
1758 expect=main.TRUE,
1759 actual=devicesResults,
1760 onpass="Device information is correct",
1761 onfail="Device information is incorrect" )
1762
1763 main.step( "Links are correct" )
1764 utilities.assert_equals(
1765 expect=main.TRUE,
1766 actual=linksResults,
1767 onpass="Link are correct",
1768 onfail="Links are incorrect" )
1769
1770 main.step( "Hosts are correct" )
1771 utilities.assert_equals(
1772 expect=main.TRUE,
1773 actual=hostsResults,
1774 onpass="Hosts are correct",
1775 onfail="Hosts are incorrect" )
1776
1777 def CASE6( self, main ):
1778 """
1779 The Failure case.
1780 """
1781 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001782 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001783 assert main, "main not defined"
1784 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001785 assert main.CLIs, "main.CLIs not defined"
1786 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001787 try:
1788 labels
1789 except NameError:
1790 main.log.error( "labels not defined, setting to []" )
1791 global labels
1792 labels = []
1793 try:
1794 data
1795 except NameError:
1796 main.log.error( "data not defined, setting to []" )
1797 global data
1798 data = []
1799 # Reset non-persistent variables
1800 try:
1801 iCounterValue = 0
1802 except NameError:
1803 main.log.error( "iCounterValue not defined, setting to 0" )
1804 iCounterValue = 0
1805
1806 main.case( "Restart entire ONOS cluster" )
1807
Jon Hall5ec6b1b2015-09-17 18:20:14 -07001808 main.step( "Checking ONOS Logs for errors" )
1809 for node in main.nodes:
1810 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1811 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1812
Jon Hall5cf14d52015-07-16 12:15:19 -07001813 main.step( "Killing ONOS nodes" )
1814 killResults = main.TRUE
1815 killTime = time.time()
Jon Halle1a3b752015-07-22 13:02:46 -07001816 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001817 killed = main.ONOSbench.onosKill( node.ip_address )
1818 killResults = killResults and killed
1819 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1820 onpass="ONOS nodes killed",
1821 onfail="ONOS kill unsuccessful" )
1822
1823 main.step( "Checking if ONOS is up yet" )
1824 for i in range( 2 ):
1825 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001826 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001827 started = main.ONOSbench.isup( node.ip_address )
1828 if not started:
1829 main.log.error( node.name + " didn't start!" )
1830 onosIsupResult = onosIsupResult and started
1831 if onosIsupResult == main.TRUE:
1832 break
1833 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1834 onpass="ONOS restarted",
1835 onfail="ONOS restart NOT successful" )
1836
Jon Hall6509dbf2016-06-21 17:01:17 -07001837 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001838 cliResults = main.TRUE
1839 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001840 for i in range( main.numCtrls ):
1841 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -07001842 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -07001843 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001844 threads.append( t )
1845 t.start()
1846
1847 for t in threads:
1848 t.join()
1849 cliResults = cliResults and t.result
1850 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1851 onpass="ONOS cli started",
1852 onfail="ONOS clis did not restart" )
1853
Jon Hall6e709752016-02-01 13:38:46 -08001854 for i in range( 10 ):
1855 ready = True
Jon Hall7ac7bc32016-05-05 10:57:02 -07001856 for i in main.activeNodes:
1857 cli = main.CLIs[i]
Jon Hall6e709752016-02-01 13:38:46 -08001858 output = cli.summary()
1859 if not output:
1860 ready = False
Jon Halld2871c22016-07-26 11:01:14 -07001861 if ready:
1862 break
Jon Hall6e709752016-02-01 13:38:46 -08001863 time.sleep( 30 )
1864 utilities.assert_equals( expect=True, actual=ready,
1865 onpass="ONOS summary command succeded",
1866 onfail="ONOS summary command failed" )
1867 if not ready:
1868 main.cleanup()
1869 main.exit()
1870
Jon Hall5cf14d52015-07-16 12:15:19 -07001871 # Grab the time of restart so we chan check how long the gossip
1872 # protocol has had time to work
1873 main.restartTime = time.time() - killTime
1874 main.log.debug( "Restart time: " + str( main.restartTime ) )
1875 labels.append( "Restart" )
1876 data.append( str( main.restartTime ) )
1877
Jon Hall5cf14d52015-07-16 12:15:19 -07001878 # Rerun for election on restarted nodes
1879 runResults = main.TRUE
Jon Hall7ac7bc32016-05-05 10:57:02 -07001880 for i in main.activeNodes:
1881 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -07001882 run = cli.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001883 if run != main.TRUE:
1884 main.log.error( "Error running for election on " + cli.name )
1885 runResults = runResults and run
1886 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1887 onpass="Reran for election",
1888 onfail="Failed to rerun for election" )
1889
1890 # TODO: Make this configurable
1891 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -07001892 node = main.activeNodes[0]
1893 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1894 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1895 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001896
1897 def CASE7( self, main ):
1898 """
1899 Check state after ONOS failure
1900 """
1901 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001902 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001903 assert main, "main not defined"
1904 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001905 assert main.CLIs, "main.CLIs not defined"
1906 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001907 main.case( "Running ONOS Constant State Tests" )
1908
1909 main.step( "Check that each switch has a master" )
1910 # Assert that each device has a master
1911 rolesNotNull = main.TRUE
1912 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001913 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001914 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001915 name="rolesNotNull-" + str( i ),
1916 args=[ ] )
1917 threads.append( t )
1918 t.start()
1919
1920 for t in threads:
1921 t.join()
1922 rolesNotNull = rolesNotNull and t.result
1923 utilities.assert_equals(
1924 expect=main.TRUE,
1925 actual=rolesNotNull,
1926 onpass="Each device has a master",
1927 onfail="Some devices don't have a master assigned" )
1928
1929 main.step( "Read device roles from ONOS" )
1930 ONOSMastership = []
1931 mastershipCheck = main.FALSE
1932 consistentMastership = True
1933 rolesResults = True
1934 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001935 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001936 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001937 name="roles-" + str( i ),
1938 args=[] )
1939 threads.append( t )
1940 t.start()
1941
1942 for t in threads:
1943 t.join()
1944 ONOSMastership.append( t.result )
1945
Jon Halla440e872016-03-31 15:15:50 -07001946 for i in range( len( ONOSMastership ) ):
1947 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001948 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001949 main.log.error( "Error in getting ONOS" + node + " roles" )
1950 main.log.warn( "ONOS" + node + " mastership response: " +
1951 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001952 rolesResults = False
1953 utilities.assert_equals(
1954 expect=True,
1955 actual=rolesResults,
1956 onpass="No error in reading roles output",
1957 onfail="Error in reading roles from ONOS" )
1958
1959 main.step( "Check for consistency in roles from each controller" )
1960 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1961 main.log.info(
1962 "Switch roles are consistent across all ONOS nodes" )
1963 else:
1964 consistentMastership = False
1965 utilities.assert_equals(
1966 expect=True,
1967 actual=consistentMastership,
1968 onpass="Switch roles are consistent across all ONOS nodes",
1969 onfail="ONOS nodes have different views of switch roles" )
1970
1971 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001972 for i in range( len( ONOSMastership ) ):
1973 node = str( main.activeNodes[i] + 1 )
1974 main.log.warn( "ONOS" + node + " roles: ",
Jon Hall6e709752016-02-01 13:38:46 -08001975 json.dumps( json.loads( ONOSMastership[ i ] ),
1976 sort_keys=True,
1977 indent=4,
1978 separators=( ',', ': ' ) ) )
1979 elif rolesResults and consistentMastership:
Jon Hall5cf14d52015-07-16 12:15:19 -07001980 mastershipCheck = main.TRUE
1981
Jon Hall5cf14d52015-07-16 12:15:19 -07001982 # NOTE: we expect mastership to change on controller failure
1983
1984 main.step( "Get the intents and compare across all nodes" )
1985 ONOSIntents = []
1986 intentCheck = main.FALSE
1987 consistentIntents = True
1988 intentsResults = True
1989 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001990 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001991 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001992 name="intents-" + str( i ),
1993 args=[],
1994 kwargs={ 'jsonFormat': True } )
1995 threads.append( t )
1996 t.start()
1997
1998 for t in threads:
1999 t.join()
2000 ONOSIntents.append( t.result )
2001
Jon Halla440e872016-03-31 15:15:50 -07002002 for i in range( len( ONOSIntents) ):
2003 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002004 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07002005 main.log.error( "Error in getting ONOS" + node + " intents" )
2006 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002007 repr( ONOSIntents[ i ] ) )
2008 intentsResults = False
2009 utilities.assert_equals(
2010 expect=True,
2011 actual=intentsResults,
2012 onpass="No error in reading intents output",
2013 onfail="Error in reading intents from ONOS" )
2014
2015 main.step( "Check for consistency in Intents from each controller" )
2016 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2017 main.log.info( "Intents are consistent across all ONOS " +
2018 "nodes" )
2019 else:
2020 consistentIntents = False
2021
2022 # Try to make it easy to figure out what is happening
2023 #
2024 # Intent ONOS1 ONOS2 ...
2025 # 0x01 INSTALLED INSTALLING
2026 # ... ... ...
2027 # ... ... ...
2028 title = " ID"
Jon Halla440e872016-03-31 15:15:50 -07002029 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002030 title += " " * 10 + "ONOS" + str( n + 1 )
2031 main.log.warn( title )
2032 # get all intent keys in the cluster
2033 keys = []
2034 for nodeStr in ONOSIntents:
2035 node = json.loads( nodeStr )
2036 for intent in node:
2037 keys.append( intent.get( 'id' ) )
2038 keys = set( keys )
2039 for key in keys:
2040 row = "%-13s" % key
2041 for nodeStr in ONOSIntents:
2042 node = json.loads( nodeStr )
2043 for intent in node:
2044 if intent.get( 'id' ) == key:
2045 row += "%-15s" % intent.get( 'state' )
2046 main.log.warn( row )
2047 # End table view
2048
2049 utilities.assert_equals(
2050 expect=True,
2051 actual=consistentIntents,
2052 onpass="Intents are consistent across all ONOS nodes",
2053 onfail="ONOS nodes have different views of intents" )
2054 intentStates = []
2055 for node in ONOSIntents: # Iter through ONOS nodes
2056 nodeStates = []
2057 # Iter through intents of a node
2058 try:
2059 for intent in json.loads( node ):
2060 nodeStates.append( intent[ 'state' ] )
2061 except ( ValueError, TypeError ):
2062 main.log.exception( "Error in parsing intents" )
2063 main.log.error( repr( node ) )
2064 intentStates.append( nodeStates )
2065 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2066 main.log.info( dict( out ) )
2067
2068 if intentsResults and not consistentIntents:
Jon Halla440e872016-03-31 15:15:50 -07002069 for i in range( len( main.activeNodes ) ):
2070 node = str( main.activeNodes[i] + 1 )
2071 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002072 main.log.warn( json.dumps(
2073 json.loads( ONOSIntents[ i ] ),
2074 sort_keys=True,
2075 indent=4,
2076 separators=( ',', ': ' ) ) )
2077 elif intentsResults and consistentIntents:
2078 intentCheck = main.TRUE
2079
2080 # NOTE: Store has no durability, so intents are lost across system
2081 # restarts
2082 """
2083 main.step( "Compare current intents with intents before the failure" )
2084 # NOTE: this requires case 5 to pass for intentState to be set.
2085 # maybe we should stop the test if that fails?
2086 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002087 try:
2088 intentState
2089 except NameError:
2090 main.log.warn( "No previous intent state was saved" )
2091 else:
2092 if intentState and intentState == ONOSIntents[ 0 ]:
2093 sameIntents = main.TRUE
2094 main.log.info( "Intents are consistent with before failure" )
2095 # TODO: possibly the states have changed? we may need to figure out
2096 # what the acceptable states are
2097 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2098 sameIntents = main.TRUE
2099 try:
2100 before = json.loads( intentState )
2101 after = json.loads( ONOSIntents[ 0 ] )
2102 for intent in before:
2103 if intent not in after:
2104 sameIntents = main.FALSE
2105 main.log.debug( "Intent is not currently in ONOS " +
2106 "(at least in the same form):" )
2107 main.log.debug( json.dumps( intent ) )
2108 except ( ValueError, TypeError ):
2109 main.log.exception( "Exception printing intents" )
2110 main.log.debug( repr( ONOSIntents[0] ) )
2111 main.log.debug( repr( intentState ) )
2112 if sameIntents == main.FALSE:
2113 try:
2114 main.log.debug( "ONOS intents before: " )
2115 main.log.debug( json.dumps( json.loads( intentState ),
2116 sort_keys=True, indent=4,
2117 separators=( ',', ': ' ) ) )
2118 main.log.debug( "Current ONOS intents: " )
2119 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2120 sort_keys=True, indent=4,
2121 separators=( ',', ': ' ) ) )
2122 except ( ValueError, TypeError ):
2123 main.log.exception( "Exception printing intents" )
2124 main.log.debug( repr( ONOSIntents[0] ) )
2125 main.log.debug( repr( intentState ) )
2126 utilities.assert_equals(
2127 expect=main.TRUE,
2128 actual=sameIntents,
2129 onpass="Intents are consistent with before failure",
2130 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002131 intentCheck = intentCheck and sameIntents
2132 """
2133 main.step( "Get the OF Table entries and compare to before " +
2134 "component failure" )
2135 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002136 for i in range( 28 ):
2137 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002138 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002139 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2140 FlowTables = FlowTables and curSwitch
2141 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002142 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002143 utilities.assert_equals(
2144 expect=main.TRUE,
2145 actual=FlowTables,
2146 onpass="No changes were found in the flow tables",
2147 onfail="Changes were found in the flow tables" )
2148
2149 main.Mininet2.pingLongKill()
2150 '''
2151 # main.step( "Check the continuous pings to ensure that no packets " +
2152 # "were dropped during component failure" )
2153 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2154 main.params[ 'TESTONIP' ] )
2155 LossInPings = main.FALSE
2156 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2157 for i in range( 8, 18 ):
2158 main.log.info(
2159 "Checking for a loss in pings along flow from s" +
2160 str( i ) )
2161 LossInPings = main.Mininet2.checkForLoss(
2162 "/tmp/ping.h" +
2163 str( i ) ) or LossInPings
2164 if LossInPings == main.TRUE:
2165 main.log.info( "Loss in ping detected" )
2166 elif LossInPings == main.ERROR:
2167 main.log.info( "There are multiple mininet process running" )
2168 elif LossInPings == main.FALSE:
2169 main.log.info( "No Loss in the pings" )
2170 main.log.info( "No loss of dataplane connectivity" )
2171 # utilities.assert_equals(
2172 # expect=main.FALSE,
2173 # actual=LossInPings,
2174 # onpass="No Loss of connectivity",
2175 # onfail="Loss of dataplane connectivity detected" )
2176
2177 # NOTE: Since intents are not persisted with IntnentStore,
2178 # we expect loss in dataplane connectivity
2179 LossInPings = main.FALSE
2180 '''
2181
2182 main.step( "Leadership Election is still functional" )
2183 # Test of LeadershipElection
2184 leaderList = []
2185 leaderResult = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002186
2187 for i in main.activeNodes:
2188 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002189 leaderN = cli.electionTestLeader()
2190 leaderList.append( leaderN )
2191 if leaderN == main.FALSE:
2192 # error in response
2193 main.log.error( "Something is wrong with " +
2194 "electionTestLeader function, check the" +
2195 " error logs" )
2196 leaderResult = main.FALSE
2197 elif leaderN is None:
2198 main.log.error( cli.name +
2199 " shows no leader for the election-app." )
2200 leaderResult = main.FALSE
2201 if len( set( leaderList ) ) != 1:
2202 leaderResult = main.FALSE
2203 main.log.error(
2204 "Inconsistent view of leader for the election test app" )
2205 # TODO: print the list
2206 utilities.assert_equals(
2207 expect=main.TRUE,
2208 actual=leaderResult,
2209 onpass="Leadership election passed",
2210 onfail="Something went wrong with Leadership election" )
2211
2212 def CASE8( self, main ):
2213 """
2214 Compare topo
2215 """
2216 import json
2217 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002218 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002219 assert main, "main not defined"
2220 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002221 assert main.CLIs, "main.CLIs not defined"
2222 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002223
2224 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002225 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002226 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002227 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002228 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002229 elapsed = 0
2230 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002231 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002232 startTime = time.time()
2233 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002234 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hallba609822015-09-18 12:00:21 -07002235 devicesResults = main.TRUE
2236 linksResults = main.TRUE
2237 hostsResults = main.TRUE
2238 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002239 count += 1
2240 cliStart = time.time()
2241 devices = []
2242 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002243 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002244 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002245 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002246 args=[ main.CLIs[i].devices, [ None ] ],
2247 kwargs= { 'sleep': 5, 'attempts': 5,
2248 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002249 threads.append( t )
2250 t.start()
2251
2252 for t in threads:
2253 t.join()
2254 devices.append( t.result )
2255 hosts = []
2256 ipResult = main.TRUE
2257 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002258 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002259 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002260 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002261 args=[ main.CLIs[i].hosts, [ None ] ],
2262 kwargs= { 'sleep': 5, 'attempts': 5,
2263 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002264 threads.append( t )
2265 t.start()
2266
2267 for t in threads:
2268 t.join()
2269 try:
2270 hosts.append( json.loads( t.result ) )
2271 except ( ValueError, TypeError ):
2272 main.log.exception( "Error parsing hosts results" )
2273 main.log.error( repr( t.result ) )
Jon Hall3afe4c92015-12-14 19:30:38 -08002274 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002275 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002276 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002277 if hosts[ controller ]:
2278 for host in hosts[ controller ]:
2279 if host is None or host.get( 'ipAddresses', [] ) == []:
2280 main.log.error(
2281 "Error with host ipAddresses on controller" +
2282 controllerStr + ": " + str( host ) )
2283 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002284 ports = []
2285 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002286 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002287 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002288 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002289 args=[ main.CLIs[i].ports, [ None ] ],
2290 kwargs= { 'sleep': 5, 'attempts': 5,
2291 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002292 threads.append( t )
2293 t.start()
2294
2295 for t in threads:
2296 t.join()
2297 ports.append( t.result )
2298 links = []
2299 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002300 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002301 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002302 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002303 args=[ main.CLIs[i].links, [ None ] ],
2304 kwargs= { 'sleep': 5, 'attempts': 5,
2305 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002306 threads.append( t )
2307 t.start()
2308
2309 for t in threads:
2310 t.join()
2311 links.append( t.result )
2312 clusters = []
2313 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002314 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002315 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002316 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002317 args=[ main.CLIs[i].clusters, [ None ] ],
2318 kwargs= { 'sleep': 5, 'attempts': 5,
2319 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002320 threads.append( t )
2321 t.start()
2322
2323 for t in threads:
2324 t.join()
2325 clusters.append( t.result )
2326
2327 elapsed = time.time() - startTime
2328 cliTime = time.time() - cliStart
2329 print "Elapsed time: " + str( elapsed )
2330 print "CLI time: " + str( cliTime )
2331
Jon Hall6e709752016-02-01 13:38:46 -08002332 if all( e is None for e in devices ) and\
2333 all( e is None for e in hosts ) and\
2334 all( e is None for e in ports ) and\
2335 all( e is None for e in links ) and\
2336 all( e is None for e in clusters ):
2337 topoFailMsg = "Could not get topology from ONOS"
2338 main.log.error( topoFailMsg )
2339 continue # Try again, No use trying to compare
2340
Jon Hall5cf14d52015-07-16 12:15:19 -07002341 mnSwitches = main.Mininet1.getSwitches()
2342 mnLinks = main.Mininet1.getLinks()
2343 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07002344 for controller in range( len( main.activeNodes ) ):
2345 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002346 if devices[ controller ] and ports[ controller ] and\
2347 "Error" not in devices[ controller ] and\
2348 "Error" not in ports[ controller ]:
2349
Jon Hallc6793552016-01-19 14:18:37 -08002350 try:
2351 currentDevicesResult = main.Mininet1.compareSwitches(
2352 mnSwitches,
2353 json.loads( devices[ controller ] ),
2354 json.loads( ports[ controller ] ) )
2355 except ( TypeError, ValueError ) as e:
2356 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2357 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002358 else:
2359 currentDevicesResult = main.FALSE
2360 utilities.assert_equals( expect=main.TRUE,
2361 actual=currentDevicesResult,
2362 onpass="ONOS" + controllerStr +
2363 " Switches view is correct",
2364 onfail="ONOS" + controllerStr +
2365 " Switches view is incorrect" )
2366
2367 if links[ controller ] and "Error" not in links[ controller ]:
2368 currentLinksResult = main.Mininet1.compareLinks(
2369 mnSwitches, mnLinks,
2370 json.loads( links[ controller ] ) )
2371 else:
2372 currentLinksResult = main.FALSE
2373 utilities.assert_equals( expect=main.TRUE,
2374 actual=currentLinksResult,
2375 onpass="ONOS" + controllerStr +
2376 " links view is correct",
2377 onfail="ONOS" + controllerStr +
2378 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002379 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002380 currentHostsResult = main.Mininet1.compareHosts(
2381 mnHosts,
2382 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002383 elif hosts[ controller ] == []:
2384 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002385 else:
2386 currentHostsResult = main.FALSE
2387 utilities.assert_equals( expect=main.TRUE,
2388 actual=currentHostsResult,
2389 onpass="ONOS" + controllerStr +
2390 " hosts exist in Mininet",
2391 onfail="ONOS" + controllerStr +
2392 " hosts don't match Mininet" )
2393 # CHECKING HOST ATTACHMENT POINTS
2394 hostAttachment = True
Jon Halla440e872016-03-31 15:15:50 -07002395 zeroHosts = False
Jon Hall5cf14d52015-07-16 12:15:19 -07002396 # FIXME: topo-HA/obelisk specific mappings:
2397 # key is mac and value is dpid
2398 mappings = {}
2399 for i in range( 1, 29 ): # hosts 1 through 28
2400 # set up correct variables:
2401 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2402 if i == 1:
2403 deviceId = "1000".zfill(16)
2404 elif i == 2:
2405 deviceId = "2000".zfill(16)
2406 elif i == 3:
2407 deviceId = "3000".zfill(16)
2408 elif i == 4:
2409 deviceId = "3004".zfill(16)
2410 elif i == 5:
2411 deviceId = "5000".zfill(16)
2412 elif i == 6:
2413 deviceId = "6000".zfill(16)
2414 elif i == 7:
2415 deviceId = "6007".zfill(16)
2416 elif i >= 8 and i <= 17:
2417 dpid = '3' + str( i ).zfill( 3 )
2418 deviceId = dpid.zfill(16)
2419 elif i >= 18 and i <= 27:
2420 dpid = '6' + str( i ).zfill( 3 )
2421 deviceId = dpid.zfill(16)
2422 elif i == 28:
2423 deviceId = "2800".zfill(16)
2424 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002425 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002426 if hosts[ controller ] == []:
2427 main.log.warn( "There are no hosts discovered" )
Jon Halla440e872016-03-31 15:15:50 -07002428 zeroHosts = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002429 else:
2430 for host in hosts[ controller ]:
2431 mac = None
2432 location = None
2433 device = None
2434 port = None
2435 try:
2436 mac = host.get( 'mac' )
2437 assert mac, "mac field could not be found for this host object"
2438
2439 location = host.get( 'location' )
2440 assert location, "location field could not be found for this host object"
2441
2442 # Trim the protocol identifier off deviceId
2443 device = str( location.get( 'elementId' ) ).split(':')[1]
2444 assert device, "elementId field could not be found for this host location object"
2445
2446 port = location.get( 'port' )
2447 assert port, "port field could not be found for this host location object"
2448
2449 # Now check if this matches where they should be
2450 if mac and device and port:
2451 if str( port ) != "1":
2452 main.log.error( "The attachment port is incorrect for " +
2453 "host " + str( mac ) +
2454 ". Expected: 1 Actual: " + str( port) )
2455 hostAttachment = False
2456 if device != mappings[ str( mac ) ]:
2457 main.log.error( "The attachment device is incorrect for " +
2458 "host " + str( mac ) +
2459 ". Expected: " + mappings[ str( mac ) ] +
2460 " Actual: " + device )
2461 hostAttachment = False
2462 else:
2463 hostAttachment = False
2464 except AssertionError:
2465 main.log.exception( "Json object not as expected" )
2466 main.log.error( repr( host ) )
2467 hostAttachment = False
2468 else:
2469 main.log.error( "No hosts json output or \"Error\"" +
2470 " in output. hosts = " +
2471 repr( hosts[ controller ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002472 if zeroHosts is False:
Jon Hall5cf14d52015-07-16 12:15:19 -07002473 # TODO: Find a way to know if there should be hosts in a
2474 # given point of the test
2475 hostAttachment = True
2476
2477 # END CHECKING HOST ATTACHMENT POINTS
2478 devicesResults = devicesResults and currentDevicesResult
2479 linksResults = linksResults and currentLinksResult
2480 hostsResults = hostsResults and currentHostsResult
2481 hostAttachmentResults = hostAttachmentResults and\
2482 hostAttachment
2483 topoResult = ( devicesResults and linksResults
2484 and hostsResults and ipResult and
2485 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002486 utilities.assert_equals( expect=True,
2487 actual=topoResult,
2488 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002489 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002490 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002491
2492 # Compare json objects for hosts and dataplane clusters
2493
2494 # hosts
2495 main.step( "Hosts view is consistent across all ONOS nodes" )
2496 consistentHostsResult = main.TRUE
2497 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002498 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002499 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002500 if hosts[ controller ] == hosts[ 0 ]:
2501 continue
2502 else: # hosts not consistent
2503 main.log.error( "hosts from ONOS" + controllerStr +
2504 " is inconsistent with ONOS1" )
2505 main.log.warn( repr( hosts[ controller ] ) )
2506 consistentHostsResult = main.FALSE
2507
2508 else:
2509 main.log.error( "Error in getting ONOS hosts from ONOS" +
2510 controllerStr )
2511 consistentHostsResult = main.FALSE
2512 main.log.warn( "ONOS" + controllerStr +
2513 " hosts response: " +
2514 repr( hosts[ controller ] ) )
2515 utilities.assert_equals(
2516 expect=main.TRUE,
2517 actual=consistentHostsResult,
2518 onpass="Hosts view is consistent across all ONOS nodes",
2519 onfail="ONOS nodes have different views of hosts" )
2520
2521 main.step( "Hosts information is correct" )
2522 hostsResults = hostsResults and ipResult
2523 utilities.assert_equals(
2524 expect=main.TRUE,
2525 actual=hostsResults,
2526 onpass="Host information is correct",
2527 onfail="Host information is incorrect" )
2528
2529 main.step( "Host attachment points to the network" )
2530 utilities.assert_equals(
2531 expect=True,
2532 actual=hostAttachmentResults,
2533 onpass="Hosts are correctly attached to the network",
2534 onfail="ONOS did not correctly attach hosts to the network" )
2535
2536 # Strongly connected clusters of devices
2537 main.step( "Clusters view is consistent across all ONOS nodes" )
2538 consistentClustersResult = main.TRUE
2539 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07002540 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002541 if "Error" not in clusters[ controller ]:
2542 if clusters[ controller ] == clusters[ 0 ]:
2543 continue
2544 else: # clusters not consistent
2545 main.log.error( "clusters from ONOS" +
2546 controllerStr +
2547 " is inconsistent with ONOS1" )
2548 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002549 else:
2550 main.log.error( "Error in getting dataplane clusters " +
2551 "from ONOS" + controllerStr )
2552 consistentClustersResult = main.FALSE
2553 main.log.warn( "ONOS" + controllerStr +
2554 " clusters response: " +
2555 repr( clusters[ controller ] ) )
2556 utilities.assert_equals(
2557 expect=main.TRUE,
2558 actual=consistentClustersResult,
2559 onpass="Clusters view is consistent across all ONOS nodes",
2560 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002561 if not consistentClustersResult:
2562 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002563
2564 main.step( "There is only one SCC" )
2565 # there should always only be one cluster
2566 try:
2567 numClusters = len( json.loads( clusters[ 0 ] ) )
2568 except ( ValueError, TypeError ):
2569 main.log.exception( "Error parsing clusters[0]: " +
2570 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002571 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002572 clusterResults = main.FALSE
2573 if numClusters == 1:
2574 clusterResults = main.TRUE
2575 utilities.assert_equals(
2576 expect=1,
2577 actual=numClusters,
2578 onpass="ONOS shows 1 SCC",
2579 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2580
2581 topoResult = ( devicesResults and linksResults
2582 and hostsResults and consistentHostsResult
2583 and consistentClustersResult and clusterResults
2584 and ipResult and hostAttachmentResults )
2585
2586 topoResult = topoResult and int( count <= 2 )
2587 note = "note it takes about " + str( int( cliTime ) ) + \
2588 " seconds for the test to make all the cli calls to fetch " +\
2589 "the topology from each ONOS instance"
2590 main.log.info(
2591 "Very crass estimate for topology discovery/convergence( " +
2592 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2593 str( count ) + " tries" )
2594
2595 main.step( "Device information is correct" )
2596 utilities.assert_equals(
2597 expect=main.TRUE,
2598 actual=devicesResults,
2599 onpass="Device information is correct",
2600 onfail="Device information is incorrect" )
2601
2602 main.step( "Links are correct" )
2603 utilities.assert_equals(
2604 expect=main.TRUE,
2605 actual=linksResults,
2606 onpass="Link are correct",
2607 onfail="Links are incorrect" )
2608
Jon Halla440e872016-03-31 15:15:50 -07002609 main.step( "Hosts are correct" )
2610 utilities.assert_equals(
2611 expect=main.TRUE,
2612 actual=hostsResults,
2613 onpass="Hosts are correct",
2614 onfail="Hosts are incorrect" )
2615
Jon Hall5cf14d52015-07-16 12:15:19 -07002616 # FIXME: move this to an ONOS state case
2617 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002618 nodeResults = utilities.retry( main.HA.nodesCheck,
2619 False,
2620 args=[main.activeNodes],
2621 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002622
Jon Hall41d39f12016-04-11 22:54:35 -07002623 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002624 onpass="Nodes check successful",
2625 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002626 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002627 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002628 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002629 main.CLIs[i].name,
2630 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002631
Jon Halld2871c22016-07-26 11:01:14 -07002632 if not topoResult:
2633 main.cleanup()
2634 main.exit()
2635
Jon Hall5cf14d52015-07-16 12:15:19 -07002636 def CASE9( self, main ):
2637 """
2638 Link s3-s28 down
2639 """
2640 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002641 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002642 assert main, "main not defined"
2643 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002644 assert main.CLIs, "main.CLIs not defined"
2645 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002646 # NOTE: You should probably run a topology check after this
2647
2648 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2649
2650 description = "Turn off a link to ensure that Link Discovery " +\
2651 "is working properly"
2652 main.case( description )
2653
2654 main.step( "Kill Link between s3 and s28" )
2655 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2656 main.log.info( "Waiting " + str( linkSleep ) +
2657 " seconds for link down to be discovered" )
2658 time.sleep( linkSleep )
2659 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2660 onpass="Link down successful",
2661 onfail="Failed to bring link down" )
2662 # TODO do some sort of check here
2663
2664 def CASE10( self, main ):
2665 """
2666 Link s3-s28 up
2667 """
2668 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002669 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002670 assert main, "main not defined"
2671 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002672 assert main.CLIs, "main.CLIs not defined"
2673 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002674 # NOTE: You should probably run a topology check after this
2675
2676 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2677
2678 description = "Restore a link to ensure that Link Discovery is " + \
2679 "working properly"
2680 main.case( description )
2681
2682 main.step( "Bring link between s3 and s28 back up" )
2683 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2684 main.log.info( "Waiting " + str( linkSleep ) +
2685 " seconds for link up to be discovered" )
2686 time.sleep( linkSleep )
2687 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2688 onpass="Link up successful",
2689 onfail="Failed to bring link up" )
2690 # TODO do some sort of check here
2691
2692 def CASE11( self, main ):
2693 """
2694 Switch Down
2695 """
2696 # NOTE: You should probably run a topology check after this
2697 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002698 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002699 assert main, "main not defined"
2700 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002701 assert main.CLIs, "main.CLIs not defined"
2702 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002703
2704 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2705
2706 description = "Killing a switch to ensure it is discovered correctly"
Jon Halla440e872016-03-31 15:15:50 -07002707 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002708 main.case( description )
2709 switch = main.params[ 'kill' ][ 'switch' ]
2710 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2711
2712 # TODO: Make this switch parameterizable
2713 main.step( "Kill " + switch )
2714 main.log.info( "Deleting " + switch )
2715 main.Mininet1.delSwitch( switch )
2716 main.log.info( "Waiting " + str( switchSleep ) +
2717 " seconds for switch down to be discovered" )
2718 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002719 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002720 # Peek at the deleted switch
2721 main.log.warn( str( device ) )
2722 result = main.FALSE
2723 if device and device[ 'available' ] is False:
2724 result = main.TRUE
2725 utilities.assert_equals( expect=main.TRUE, actual=result,
2726 onpass="Kill switch successful",
2727 onfail="Failed to kill switch?" )
2728
2729 def CASE12( self, main ):
2730 """
2731 Switch Up
2732 """
2733 # NOTE: You should probably run a topology check after this
2734 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002735 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002736 assert main, "main not defined"
2737 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002738 assert main.CLIs, "main.CLIs not defined"
2739 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002740 assert ONOS1Port, "ONOS1Port not defined"
2741 assert ONOS2Port, "ONOS2Port not defined"
2742 assert ONOS3Port, "ONOS3Port not defined"
2743 assert ONOS4Port, "ONOS4Port not defined"
2744 assert ONOS5Port, "ONOS5Port not defined"
2745 assert ONOS6Port, "ONOS6Port not defined"
2746 assert ONOS7Port, "ONOS7Port not defined"
2747
2748 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2749 switch = main.params[ 'kill' ][ 'switch' ]
2750 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2751 links = main.params[ 'kill' ][ 'links' ].split()
Jon Halla440e872016-03-31 15:15:50 -07002752 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002753 description = "Adding a switch to ensure it is discovered correctly"
2754 main.case( description )
2755
2756 main.step( "Add back " + switch )
2757 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2758 for peer in links:
2759 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07002760 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002761 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2762 main.log.info( "Waiting " + str( switchSleep ) +
2763 " seconds for switch up to be discovered" )
2764 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002765 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002766 # Peek at the deleted switch
2767 main.log.warn( str( device ) )
2768 result = main.FALSE
2769 if device and device[ 'available' ]:
2770 result = main.TRUE
2771 utilities.assert_equals( expect=main.TRUE, actual=result,
2772 onpass="add switch successful",
2773 onfail="Failed to add switch?" )
2774
2775 def CASE13( self, main ):
2776 """
2777 Clean up
2778 """
2779 import os
2780 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002781 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002782 assert main, "main not defined"
2783 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002784 assert main.CLIs, "main.CLIs not defined"
2785 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002786
2787 # printing colors to terminal
2788 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2789 'blue': '\033[94m', 'green': '\033[92m',
2790 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2791 main.case( "Test Cleanup" )
2792 main.step( "Killing tcpdumps" )
2793 main.Mininet2.stopTcpdump()
2794
2795 testname = main.TEST
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002796 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002797 main.step( "Copying MN pcap and ONOS log files to test station" )
2798 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2799 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002800 # NOTE: MN Pcap file is being saved to logdir.
2801 # We scp this file as MN and TestON aren't necessarily the same vm
2802
2803 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002804 # TODO: Load these from params
2805 # NOTE: must end in /
2806 logFolder = "/opt/onos/log/"
2807 logFiles = [ "karaf.log", "karaf.log.1" ]
2808 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002809 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002810 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002811 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002812 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2813 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002814 # std*.log's
2815 # NOTE: must end in /
2816 logFolder = "/opt/onos/var/"
2817 logFiles = [ "stderr.log", "stdout.log" ]
2818 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002819 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002820 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002821 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002822 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2823 logFolder + f, dstName )
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002824 else:
2825 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002826
2827 main.step( "Stopping Mininet" )
2828 mnResult = main.Mininet1.stopNet()
2829 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2830 onpass="Mininet stopped",
2831 onfail="MN cleanup NOT successful" )
2832
2833 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002834 for node in main.nodes:
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002835 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2836 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002837
2838 try:
2839 timerLog = open( main.logdir + "/Timers.csv", 'w')
2840 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2841 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2842 timerLog.close()
2843 except NameError, e:
2844 main.log.exception(e)
2845
2846 def CASE14( self, main ):
2847 """
2848 start election app on all onos nodes
2849 """
Jon Halle1a3b752015-07-22 13:02:46 -07002850 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002851 assert main, "main not defined"
2852 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002853 assert main.CLIs, "main.CLIs not defined"
2854 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002855
2856 main.case("Start Leadership Election app")
2857 main.step( "Install leadership election app" )
Jon Halla440e872016-03-31 15:15:50 -07002858 onosCli = main.CLIs[ main.activeNodes[0] ]
2859 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002860 utilities.assert_equals(
2861 expect=main.TRUE,
2862 actual=appResult,
2863 onpass="Election app installed",
2864 onfail="Something went wrong with installing Leadership election" )
2865
2866 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002867 for i in main.activeNodes:
2868 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002869 time.sleep(5)
2870 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2871 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002872 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002873 expect=True,
2874 actual=sameResult,
2875 onpass="All nodes see the same leaderboards",
2876 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002877
Jon Hall25463a82016-04-13 14:03:52 -07002878 if sameResult:
2879 leader = leaders[ 0 ][ 0 ]
2880 if main.nodes[main.activeNodes[0]].ip_address in leader:
2881 correctLeader = True
2882 else:
2883 correctLeader = False
2884 main.step( "First node was elected leader" )
2885 utilities.assert_equals(
2886 expect=True,
2887 actual=correctLeader,
2888 onpass="Correct leader was elected",
2889 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002890
2891 def CASE15( self, main ):
2892 """
2893 Check that Leadership Election is still functional
acsmars9475b1c2015-08-28 18:02:08 -07002894 15.1 Run election on each node
2895 15.2 Check that each node has the same leaders and candidates
2896 15.3 Find current leader and withdraw
2897 15.4 Check that a new node was elected leader
2898 15.5 Check that that new leader was the candidate of old leader
2899 15.6 Run for election on old leader
2900 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2901 15.8 Make sure that the old leader was added to the candidate list
2902
2903 old and new variable prefixes refer to data from before vs after
2904 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002905 """
2906 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002907 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002908 assert main, "main not defined"
2909 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002910 assert main.CLIs, "main.CLIs not defined"
2911 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002912
Jon Hall5cf14d52015-07-16 12:15:19 -07002913 description = "Check that Leadership Election is still functional"
2914 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002915 # NOTE: Need to re-run after restarts since being a canidate is not persistant
acsmars9475b1c2015-08-28 18:02:08 -07002916
Jon Halla440e872016-03-31 15:15:50 -07002917 oldLeaders = [] # list of lists of each nodes' candidates before
2918 newLeaders = [] # list of lists of each nodes' candidates after
acsmars9475b1c2015-08-28 18:02:08 -07002919 oldLeader = '' # the old leader from oldLeaders, None if not same
2920 newLeader = '' # the new leaders fron newLoeaders, None if not same
2921 oldLeaderCLI = None # the CLI of the old leader used for re-electing
acsmars71adceb2015-08-31 15:09:26 -07002922 expectNoLeader = False # True when there is only one leader
2923 if main.numCtrls == 1:
2924 expectNoLeader = True
acsmars9475b1c2015-08-28 18:02:08 -07002925
Jon Hall5cf14d52015-07-16 12:15:19 -07002926 main.step( "Run for election on each node" )
acsmars9475b1c2015-08-28 18:02:08 -07002927 electionResult = main.TRUE
2928
Jon Halla440e872016-03-31 15:15:50 -07002929 for i in main.activeNodes: # run test election on each node
2930 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars9475b1c2015-08-28 18:02:08 -07002931 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002932 utilities.assert_equals(
2933 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002934 actual=electionResult,
2935 onpass="All nodes successfully ran for leadership",
2936 onfail="At least one node failed to run for leadership" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002937
acsmars3a72bde2015-09-02 14:16:22 -07002938 if electionResult == main.FALSE:
2939 main.log.error(
2940 "Skipping Test Case because Election Test App isn't loaded" )
2941 main.skipCase()
2942
acsmars9475b1c2015-08-28 18:02:08 -07002943 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002944 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002945 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002946 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002947 if sameResult:
2948 oldLeader = oldLeaders[ 0 ][ 0 ]
2949 main.log.warn( oldLeader )
acsmars9475b1c2015-08-28 18:02:08 -07002950 else:
Jon Halla440e872016-03-31 15:15:50 -07002951 oldLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002952 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002953 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002954 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002955 onpass="Leaderboards are consistent for the election topic",
acsmars9475b1c2015-08-28 18:02:08 -07002956 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002957
2958 main.step( "Find current leader and withdraw" )
acsmars9475b1c2015-08-28 18:02:08 -07002959 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002960 # do some sanity checking on leader before using it
acsmars9475b1c2015-08-28 18:02:08 -07002961 if oldLeader is None:
2962 main.log.error( "Leadership isn't consistent." )
2963 withdrawResult = main.FALSE
2964 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002965 for i in main.activeNodes:
acsmars9475b1c2015-08-28 18:02:08 -07002966 if oldLeader == main.nodes[ i ].ip_address:
2967 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002968 break
2969 else: # FOR/ELSE statement
2970 main.log.error( "Leader election, could not find current leader" )
2971 if oldLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002972 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002973 utilities.assert_equals(
2974 expect=main.TRUE,
2975 actual=withdrawResult,
2976 onpass="Node was withdrawn from election",
2977 onfail="Node was not withdrawn from election" )
2978
acsmars9475b1c2015-08-28 18:02:08 -07002979 main.step( "Check that a new node was elected leader" )
acsmars9475b1c2015-08-28 18:02:08 -07002980 failMessage = "Nodes have different leaders"
acsmars9475b1c2015-08-28 18:02:08 -07002981 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002982 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002983 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002984 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002985 if newLeaders[ 0 ][ 0 ] == 'none':
2986 main.log.error( "No leader was elected on at least 1 node" )
2987 if not expectNoLeader:
2988 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002989 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002990
acsmars9475b1c2015-08-28 18:02:08 -07002991 # Check that the new leader is not the older leader, which was withdrawn
2992 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002993 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08002994 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars9475b1c2015-08-28 18:02:08 -07002995 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002996 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002997 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07002998 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002999 onpass="Leadership election passed",
3000 onfail="Something went wrong with Leadership election" )
3001
Jon Halla440e872016-03-31 15:15:50 -07003002 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08003003 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars9475b1c2015-08-28 18:02:08 -07003004 correctCandidateResult = main.TRUE
acsmars71adceb2015-08-31 15:09:26 -07003005 if expectNoLeader:
3006 if newLeader == 'none':
3007 main.log.info( "No leader expected. None found. Pass" )
3008 correctCandidateResult = main.TRUE
3009 else:
3010 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3011 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003012 elif len( oldLeaders[0] ) >= 3:
3013 if newLeader == oldLeaders[ 0 ][ 2 ]:
3014 # correct leader was elected
3015 correctCandidateResult = main.TRUE
3016 else:
3017 correctCandidateResult = main.FALSE
3018 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3019 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003020 else:
3021 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003022 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003023 correctCandidateResult = main.FALSE
acsmars9475b1c2015-08-28 18:02:08 -07003024 utilities.assert_equals(
3025 expect=main.TRUE,
3026 actual=correctCandidateResult,
3027 onpass="Correct Candidate Elected",
3028 onfail="Incorrect Candidate Elected" )
3029
Jon Hall5cf14d52015-07-16 12:15:19 -07003030 main.step( "Run for election on old leader( just so everyone " +
3031 "is in the hat )" )
acsmars9475b1c2015-08-28 18:02:08 -07003032 if oldLeaderCLI is not None:
3033 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003034 else:
acsmars9475b1c2015-08-28 18:02:08 -07003035 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003036 runResult = main.FALSE
3037 utilities.assert_equals(
3038 expect=main.TRUE,
3039 actual=runResult,
3040 onpass="App re-ran for election",
3041 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003042
acsmars9475b1c2015-08-28 18:02:08 -07003043 main.step(
3044 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003045 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003046 # Get new leaders and candidates
3047 reRunLeaders = []
3048 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003049 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003050
acsmars9475b1c2015-08-28 18:02:08 -07003051 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07003052 if not reRunLeaders[0]:
3053 positionResult = main.FALSE
3054 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07003055 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3056 str( reRunLeaders[ 0 ] ) ) )
acsmars9475b1c2015-08-28 18:02:08 -07003057 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003058 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003059 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07003060 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003061 onpass="Old leader successfully re-ran for election",
3062 onfail="Something went wrong with Leadership election after " +
3063 "the old leader re-ran for election" )
3064
3065 def CASE16( self, main ):
3066 """
3067 Install Distributed Primitives app
3068 """
3069 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003070 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003071 assert main, "main not defined"
3072 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003073 assert main.CLIs, "main.CLIs not defined"
3074 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003075
3076 # Variables for the distributed primitives tests
3077 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003078 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003079 global onosSet
3080 global onosSetName
3081 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003082 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003083 onosSet = set([])
3084 onosSetName = "TestON-set"
3085
3086 description = "Install Primitives app"
3087 main.case( description )
3088 main.step( "Install Primitives app" )
3089 appName = "org.onosproject.distributedprimitives"
Jon Halla440e872016-03-31 15:15:50 -07003090 node = main.activeNodes[0]
3091 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003092 utilities.assert_equals( expect=main.TRUE,
3093 actual=appResults,
3094 onpass="Primitives app activated",
3095 onfail="Primitives app not activated" )
3096 time.sleep( 5 ) # To allow all nodes to activate
3097
3098 def CASE17( self, main ):
3099 """
3100 Check for basic functionality with distributed primitives
3101 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003102 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003103 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003104 assert main, "main not defined"
3105 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003106 assert main.CLIs, "main.CLIs not defined"
3107 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003108 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003109 assert onosSetName, "onosSetName not defined"
3110 # NOTE: assert fails if value is 0/None/Empty/False
3111 try:
3112 pCounterValue
3113 except NameError:
3114 main.log.error( "pCounterValue not defined, setting to 0" )
3115 pCounterValue = 0
3116 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003117 onosSet
3118 except NameError:
3119 main.log.error( "onosSet not defined, setting to empty Set" )
3120 onosSet = set([])
3121 # Variables for the distributed primitives tests. These are local only
3122 addValue = "a"
3123 addAllValue = "a b c d e f"
3124 retainValue = "c d e f"
3125
3126 description = "Check for basic functionality with distributed " +\
3127 "primitives"
3128 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003129 main.caseExplanation = "Test the methods of the distributed " +\
3130 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003131 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003132 # Partitioned counters
3133 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003134 pCounters = []
3135 threads = []
3136 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003137 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003138 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3139 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003140 args=[ pCounterName ] )
3141 pCounterValue += 1
3142 addedPValues.append( pCounterValue )
3143 threads.append( t )
3144 t.start()
3145
3146 for t in threads:
3147 t.join()
3148 pCounters.append( t.result )
3149 # Check that counter incremented numController times
3150 pCounterResults = True
3151 for i in addedPValues:
3152 tmpResult = i in pCounters
3153 pCounterResults = pCounterResults and tmpResult
3154 if not tmpResult:
3155 main.log.error( str( i ) + " is not in partitioned "
3156 "counter incremented results" )
3157 utilities.assert_equals( expect=True,
3158 actual=pCounterResults,
3159 onpass="Default counter incremented",
3160 onfail="Error incrementing default" +
3161 " counter" )
3162
Jon Halle1a3b752015-07-22 13:02:46 -07003163 main.step( "Get then Increment a default counter on each node" )
3164 pCounters = []
3165 threads = []
3166 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003167 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003168 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3169 name="counterGetAndAdd-" + str( i ),
3170 args=[ pCounterName ] )
3171 addedPValues.append( pCounterValue )
3172 pCounterValue += 1
3173 threads.append( t )
3174 t.start()
3175
3176 for t in threads:
3177 t.join()
3178 pCounters.append( t.result )
3179 # Check that counter incremented numController times
3180 pCounterResults = True
3181 for i in addedPValues:
3182 tmpResult = i in pCounters
3183 pCounterResults = pCounterResults and tmpResult
3184 if not tmpResult:
3185 main.log.error( str( i ) + " is not in partitioned "
3186 "counter incremented results" )
3187 utilities.assert_equals( expect=True,
3188 actual=pCounterResults,
3189 onpass="Default counter incremented",
3190 onfail="Error incrementing default" +
3191 " counter" )
3192
3193 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003194 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003195 utilities.assert_equals( expect=main.TRUE,
3196 actual=incrementCheck,
3197 onpass="Added counters are correct",
3198 onfail="Added counters are incorrect" )
3199
3200 main.step( "Add -8 to then get a default counter on each node" )
3201 pCounters = []
3202 threads = []
3203 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003204 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003205 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3206 name="counterIncrement-" + str( i ),
3207 args=[ pCounterName ],
3208 kwargs={ "delta": -8 } )
3209 pCounterValue += -8
3210 addedPValues.append( pCounterValue )
3211 threads.append( t )
3212 t.start()
3213
3214 for t in threads:
3215 t.join()
3216 pCounters.append( t.result )
3217 # Check that counter incremented numController times
3218 pCounterResults = True
3219 for i in addedPValues:
3220 tmpResult = i in pCounters
3221 pCounterResults = pCounterResults and tmpResult
3222 if not tmpResult:
3223 main.log.error( str( i ) + " is not in partitioned "
3224 "counter incremented results" )
3225 utilities.assert_equals( expect=True,
3226 actual=pCounterResults,
3227 onpass="Default counter incremented",
3228 onfail="Error incrementing default" +
3229 " counter" )
3230
3231 main.step( "Add 5 to then get a default counter on each node" )
3232 pCounters = []
3233 threads = []
3234 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003235 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003236 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3237 name="counterIncrement-" + str( i ),
3238 args=[ pCounterName ],
3239 kwargs={ "delta": 5 } )
3240 pCounterValue += 5
3241 addedPValues.append( pCounterValue )
3242 threads.append( t )
3243 t.start()
3244
3245 for t in threads:
3246 t.join()
3247 pCounters.append( t.result )
3248 # Check that counter incremented numController times
3249 pCounterResults = True
3250 for i in addedPValues:
3251 tmpResult = i in pCounters
3252 pCounterResults = pCounterResults and tmpResult
3253 if not tmpResult:
3254 main.log.error( str( i ) + " is not in partitioned "
3255 "counter incremented results" )
3256 utilities.assert_equals( expect=True,
3257 actual=pCounterResults,
3258 onpass="Default counter incremented",
3259 onfail="Error incrementing default" +
3260 " counter" )
3261
3262 main.step( "Get then add 5 to a default counter on each node" )
3263 pCounters = []
3264 threads = []
3265 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003266 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003267 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3268 name="counterIncrement-" + str( i ),
3269 args=[ pCounterName ],
3270 kwargs={ "delta": 5 } )
3271 addedPValues.append( pCounterValue )
3272 pCounterValue += 5
3273 threads.append( t )
3274 t.start()
3275
3276 for t in threads:
3277 t.join()
3278 pCounters.append( t.result )
3279 # Check that counter incremented numController times
3280 pCounterResults = True
3281 for i in addedPValues:
3282 tmpResult = i in pCounters
3283 pCounterResults = pCounterResults and tmpResult
3284 if not tmpResult:
3285 main.log.error( str( i ) + " is not in partitioned "
3286 "counter incremented results" )
3287 utilities.assert_equals( expect=True,
3288 actual=pCounterResults,
3289 onpass="Default counter incremented",
3290 onfail="Error incrementing default" +
3291 " counter" )
3292
3293 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003294 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003295 utilities.assert_equals( expect=main.TRUE,
3296 actual=incrementCheck,
3297 onpass="Added counters are correct",
3298 onfail="Added counters are incorrect" )
3299
Jon Hall5cf14d52015-07-16 12:15:19 -07003300 # DISTRIBUTED SETS
3301 main.step( "Distributed Set get" )
3302 size = len( onosSet )
3303 getResponses = []
3304 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003305 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003306 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003307 name="setTestGet-" + str( i ),
3308 args=[ onosSetName ] )
3309 threads.append( t )
3310 t.start()
3311 for t in threads:
3312 t.join()
3313 getResponses.append( t.result )
3314
3315 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003316 for i in range( len( main.activeNodes ) ):
3317 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003318 if isinstance( getResponses[ i ], list):
3319 current = set( getResponses[ i ] )
3320 if len( current ) == len( getResponses[ i ] ):
3321 # no repeats
3322 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003323 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003324 " has incorrect view" +
3325 " of set " + onosSetName + ":\n" +
3326 str( getResponses[ i ] ) )
3327 main.log.debug( "Expected: " + str( onosSet ) )
3328 main.log.debug( "Actual: " + str( current ) )
3329 getResults = main.FALSE
3330 else:
3331 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003332 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003333 " has repeat elements in" +
3334 " set " + onosSetName + ":\n" +
3335 str( getResponses[ i ] ) )
3336 getResults = main.FALSE
3337 elif getResponses[ i ] == main.ERROR:
3338 getResults = main.FALSE
3339 utilities.assert_equals( expect=main.TRUE,
3340 actual=getResults,
3341 onpass="Set elements are correct",
3342 onfail="Set elements are incorrect" )
3343
3344 main.step( "Distributed Set size" )
3345 sizeResponses = []
3346 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003347 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003348 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003349 name="setTestSize-" + str( i ),
3350 args=[ onosSetName ] )
3351 threads.append( t )
3352 t.start()
3353 for t in threads:
3354 t.join()
3355 sizeResponses.append( t.result )
3356
3357 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003358 for i in range( len( main.activeNodes ) ):
3359 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003360 if size != sizeResponses[ i ]:
3361 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003362 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003363 " expected a size of " + str( size ) +
3364 " for set " + onosSetName +
3365 " but got " + str( sizeResponses[ i ] ) )
3366 utilities.assert_equals( expect=main.TRUE,
3367 actual=sizeResults,
3368 onpass="Set sizes are correct",
3369 onfail="Set sizes are incorrect" )
3370
3371 main.step( "Distributed Set add()" )
3372 onosSet.add( addValue )
3373 addResponses = []
3374 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003375 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003376 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003377 name="setTestAdd-" + str( i ),
3378 args=[ onosSetName, addValue ] )
3379 threads.append( t )
3380 t.start()
3381 for t in threads:
3382 t.join()
3383 addResponses.append( t.result )
3384
3385 # main.TRUE = successfully changed the set
3386 # main.FALSE = action resulted in no change in set
3387 # main.ERROR - Some error in executing the function
3388 addResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003389 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003390 if addResponses[ i ] == main.TRUE:
3391 # All is well
3392 pass
3393 elif addResponses[ i ] == main.FALSE:
3394 # Already in set, probably fine
3395 pass
3396 elif addResponses[ i ] == main.ERROR:
3397 # Error in execution
3398 addResults = main.FALSE
3399 else:
3400 # unexpected result
3401 addResults = main.FALSE
3402 if addResults != main.TRUE:
3403 main.log.error( "Error executing set add" )
3404
3405 # Check if set is still correct
3406 size = len( onosSet )
3407 getResponses = []
3408 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003409 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003410 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003411 name="setTestGet-" + str( i ),
3412 args=[ onosSetName ] )
3413 threads.append( t )
3414 t.start()
3415 for t in threads:
3416 t.join()
3417 getResponses.append( t.result )
3418 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003419 for i in range( len( main.activeNodes ) ):
3420 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003421 if isinstance( getResponses[ i ], list):
3422 current = set( getResponses[ i ] )
3423 if len( current ) == len( getResponses[ i ] ):
3424 # no repeats
3425 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003426 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003427 " of set " + onosSetName + ":\n" +
3428 str( getResponses[ i ] ) )
3429 main.log.debug( "Expected: " + str( onosSet ) )
3430 main.log.debug( "Actual: " + str( current ) )
3431 getResults = main.FALSE
3432 else:
3433 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003434 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003435 " set " + onosSetName + ":\n" +
3436 str( getResponses[ i ] ) )
3437 getResults = main.FALSE
3438 elif getResponses[ i ] == main.ERROR:
3439 getResults = main.FALSE
3440 sizeResponses = []
3441 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003442 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003443 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003444 name="setTestSize-" + str( i ),
3445 args=[ onosSetName ] )
3446 threads.append( t )
3447 t.start()
3448 for t in threads:
3449 t.join()
3450 sizeResponses.append( t.result )
3451 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003452 for i in range( len( main.activeNodes ) ):
3453 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003454 if size != sizeResponses[ i ]:
3455 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003456 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003457 " expected a size of " + str( size ) +
3458 " for set " + onosSetName +
3459 " but got " + str( sizeResponses[ i ] ) )
3460 addResults = addResults and getResults and sizeResults
3461 utilities.assert_equals( expect=main.TRUE,
3462 actual=addResults,
3463 onpass="Set add correct",
3464 onfail="Set add was incorrect" )
3465
3466 main.step( "Distributed Set addAll()" )
3467 onosSet.update( addAllValue.split() )
3468 addResponses = []
3469 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003470 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003471 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003472 name="setTestAddAll-" + str( i ),
3473 args=[ onosSetName, addAllValue ] )
3474 threads.append( t )
3475 t.start()
3476 for t in threads:
3477 t.join()
3478 addResponses.append( t.result )
3479
3480 # main.TRUE = successfully changed the set
3481 # main.FALSE = action resulted in no change in set
3482 # main.ERROR - Some error in executing the function
3483 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003484 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003485 if addResponses[ i ] == main.TRUE:
3486 # All is well
3487 pass
3488 elif addResponses[ i ] == main.FALSE:
3489 # Already in set, probably fine
3490 pass
3491 elif addResponses[ i ] == main.ERROR:
3492 # Error in execution
3493 addAllResults = main.FALSE
3494 else:
3495 # unexpected result
3496 addAllResults = main.FALSE
3497 if addAllResults != main.TRUE:
3498 main.log.error( "Error executing set addAll" )
3499
3500 # Check if set is still correct
3501 size = len( onosSet )
3502 getResponses = []
3503 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003504 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003505 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003506 name="setTestGet-" + str( i ),
3507 args=[ onosSetName ] )
3508 threads.append( t )
3509 t.start()
3510 for t in threads:
3511 t.join()
3512 getResponses.append( t.result )
3513 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003514 for i in range( len( main.activeNodes ) ):
3515 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003516 if isinstance( getResponses[ i ], list):
3517 current = set( getResponses[ i ] )
3518 if len( current ) == len( getResponses[ i ] ):
3519 # no repeats
3520 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003521 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003522 " has incorrect view" +
3523 " of set " + onosSetName + ":\n" +
3524 str( getResponses[ i ] ) )
3525 main.log.debug( "Expected: " + str( onosSet ) )
3526 main.log.debug( "Actual: " + str( current ) )
3527 getResults = main.FALSE
3528 else:
3529 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003530 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003531 " has repeat elements in" +
3532 " set " + onosSetName + ":\n" +
3533 str( getResponses[ i ] ) )
3534 getResults = main.FALSE
3535 elif getResponses[ i ] == main.ERROR:
3536 getResults = main.FALSE
3537 sizeResponses = []
3538 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003539 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003540 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003541 name="setTestSize-" + str( i ),
3542 args=[ onosSetName ] )
3543 threads.append( t )
3544 t.start()
3545 for t in threads:
3546 t.join()
3547 sizeResponses.append( t.result )
3548 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003549 for i in range( len( main.activeNodes ) ):
3550 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003551 if size != sizeResponses[ i ]:
3552 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003553 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003554 " expected a size of " + str( size ) +
3555 " for set " + onosSetName +
3556 " but got " + str( sizeResponses[ i ] ) )
3557 addAllResults = addAllResults and getResults and sizeResults
3558 utilities.assert_equals( expect=main.TRUE,
3559 actual=addAllResults,
3560 onpass="Set addAll correct",
3561 onfail="Set addAll was incorrect" )
3562
3563 main.step( "Distributed Set contains()" )
3564 containsResponses = []
3565 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003566 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003567 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003568 name="setContains-" + str( i ),
3569 args=[ onosSetName ],
3570 kwargs={ "values": addValue } )
3571 threads.append( t )
3572 t.start()
3573 for t in threads:
3574 t.join()
3575 # NOTE: This is the tuple
3576 containsResponses.append( t.result )
3577
3578 containsResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003579 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003580 if containsResponses[ i ] == main.ERROR:
3581 containsResults = main.FALSE
3582 else:
3583 containsResults = containsResults and\
3584 containsResponses[ i ][ 1 ]
3585 utilities.assert_equals( expect=main.TRUE,
3586 actual=containsResults,
3587 onpass="Set contains is functional",
3588 onfail="Set contains failed" )
3589
3590 main.step( "Distributed Set containsAll()" )
3591 containsAllResponses = []
3592 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003593 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003594 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003595 name="setContainsAll-" + str( i ),
3596 args=[ onosSetName ],
3597 kwargs={ "values": addAllValue } )
3598 threads.append( t )
3599 t.start()
3600 for t in threads:
3601 t.join()
3602 # NOTE: This is the tuple
3603 containsAllResponses.append( t.result )
3604
3605 containsAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003606 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003607 if containsResponses[ i ] == main.ERROR:
3608 containsResults = main.FALSE
3609 else:
3610 containsResults = containsResults and\
3611 containsResponses[ i ][ 1 ]
3612 utilities.assert_equals( expect=main.TRUE,
3613 actual=containsAllResults,
3614 onpass="Set containsAll is functional",
3615 onfail="Set containsAll failed" )
3616
3617 main.step( "Distributed Set remove()" )
3618 onosSet.remove( addValue )
3619 removeResponses = []
3620 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003621 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003622 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003623 name="setTestRemove-" + str( i ),
3624 args=[ onosSetName, addValue ] )
3625 threads.append( t )
3626 t.start()
3627 for t in threads:
3628 t.join()
3629 removeResponses.append( t.result )
3630
3631 # main.TRUE = successfully changed the set
3632 # main.FALSE = action resulted in no change in set
3633 # main.ERROR - Some error in executing the function
3634 removeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003635 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003636 if removeResponses[ i ] == main.TRUE:
3637 # All is well
3638 pass
3639 elif removeResponses[ i ] == main.FALSE:
3640 # not in set, probably fine
3641 pass
3642 elif removeResponses[ i ] == main.ERROR:
3643 # Error in execution
3644 removeResults = main.FALSE
3645 else:
3646 # unexpected result
3647 removeResults = main.FALSE
3648 if removeResults != main.TRUE:
3649 main.log.error( "Error executing set remove" )
3650
3651 # Check if set is still correct
3652 size = len( onosSet )
3653 getResponses = []
3654 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003655 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003656 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003657 name="setTestGet-" + str( i ),
3658 args=[ onosSetName ] )
3659 threads.append( t )
3660 t.start()
3661 for t in threads:
3662 t.join()
3663 getResponses.append( t.result )
3664 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003665 for i in range( len( main.activeNodes ) ):
3666 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003667 if isinstance( getResponses[ i ], list):
3668 current = set( getResponses[ i ] )
3669 if len( current ) == len( getResponses[ i ] ):
3670 # no repeats
3671 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003672 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003673 " has incorrect view" +
3674 " of set " + onosSetName + ":\n" +
3675 str( getResponses[ i ] ) )
3676 main.log.debug( "Expected: " + str( onosSet ) )
3677 main.log.debug( "Actual: " + str( current ) )
3678 getResults = main.FALSE
3679 else:
3680 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003681 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003682 " has repeat elements in" +
3683 " set " + onosSetName + ":\n" +
3684 str( getResponses[ i ] ) )
3685 getResults = main.FALSE
3686 elif getResponses[ i ] == main.ERROR:
3687 getResults = main.FALSE
3688 sizeResponses = []
3689 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003690 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003691 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003692 name="setTestSize-" + str( i ),
3693 args=[ onosSetName ] )
3694 threads.append( t )
3695 t.start()
3696 for t in threads:
3697 t.join()
3698 sizeResponses.append( t.result )
3699 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003700 for i in range( len( main.activeNodes ) ):
3701 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003702 if size != sizeResponses[ i ]:
3703 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003704 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003705 " expected a size of " + str( size ) +
3706 " for set " + onosSetName +
3707 " but got " + str( sizeResponses[ i ] ) )
3708 removeResults = removeResults and getResults and sizeResults
3709 utilities.assert_equals( expect=main.TRUE,
3710 actual=removeResults,
3711 onpass="Set remove correct",
3712 onfail="Set remove was incorrect" )
3713
3714 main.step( "Distributed Set removeAll()" )
3715 onosSet.difference_update( addAllValue.split() )
3716 removeAllResponses = []
3717 threads = []
3718 try:
Jon Halla440e872016-03-31 15:15:50 -07003719 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003720 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003721 name="setTestRemoveAll-" + str( i ),
3722 args=[ onosSetName, addAllValue ] )
3723 threads.append( t )
3724 t.start()
3725 for t in threads:
3726 t.join()
3727 removeAllResponses.append( t.result )
3728 except Exception, e:
3729 main.log.exception(e)
3730
3731 # main.TRUE = successfully changed the set
3732 # main.FALSE = action resulted in no change in set
3733 # main.ERROR - Some error in executing the function
3734 removeAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003735 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003736 if removeAllResponses[ i ] == main.TRUE:
3737 # All is well
3738 pass
3739 elif removeAllResponses[ i ] == main.FALSE:
3740 # not in set, probably fine
3741 pass
3742 elif removeAllResponses[ i ] == main.ERROR:
3743 # Error in execution
3744 removeAllResults = main.FALSE
3745 else:
3746 # unexpected result
3747 removeAllResults = main.FALSE
3748 if removeAllResults != main.TRUE:
3749 main.log.error( "Error executing set removeAll" )
3750
3751 # Check if set is still correct
3752 size = len( onosSet )
3753 getResponses = []
3754 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003755 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003756 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003757 name="setTestGet-" + str( i ),
3758 args=[ onosSetName ] )
3759 threads.append( t )
3760 t.start()
3761 for t in threads:
3762 t.join()
3763 getResponses.append( t.result )
3764 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003765 for i in range( len( main.activeNodes ) ):
3766 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003767 if isinstance( getResponses[ i ], list):
3768 current = set( getResponses[ i ] )
3769 if len( current ) == len( getResponses[ i ] ):
3770 # no repeats
3771 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003772 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003773 " has incorrect view" +
3774 " of set " + onosSetName + ":\n" +
3775 str( getResponses[ i ] ) )
3776 main.log.debug( "Expected: " + str( onosSet ) )
3777 main.log.debug( "Actual: " + str( current ) )
3778 getResults = main.FALSE
3779 else:
3780 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003781 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003782 " has repeat elements in" +
3783 " set " + onosSetName + ":\n" +
3784 str( getResponses[ i ] ) )
3785 getResults = main.FALSE
3786 elif getResponses[ i ] == main.ERROR:
3787 getResults = main.FALSE
3788 sizeResponses = []
3789 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003790 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003791 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003792 name="setTestSize-" + str( i ),
3793 args=[ onosSetName ] )
3794 threads.append( t )
3795 t.start()
3796 for t in threads:
3797 t.join()
3798 sizeResponses.append( t.result )
3799 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003800 for i in range( len( main.activeNodes ) ):
3801 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003802 if size != sizeResponses[ i ]:
3803 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003804 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003805 " expected a size of " + str( size ) +
3806 " for set " + onosSetName +
3807 " but got " + str( sizeResponses[ i ] ) )
3808 removeAllResults = removeAllResults and getResults and sizeResults
3809 utilities.assert_equals( expect=main.TRUE,
3810 actual=removeAllResults,
3811 onpass="Set removeAll correct",
3812 onfail="Set removeAll was incorrect" )
3813
3814 main.step( "Distributed Set addAll()" )
3815 onosSet.update( addAllValue.split() )
3816 addResponses = []
3817 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003818 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003819 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003820 name="setTestAddAll-" + str( i ),
3821 args=[ onosSetName, addAllValue ] )
3822 threads.append( t )
3823 t.start()
3824 for t in threads:
3825 t.join()
3826 addResponses.append( t.result )
3827
3828 # main.TRUE = successfully changed the set
3829 # main.FALSE = action resulted in no change in set
3830 # main.ERROR - Some error in executing the function
3831 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003832 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003833 if addResponses[ i ] == main.TRUE:
3834 # All is well
3835 pass
3836 elif addResponses[ i ] == main.FALSE:
3837 # Already in set, probably fine
3838 pass
3839 elif addResponses[ i ] == main.ERROR:
3840 # Error in execution
3841 addAllResults = main.FALSE
3842 else:
3843 # unexpected result
3844 addAllResults = main.FALSE
3845 if addAllResults != main.TRUE:
3846 main.log.error( "Error executing set addAll" )
3847
3848 # Check if set is still correct
3849 size = len( onosSet )
3850 getResponses = []
3851 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003852 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003853 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003854 name="setTestGet-" + str( i ),
3855 args=[ onosSetName ] )
3856 threads.append( t )
3857 t.start()
3858 for t in threads:
3859 t.join()
3860 getResponses.append( t.result )
3861 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003862 for i in range( len( main.activeNodes ) ):
3863 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003864 if isinstance( getResponses[ i ], list):
3865 current = set( getResponses[ i ] )
3866 if len( current ) == len( getResponses[ i ] ):
3867 # no repeats
3868 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003869 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003870 " has incorrect view" +
3871 " of set " + onosSetName + ":\n" +
3872 str( getResponses[ i ] ) )
3873 main.log.debug( "Expected: " + str( onosSet ) )
3874 main.log.debug( "Actual: " + str( current ) )
3875 getResults = main.FALSE
3876 else:
3877 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003878 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003879 " has repeat elements in" +
3880 " set " + onosSetName + ":\n" +
3881 str( getResponses[ i ] ) )
3882 getResults = main.FALSE
3883 elif getResponses[ i ] == main.ERROR:
3884 getResults = main.FALSE
3885 sizeResponses = []
3886 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003887 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003888 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003889 name="setTestSize-" + str( i ),
3890 args=[ onosSetName ] )
3891 threads.append( t )
3892 t.start()
3893 for t in threads:
3894 t.join()
3895 sizeResponses.append( t.result )
3896 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003897 for i in range( len( main.activeNodes ) ):
3898 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003899 if size != sizeResponses[ i ]:
3900 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003901 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003902 " expected a size of " + str( size ) +
3903 " for set " + onosSetName +
3904 " but got " + str( sizeResponses[ i ] ) )
3905 addAllResults = addAllResults and getResults and sizeResults
3906 utilities.assert_equals( expect=main.TRUE,
3907 actual=addAllResults,
3908 onpass="Set addAll correct",
3909 onfail="Set addAll was incorrect" )
3910
3911 main.step( "Distributed Set clear()" )
3912 onosSet.clear()
3913 clearResponses = []
3914 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003915 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003916 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003917 name="setTestClear-" + str( i ),
3918 args=[ onosSetName, " "], # Values doesn't matter
3919 kwargs={ "clear": True } )
3920 threads.append( t )
3921 t.start()
3922 for t in threads:
3923 t.join()
3924 clearResponses.append( t.result )
3925
3926 # main.TRUE = successfully changed the set
3927 # main.FALSE = action resulted in no change in set
3928 # main.ERROR - Some error in executing the function
3929 clearResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003930 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003931 if clearResponses[ i ] == main.TRUE:
3932 # All is well
3933 pass
3934 elif clearResponses[ i ] == main.FALSE:
3935 # Nothing set, probably fine
3936 pass
3937 elif clearResponses[ i ] == main.ERROR:
3938 # Error in execution
3939 clearResults = main.FALSE
3940 else:
3941 # unexpected result
3942 clearResults = main.FALSE
3943 if clearResults != main.TRUE:
3944 main.log.error( "Error executing set clear" )
3945
3946 # Check if set is still correct
3947 size = len( onosSet )
3948 getResponses = []
3949 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003950 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003951 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003952 name="setTestGet-" + str( i ),
3953 args=[ onosSetName ] )
3954 threads.append( t )
3955 t.start()
3956 for t in threads:
3957 t.join()
3958 getResponses.append( t.result )
3959 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003960 for i in range( len( main.activeNodes ) ):
3961 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003962 if isinstance( getResponses[ i ], list):
3963 current = set( getResponses[ i ] )
3964 if len( current ) == len( getResponses[ i ] ):
3965 # no repeats
3966 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003967 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003968 " has incorrect view" +
3969 " of set " + onosSetName + ":\n" +
3970 str( getResponses[ i ] ) )
3971 main.log.debug( "Expected: " + str( onosSet ) )
3972 main.log.debug( "Actual: " + str( current ) )
3973 getResults = main.FALSE
3974 else:
3975 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003976 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003977 " has repeat elements in" +
3978 " set " + onosSetName + ":\n" +
3979 str( getResponses[ i ] ) )
3980 getResults = main.FALSE
3981 elif getResponses[ i ] == main.ERROR:
3982 getResults = main.FALSE
3983 sizeResponses = []
3984 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003985 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003986 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003987 name="setTestSize-" + str( i ),
3988 args=[ onosSetName ] )
3989 threads.append( t )
3990 t.start()
3991 for t in threads:
3992 t.join()
3993 sizeResponses.append( t.result )
3994 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003995 for i in range( len( main.activeNodes ) ):
3996 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003997 if size != sizeResponses[ i ]:
3998 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003999 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004000 " expected a size of " + str( size ) +
4001 " for set " + onosSetName +
4002 " but got " + str( sizeResponses[ i ] ) )
4003 clearResults = clearResults and getResults and sizeResults
4004 utilities.assert_equals( expect=main.TRUE,
4005 actual=clearResults,
4006 onpass="Set clear correct",
4007 onfail="Set clear was incorrect" )
4008
4009 main.step( "Distributed Set addAll()" )
4010 onosSet.update( addAllValue.split() )
4011 addResponses = []
4012 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004013 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004014 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004015 name="setTestAddAll-" + str( i ),
4016 args=[ onosSetName, addAllValue ] )
4017 threads.append( t )
4018 t.start()
4019 for t in threads:
4020 t.join()
4021 addResponses.append( t.result )
4022
4023 # main.TRUE = successfully changed the set
4024 # main.FALSE = action resulted in no change in set
4025 # main.ERROR - Some error in executing the function
4026 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004027 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004028 if addResponses[ i ] == main.TRUE:
4029 # All is well
4030 pass
4031 elif addResponses[ i ] == main.FALSE:
4032 # Already in set, probably fine
4033 pass
4034 elif addResponses[ i ] == main.ERROR:
4035 # Error in execution
4036 addAllResults = main.FALSE
4037 else:
4038 # unexpected result
4039 addAllResults = main.FALSE
4040 if addAllResults != main.TRUE:
4041 main.log.error( "Error executing set addAll" )
4042
4043 # Check if set is still correct
4044 size = len( onosSet )
4045 getResponses = []
4046 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004047 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004048 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004049 name="setTestGet-" + str( i ),
4050 args=[ onosSetName ] )
4051 threads.append( t )
4052 t.start()
4053 for t in threads:
4054 t.join()
4055 getResponses.append( t.result )
4056 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004057 for i in range( len( main.activeNodes ) ):
4058 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004059 if isinstance( getResponses[ i ], list):
4060 current = set( getResponses[ i ] )
4061 if len( current ) == len( getResponses[ i ] ):
4062 # no repeats
4063 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004064 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004065 " has incorrect view" +
4066 " of set " + onosSetName + ":\n" +
4067 str( getResponses[ i ] ) )
4068 main.log.debug( "Expected: " + str( onosSet ) )
4069 main.log.debug( "Actual: " + str( current ) )
4070 getResults = main.FALSE
4071 else:
4072 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004073 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004074 " has repeat elements in" +
4075 " set " + onosSetName + ":\n" +
4076 str( getResponses[ i ] ) )
4077 getResults = main.FALSE
4078 elif getResponses[ i ] == main.ERROR:
4079 getResults = main.FALSE
4080 sizeResponses = []
4081 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004082 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004083 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004084 name="setTestSize-" + str( i ),
4085 args=[ onosSetName ] )
4086 threads.append( t )
4087 t.start()
4088 for t in threads:
4089 t.join()
4090 sizeResponses.append( t.result )
4091 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004092 for i in range( len( main.activeNodes ) ):
4093 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004094 if size != sizeResponses[ i ]:
4095 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004096 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004097 " expected a size of " + str( size ) +
4098 " for set " + onosSetName +
4099 " but got " + str( sizeResponses[ i ] ) )
4100 addAllResults = addAllResults and getResults and sizeResults
4101 utilities.assert_equals( expect=main.TRUE,
4102 actual=addAllResults,
4103 onpass="Set addAll correct",
4104 onfail="Set addAll was incorrect" )
4105
4106 main.step( "Distributed Set retain()" )
4107 onosSet.intersection_update( retainValue.split() )
4108 retainResponses = []
4109 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004110 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004111 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004112 name="setTestRetain-" + str( i ),
4113 args=[ onosSetName, retainValue ],
4114 kwargs={ "retain": True } )
4115 threads.append( t )
4116 t.start()
4117 for t in threads:
4118 t.join()
4119 retainResponses.append( t.result )
4120
4121 # main.TRUE = successfully changed the set
4122 # main.FALSE = action resulted in no change in set
4123 # main.ERROR - Some error in executing the function
4124 retainResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004125 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004126 if retainResponses[ i ] == main.TRUE:
4127 # All is well
4128 pass
4129 elif retainResponses[ i ] == main.FALSE:
4130 # Already in set, probably fine
4131 pass
4132 elif retainResponses[ i ] == main.ERROR:
4133 # Error in execution
4134 retainResults = main.FALSE
4135 else:
4136 # unexpected result
4137 retainResults = main.FALSE
4138 if retainResults != main.TRUE:
4139 main.log.error( "Error executing set retain" )
4140
4141 # Check if set is still correct
4142 size = len( onosSet )
4143 getResponses = []
4144 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004145 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004146 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004147 name="setTestGet-" + str( i ),
4148 args=[ onosSetName ] )
4149 threads.append( t )
4150 t.start()
4151 for t in threads:
4152 t.join()
4153 getResponses.append( t.result )
4154 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004155 for i in range( len( main.activeNodes ) ):
4156 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004157 if isinstance( getResponses[ i ], list):
4158 current = set( getResponses[ i ] )
4159 if len( current ) == len( getResponses[ i ] ):
4160 # no repeats
4161 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004162 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004163 " has incorrect view" +
4164 " of set " + onosSetName + ":\n" +
4165 str( getResponses[ i ] ) )
4166 main.log.debug( "Expected: " + str( onosSet ) )
4167 main.log.debug( "Actual: " + str( current ) )
4168 getResults = main.FALSE
4169 else:
4170 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004171 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004172 " has repeat elements in" +
4173 " set " + onosSetName + ":\n" +
4174 str( getResponses[ i ] ) )
4175 getResults = main.FALSE
4176 elif getResponses[ i ] == main.ERROR:
4177 getResults = main.FALSE
4178 sizeResponses = []
4179 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004180 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004181 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004182 name="setTestSize-" + str( i ),
4183 args=[ onosSetName ] )
4184 threads.append( t )
4185 t.start()
4186 for t in threads:
4187 t.join()
4188 sizeResponses.append( t.result )
4189 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004190 for i in range( len( main.activeNodes ) ):
4191 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004192 if size != sizeResponses[ i ]:
4193 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004194 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004195 str( size ) + " for set " + onosSetName +
4196 " but got " + str( sizeResponses[ i ] ) )
4197 retainResults = retainResults and getResults and sizeResults
4198 utilities.assert_equals( expect=main.TRUE,
4199 actual=retainResults,
4200 onpass="Set retain correct",
4201 onfail="Set retain was incorrect" )
4202
Jon Hall2a5002c2015-08-21 16:49:11 -07004203 # Transactional maps
4204 main.step( "Partitioned Transactional maps put" )
4205 tMapValue = "Testing"
4206 numKeys = 100
4207 putResult = True
Jon Halla440e872016-03-31 15:15:50 -07004208 node = main.activeNodes[0]
4209 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004210 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004211 for i in putResponses:
4212 if putResponses[ i ][ 'value' ] != tMapValue:
4213 putResult = False
4214 else:
4215 putResult = False
4216 if not putResult:
4217 main.log.debug( "Put response values: " + str( putResponses ) )
4218 utilities.assert_equals( expect=True,
4219 actual=putResult,
4220 onpass="Partitioned Transactional Map put successful",
4221 onfail="Partitioned Transactional Map put values are incorrect" )
4222
4223 main.step( "Partitioned Transactional maps get" )
Jon Hall9bfadd22016-05-11 14:48:07 -07004224 # FIXME: is this sleep needed?
4225 time.sleep( 5 )
4226
Jon Hall2a5002c2015-08-21 16:49:11 -07004227 getCheck = True
4228 for n in range( 1, numKeys + 1 ):
4229 getResponses = []
4230 threads = []
4231 valueCheck = True
Jon Halla440e872016-03-31 15:15:50 -07004232 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004233 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4234 name="TMap-get-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08004235 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004236 threads.append( t )
4237 t.start()
4238 for t in threads:
4239 t.join()
4240 getResponses.append( t.result )
4241 for node in getResponses:
4242 if node != tMapValue:
4243 valueCheck = False
4244 if not valueCheck:
4245 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4246 main.log.warn( getResponses )
4247 getCheck = getCheck and valueCheck
4248 utilities.assert_equals( expect=True,
4249 actual=getCheck,
4250 onpass="Partitioned Transactional Map get values were correct",
4251 onfail="Partitioned Transactional Map values incorrect" )