blob: f903f9563e82e5b9abb7a097488654b707673d55 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAclusterRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080051 import time
Jon Hall5cf14d52015-07-16 12:15:19 -070052 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
53 "initialization" )
54 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070055 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "installing ONOS, starting Mininet and ONOS" +\
57 "cli sessions."
58 # TODO: save all the timers and output them for plotting
59
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
71 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79 # These are for csv plotting in jenkins
80 global labels
81 global data
82 labels = []
83 data = []
84
85 # FIXME: just get controller port from params?
86 # TODO: do we really need all these?
87 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
88 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
89 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
90 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
91 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
92 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
93 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
94
Jon Halle1a3b752015-07-22 13:02:46 -070095 try:
96 fileName = "Counters"
97 path = main.params[ 'imports' ][ 'path' ]
98 main.Counters = imp.load_source( fileName,
99 path + fileName + ".py" )
100 except Exception as e:
101 main.log.exception( e )
102 main.cleanup()
103 main.exit()
104
105 main.CLIs = []
106 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700107 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700108 for i in range( 1, main.numCtrls + 1 ):
109 try:
110 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
111 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
112 ipList.append( main.nodes[ -1 ].ip_address )
113 except AttributeError:
114 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700115
116 main.step( "Create cell file" )
117 cellAppString = main.params[ 'ENV' ][ 'appString' ]
118 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
119 main.Mininet1.ip_address,
120 cellAppString, ipList )
121 main.step( "Applying cell variable to environment" )
122 cellResult = main.ONOSbench.setCell( cellName )
123 verifyResult = main.ONOSbench.verifyCell()
124
125 # FIXME:this is short term fix
126 main.log.info( "Removing raft logs" )
127 main.ONOSbench.onosRemoveRaftLogs()
128
129 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700130 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700131 main.ONOSbench.onosUninstall( node.ip_address )
132
133 # Make sure ONOS is DEAD
134 main.log.info( "Killing any ONOS processes" )
135 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700136 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700137 killed = main.ONOSbench.onosKill( node.ip_address )
138 killResults = killResults and killed
139
140 cleanInstallResult = main.TRUE
141 gitPullResult = main.TRUE
142
143 main.step( "Starting Mininet" )
144 # scp topo file to mininet
145 # TODO: move to params?
146 topoName = "obelisk.py"
147 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700148 main.ONOSbench.scp( main.Mininet1,
149 filePath + topoName,
150 main.Mininet1.home,
151 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700152 mnResult = main.Mininet1.startNet( )
153 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
154 onpass="Mininet Started",
155 onfail="Error starting Mininet" )
156
157 main.step( "Git checkout and pull " + gitBranch )
158 if PULLCODE:
159 main.ONOSbench.gitCheckout( gitBranch )
160 gitPullResult = main.ONOSbench.gitPull()
161 # values of 1 or 3 are good
162 utilities.assert_lesser( expect=0, actual=gitPullResult,
163 onpass="Git pull successful",
164 onfail="Git pull failed" )
165 main.ONOSbench.getVersion( report=True )
166
167 main.step( "Using mvn clean install" )
168 cleanInstallResult = main.TRUE
169 if PULLCODE and gitPullResult == main.TRUE:
170 cleanInstallResult = main.ONOSbench.cleanInstall()
171 else:
172 main.log.warn( "Did not pull new code so skipping mvn " +
173 "clean install" )
174 utilities.assert_equals( expect=main.TRUE,
175 actual=cleanInstallResult,
176 onpass="MCI successful",
177 onfail="MCI failed" )
178 # GRAPHS
179 # NOTE: important params here:
180 # job = name of Jenkins job
181 # Plot Name = Plot-HA, only can be used if multiple plots
182 # index = The number of the graph under plot name
183 job = "HAclusterRestart"
184 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700185 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700186 graphs = '<ac:structured-macro ac:name="html">\n'
187 graphs += '<ac:plain-text-body><![CDATA[\n'
188 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800189 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700190 '&width=500&height=300"' +\
191 'noborder="0" width="500" height="300" scrolling="yes" ' +\
192 'seamless="seamless"></iframe>\n'
193 graphs += ']]></ac:plain-text-body>\n'
194 graphs += '</ac:structured-macro>\n'
195 main.log.wiki(graphs)
196
197 main.step( "Creating ONOS package" )
198 packageResult = main.ONOSbench.onosPackage()
199 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
200 onpass="ONOS package successful",
201 onfail="ONOS package failed" )
202
203 main.step( "Installing ONOS package" )
204 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700205 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700206 tmpResult = main.ONOSbench.onosInstall( options="-f",
207 node=node.ip_address )
208 onosInstallResult = onosInstallResult and tmpResult
209 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
210 onpass="ONOS install successful",
211 onfail="ONOS install failed" )
212
213 main.step( "Checking if ONOS is up yet" )
214 for i in range( 2 ):
215 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700216 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700217 started = main.ONOSbench.isup( node.ip_address )
218 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800219 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700220 onosIsupResult = onosIsupResult and started
221 if onosIsupResult == main.TRUE:
222 break
223 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
224 onpass="ONOS startup successful",
225 onfail="ONOS startup failed" )
226
227 main.log.step( "Starting ONOS CLI sessions" )
228 cliResults = main.TRUE
229 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700230 for i in range( main.numCtrls ):
231 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700232 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700233 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700234 threads.append( t )
235 t.start()
236
237 for t in threads:
238 t.join()
239 cliResults = cliResults and t.result
240 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
241 onpass="ONOS cli startup successful",
242 onfail="ONOS cli startup failed" )
243
244 if main.params[ 'tcpdump' ].lower() == "true":
245 main.step( "Start Packet Capture MN" )
246 main.Mininet2.startTcpdump(
247 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
248 + "-MN.pcap",
249 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
250 port=main.params[ 'MNtcpdump' ][ 'port' ] )
251
252 main.step( "App Ids check" )
Jon Hallf3d16e72015-12-16 17:45:08 -0800253 time.sleep(60)
Jon Hall5cf14d52015-07-16 12:15:19 -0700254 appCheck = main.TRUE
255 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700256 for i in range( main.numCtrls ):
257 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700258 name="appToIDCheck-" + str( i ),
259 args=[] )
260 threads.append( t )
261 t.start()
262
263 for t in threads:
264 t.join()
265 appCheck = appCheck and t.result
266 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700267 main.log.warn( main.CLIs[0].apps() )
268 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700269 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
270 onpass="App Ids seem to be correct",
271 onfail="Something is wrong with app Ids" )
272
273 if cliResults == main.FALSE:
274 main.log.error( "Failed to start ONOS, stopping test" )
275 main.cleanup()
276 main.exit()
277
278 def CASE2( self, main ):
279 """
280 Assign devices to controllers
281 """
282 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700283 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700284 assert main, "main not defined"
285 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700286 assert main.CLIs, "main.CLIs not defined"
287 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700288 assert ONOS1Port, "ONOS1Port not defined"
289 assert ONOS2Port, "ONOS2Port not defined"
290 assert ONOS3Port, "ONOS3Port not defined"
291 assert ONOS4Port, "ONOS4Port not defined"
292 assert ONOS5Port, "ONOS5Port not defined"
293 assert ONOS6Port, "ONOS6Port not defined"
294 assert ONOS7Port, "ONOS7Port not defined"
295
296 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700297 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700298 "and check that an ONOS node becomes the " +\
299 "master of the device."
300 main.step( "Assign switches to controllers" )
301
302 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700303 for i in range( main.numCtrls ):
304 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700305 swList = []
306 for i in range( 1, 29 ):
307 swList.append( "s" + str( i ) )
308 main.Mininet1.assignSwController( sw=swList, ip=ipList )
309
310 mastershipCheck = main.TRUE
311 for i in range( 1, 29 ):
312 response = main.Mininet1.getSwController( "s" + str( i ) )
313 try:
314 main.log.info( str( response ) )
315 except Exception:
316 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700317 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700318 if re.search( "tcp:" + node.ip_address, response ):
319 mastershipCheck = mastershipCheck and main.TRUE
320 else:
321 main.log.error( "Error, node " + node.ip_address + " is " +
322 "not in the list of controllers s" +
323 str( i ) + " is connecting to." )
324 mastershipCheck = main.FALSE
325 utilities.assert_equals(
326 expect=main.TRUE,
327 actual=mastershipCheck,
328 onpass="Switch mastership assigned correctly",
329 onfail="Switches not assigned correctly to controllers" )
330
331 def CASE21( self, main ):
332 """
333 Assign mastership to controllers
334 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700335 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700336 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700337 assert main, "main not defined"
338 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700339 assert main.CLIs, "main.CLIs not defined"
340 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700341 assert ONOS1Port, "ONOS1Port not defined"
342 assert ONOS2Port, "ONOS2Port not defined"
343 assert ONOS3Port, "ONOS3Port not defined"
344 assert ONOS4Port, "ONOS4Port not defined"
345 assert ONOS5Port, "ONOS5Port not defined"
346 assert ONOS6Port, "ONOS6Port not defined"
347 assert ONOS7Port, "ONOS7Port not defined"
348
349 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700350 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700351 "device. Then manually assign" +\
352 " mastership to specific ONOS nodes using" +\
353 " 'device-role'"
354 main.step( "Assign mastership of switches to specific controllers" )
355 # Manually assign mastership to the controller we want
356 roleCall = main.TRUE
357
358 ipList = [ ]
359 deviceList = []
360 try:
361 # Assign mastership to specific controllers. This assignment was
362 # determined for a 7 node cluser, but will work with any sized
363 # cluster
364 for i in range( 1, 29 ): # switches 1 through 28
365 # set up correct variables:
366 if i == 1:
367 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700368 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700369 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
370 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700371 c = 1 % main.numCtrls
372 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700373 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
374 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700375 c = 1 % main.numCtrls
376 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700377 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
378 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700379 c = 3 % main.numCtrls
380 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700381 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
382 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700383 c = 2 % main.numCtrls
384 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700385 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
386 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700387 c = 2 % main.numCtrls
388 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700389 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
390 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700391 c = 5 % main.numCtrls
392 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700393 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
394 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700395 c = 4 % main.numCtrls
396 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700397 dpid = '3' + str( i ).zfill( 3 )
398 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
399 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700400 c = 6 % main.numCtrls
401 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700402 dpid = '6' + str( i ).zfill( 3 )
403 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
404 elif i == 28:
405 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700406 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700407 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
408 else:
409 main.log.error( "You didn't write an else statement for " +
410 "switch s" + str( i ) )
411 roleCall = main.FALSE
412 # Assign switch
413 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
414 # TODO: make this controller dynamic
415 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
416 ip )
417 ipList.append( ip )
418 deviceList.append( deviceId )
419 except ( AttributeError, AssertionError ):
420 main.log.exception( "Something is wrong with ONOS device view" )
421 main.log.info( main.ONOScli1.devices() )
422 utilities.assert_equals(
423 expect=main.TRUE,
424 actual=roleCall,
425 onpass="Re-assigned switch mastership to designated controller",
426 onfail="Something wrong with deviceRole calls" )
427
428 main.step( "Check mastership was correctly assigned" )
429 roleCheck = main.TRUE
430 # NOTE: This is due to the fact that device mastership change is not
431 # atomic and is actually a multi step process
432 time.sleep( 5 )
433 for i in range( len( ipList ) ):
434 ip = ipList[i]
435 deviceId = deviceList[i]
436 # Check assignment
437 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
438 if ip in master:
439 roleCheck = roleCheck and main.TRUE
440 else:
441 roleCheck = roleCheck and main.FALSE
442 main.log.error( "Error, controller " + ip + " is not" +
443 " master " + "of device " +
444 str( deviceId ) + ". Master is " +
445 repr( master ) + "." )
446 utilities.assert_equals(
447 expect=main.TRUE,
448 actual=roleCheck,
449 onpass="Switches were successfully reassigned to designated " +
450 "controller",
451 onfail="Switches were not successfully reassigned" )
452
453 def CASE3( self, main ):
454 """
455 Assign intents
456 """
457 import time
458 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700459 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700460 assert main, "main not defined"
461 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700462 assert main.CLIs, "main.CLIs not defined"
463 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700464 try:
465 labels
466 except NameError:
467 main.log.error( "labels not defined, setting to []" )
468 labels = []
469 try:
470 data
471 except NameError:
472 main.log.error( "data not defined, setting to []" )
473 data = []
474 # NOTE: we must reinstall intents until we have a persistant intent
475 # datastore!
476 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700477 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700478 "assign predetermined host-to-host intents." +\
479 " After installation, check that the intent" +\
480 " is distributed to all nodes and the state" +\
481 " is INSTALLED"
482
483 # install onos-app-fwd
484 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700485 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700486 utilities.assert_equals( expect=main.TRUE, actual=installResults,
487 onpass="Install fwd successful",
488 onfail="Install fwd failed" )
489
490 main.step( "Check app ids" )
491 appCheck = main.TRUE
492 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700493 for i in range( main.numCtrls ):
494 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700495 name="appToIDCheck-" + str( i ),
496 args=[] )
497 threads.append( t )
498 t.start()
499
500 for t in threads:
501 t.join()
502 appCheck = appCheck and t.result
503 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700504 main.log.warn( main.CLIs[0].apps() )
505 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700506 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
507 onpass="App Ids seem to be correct",
508 onfail="Something is wrong with app Ids" )
509
510 main.step( "Discovering Hosts( Via pingall for now )" )
511 # FIXME: Once we have a host discovery mechanism, use that instead
512 # REACTIVE FWD test
513 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700514 passMsg = "Reactive Pingall test passed"
515 time1 = time.time()
516 pingResult = main.Mininet1.pingall()
517 time2 = time.time()
518 if not pingResult:
519 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700520 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700521 passMsg += " on the second try"
522 utilities.assert_equals(
523 expect=main.TRUE,
524 actual=pingResult,
525 onpass= passMsg,
526 onfail="Reactive Pingall failed, " +
527 "one or more ping pairs failed" )
528 main.log.info( "Time for pingall: %2f seconds" %
529 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700530 # timeout for fwd flows
531 time.sleep( 11 )
532 # uninstall onos-app-fwd
533 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700534 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700535 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
536 onpass="Uninstall fwd successful",
537 onfail="Uninstall fwd failed" )
538
539 main.step( "Check app ids" )
540 threads = []
541 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700542 for i in range( main.numCtrls ):
543 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700544 name="appToIDCheck-" + str( i ),
545 args=[] )
546 threads.append( t )
547 t.start()
548
549 for t in threads:
550 t.join()
551 appCheck2 = appCheck2 and t.result
552 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700553 main.log.warn( main.CLIs[0].apps() )
554 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700555 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
556 onpass="App Ids seem to be correct",
557 onfail="Something is wrong with app Ids" )
558
559 main.step( "Add host intents via cli" )
560 intentIds = []
Jon Hall6e709752016-02-01 13:38:46 -0800561 # TODO: move the host numbers to params
562 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700563 intentAddResult = True
564 hostResult = main.TRUE
565 for i in range( 8, 18 ):
566 main.log.info( "Adding host intent between h" + str( i ) +
567 " and h" + str( i + 10 ) )
568 host1 = "00:00:00:00:00:" + \
569 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
570 host2 = "00:00:00:00:00:" + \
571 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
572 # NOTE: getHost can return None
573 host1Dict = main.ONOScli1.getHost( host1 )
574 host2Dict = main.ONOScli1.getHost( host2 )
575 host1Id = None
576 host2Id = None
577 if host1Dict and host2Dict:
578 host1Id = host1Dict.get( 'id', None )
579 host2Id = host2Dict.get( 'id', None )
580 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700581 nodeNum = ( i % main.numCtrls )
582 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700583 if tmpId:
584 main.log.info( "Added intent with id: " + tmpId )
585 intentIds.append( tmpId )
586 else:
587 main.log.error( "addHostIntent returned: " +
588 repr( tmpId ) )
589 else:
590 main.log.error( "Error, getHost() failed for h" + str( i ) +
591 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700592 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700593 main.log.warn( "Hosts output: " )
594 try:
595 main.log.warn( json.dumps( json.loads( hosts ),
596 sort_keys=True,
597 indent=4,
598 separators=( ',', ': ' ) ) )
599 except ( ValueError, TypeError ):
600 main.log.warn( repr( hosts ) )
601 hostResult = main.FALSE
602 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
603 onpass="Found a host id for each host",
604 onfail="Error looking up host ids" )
605
606 intentStart = time.time()
607 onosIds = main.ONOScli1.getAllIntentsId()
608 main.log.info( "Submitted intents: " + str( intentIds ) )
609 main.log.info( "Intents in ONOS: " + str( onosIds ) )
610 for intent in intentIds:
611 if intent in onosIds:
612 pass # intent submitted is in onos
613 else:
614 intentAddResult = False
615 if intentAddResult:
616 intentStop = time.time()
617 else:
618 intentStop = None
619 # Print the intent states
620 intents = main.ONOScli1.intents()
621 intentStates = []
622 installedCheck = True
623 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
624 count = 0
625 try:
626 for intent in json.loads( intents ):
627 state = intent.get( 'state', None )
628 if "INSTALLED" not in state:
629 installedCheck = False
630 intentId = intent.get( 'id', None )
631 intentStates.append( ( intentId, state ) )
632 except ( ValueError, TypeError ):
633 main.log.exception( "Error parsing intents" )
634 # add submitted intents not in the store
635 tmplist = [ i for i, s in intentStates ]
636 missingIntents = False
637 for i in intentIds:
638 if i not in tmplist:
639 intentStates.append( ( i, " - " ) )
640 missingIntents = True
641 intentStates.sort()
642 for i, s in intentStates:
643 count += 1
644 main.log.info( "%-6s%-15s%-15s" %
645 ( str( count ), str( i ), str( s ) ) )
646 leaders = main.ONOScli1.leaders()
647 try:
648 missing = False
649 if leaders:
650 parsedLeaders = json.loads( leaders )
651 main.log.warn( json.dumps( parsedLeaders,
652 sort_keys=True,
653 indent=4,
654 separators=( ',', ': ' ) ) )
655 # check for all intent partitions
656 topics = []
657 for i in range( 14 ):
658 topics.append( "intent-partition-" + str( i ) )
659 main.log.debug( topics )
660 ONOStopics = [ j['topic'] for j in parsedLeaders ]
661 for topic in topics:
662 if topic not in ONOStopics:
663 main.log.error( "Error: " + topic +
664 " not in leaders" )
665 missing = True
666 else:
667 main.log.error( "leaders() returned None" )
668 except ( ValueError, TypeError ):
669 main.log.exception( "Error parsing leaders" )
670 main.log.error( repr( leaders ) )
671 # Check all nodes
672 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700673 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700674 response = node.leaders( jsonFormat=False)
675 main.log.warn( str( node.name ) + " leaders output: \n" +
676 str( response ) )
677
678 partitions = main.ONOScli1.partitions()
679 try:
680 if partitions :
681 parsedPartitions = json.loads( partitions )
682 main.log.warn( json.dumps( parsedPartitions,
683 sort_keys=True,
684 indent=4,
685 separators=( ',', ': ' ) ) )
686 # TODO check for a leader in all paritions
687 # TODO check for consistency among nodes
688 else:
689 main.log.error( "partitions() returned None" )
690 except ( ValueError, TypeError ):
691 main.log.exception( "Error parsing partitions" )
692 main.log.error( repr( partitions ) )
693 pendingMap = main.ONOScli1.pendingMap()
694 try:
695 if pendingMap :
696 parsedPending = json.loads( pendingMap )
697 main.log.warn( json.dumps( parsedPending,
698 sort_keys=True,
699 indent=4,
700 separators=( ',', ': ' ) ) )
701 # TODO check something here?
702 else:
703 main.log.error( "pendingMap() returned None" )
704 except ( ValueError, TypeError ):
705 main.log.exception( "Error parsing pending map" )
706 main.log.error( repr( pendingMap ) )
707
708 intentAddResult = bool( intentAddResult and not missingIntents and
709 installedCheck )
710 if not intentAddResult:
711 main.log.error( "Error in pushing host intents to ONOS" )
712
713 main.step( "Intent Anti-Entropy dispersion" )
714 for i in range(100):
715 correct = True
716 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700717 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700718 onosIds = []
719 ids = cli.getAllIntentsId()
720 onosIds.append( ids )
721 main.log.debug( "Intents in " + cli.name + ": " +
722 str( sorted( onosIds ) ) )
723 if sorted( ids ) != sorted( intentIds ):
724 main.log.warn( "Set of intent IDs doesn't match" )
725 correct = False
726 break
727 else:
728 intents = json.loads( cli.intents() )
729 for intent in intents:
730 if intent[ 'state' ] != "INSTALLED":
731 main.log.warn( "Intent " + intent[ 'id' ] +
732 " is " + intent[ 'state' ] )
733 correct = False
734 break
735 if correct:
736 break
737 else:
738 time.sleep(1)
739 if not intentStop:
740 intentStop = time.time()
741 global gossipTime
742 gossipTime = intentStop - intentStart
743 main.log.info( "It took about " + str( gossipTime ) +
744 " seconds for all intents to appear in each node" )
745 append = False
746 title = "Gossip Intents"
747 count = 1
748 while append is False:
749 curTitle = title + str( count )
750 if curTitle not in labels:
751 labels.append( curTitle )
752 data.append( str( gossipTime ) )
753 append = True
754 else:
755 count += 1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700756 gossipPeriod = int( main.params['timers']['gossip'] )
757 maxGossipTime = gossipPeriod * len( main.nodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700758 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700759 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700760 onpass="ECM anti-entropy for intents worked within " +
761 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700762 onfail="Intent ECM anti-entropy took too long. " +
763 "Expected time:{}, Actual time:{}".format( maxGossipTime,
764 gossipTime ) )
765 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700766 intentAddResult = True
767
768 if not intentAddResult or "key" in pendingMap:
769 import time
770 installedCheck = True
771 main.log.info( "Sleeping 60 seconds to see if intents are found" )
772 time.sleep( 60 )
773 onosIds = main.ONOScli1.getAllIntentsId()
774 main.log.info( "Submitted intents: " + str( intentIds ) )
775 main.log.info( "Intents in ONOS: " + str( onosIds ) )
776 # Print the intent states
777 intents = main.ONOScli1.intents()
778 intentStates = []
779 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
780 count = 0
781 try:
782 for intent in json.loads( intents ):
783 # Iter through intents of a node
784 state = intent.get( 'state', None )
785 if "INSTALLED" not in state:
786 installedCheck = False
787 intentId = intent.get( 'id', None )
788 intentStates.append( ( intentId, state ) )
789 except ( ValueError, TypeError ):
790 main.log.exception( "Error parsing intents" )
791 # add submitted intents not in the store
792 tmplist = [ i for i, s in intentStates ]
793 for i in intentIds:
794 if i not in tmplist:
795 intentStates.append( ( i, " - " ) )
796 intentStates.sort()
797 for i, s in intentStates:
798 count += 1
799 main.log.info( "%-6s%-15s%-15s" %
800 ( str( count ), str( i ), str( s ) ) )
801 leaders = main.ONOScli1.leaders()
802 try:
803 missing = False
804 if leaders:
805 parsedLeaders = json.loads( leaders )
806 main.log.warn( json.dumps( parsedLeaders,
807 sort_keys=True,
808 indent=4,
809 separators=( ',', ': ' ) ) )
810 # check for all intent partitions
811 # check for election
812 topics = []
813 for i in range( 14 ):
814 topics.append( "intent-partition-" + str( i ) )
815 # FIXME: this should only be after we start the app
816 topics.append( "org.onosproject.election" )
817 main.log.debug( topics )
818 ONOStopics = [ j['topic'] for j in parsedLeaders ]
819 for topic in topics:
820 if topic not in ONOStopics:
821 main.log.error( "Error: " + topic +
822 " not in leaders" )
823 missing = True
824 else:
825 main.log.error( "leaders() returned None" )
826 except ( ValueError, TypeError ):
827 main.log.exception( "Error parsing leaders" )
828 main.log.error( repr( leaders ) )
829 # Check all nodes
830 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700831 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700832 response = node.leaders( jsonFormat=False)
833 main.log.warn( str( node.name ) + " leaders output: \n" +
834 str( response ) )
835
836 partitions = main.ONOScli1.partitions()
837 try:
838 if partitions :
839 parsedPartitions = json.loads( partitions )
840 main.log.warn( json.dumps( parsedPartitions,
841 sort_keys=True,
842 indent=4,
843 separators=( ',', ': ' ) ) )
844 # TODO check for a leader in all paritions
845 # TODO check for consistency among nodes
846 else:
847 main.log.error( "partitions() returned None" )
848 except ( ValueError, TypeError ):
849 main.log.exception( "Error parsing partitions" )
850 main.log.error( repr( partitions ) )
851 pendingMap = main.ONOScli1.pendingMap()
852 try:
853 if pendingMap :
854 parsedPending = json.loads( pendingMap )
855 main.log.warn( json.dumps( parsedPending,
856 sort_keys=True,
857 indent=4,
858 separators=( ',', ': ' ) ) )
859 # TODO check something here?
860 else:
861 main.log.error( "pendingMap() returned None" )
862 except ( ValueError, TypeError ):
863 main.log.exception( "Error parsing pending map" )
864 main.log.error( repr( pendingMap ) )
865
866 def CASE4( self, main ):
867 """
868 Ping across added host intents
869 """
870 import json
871 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700872 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700873 assert main, "main not defined"
874 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700875 assert main.CLIs, "main.CLIs not defined"
876 assert main.nodes, "main.nodes not defined"
Jon Hall6e709752016-02-01 13:38:46 -0800877 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700878 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700879 "functionality and check the state of " +\
880 "the intent"
881 main.step( "Ping across added host intents" )
882 PingResult = main.TRUE
883 for i in range( 8, 18 ):
884 ping = main.Mininet1.pingHost( src="h" + str( i ),
885 target="h" + str( i + 10 ) )
886 PingResult = PingResult and ping
887 if ping == main.FALSE:
888 main.log.warn( "Ping failed between h" + str( i ) +
889 " and h" + str( i + 10 ) )
890 elif ping == main.TRUE:
891 main.log.info( "Ping test passed!" )
892 # Don't set PingResult or you'd override failures
893 if PingResult == main.FALSE:
894 main.log.error(
895 "Intents have not been installed correctly, pings failed." )
896 # TODO: pretty print
897 main.log.warn( "ONOS1 intents: " )
898 try:
899 tmpIntents = main.ONOScli1.intents()
900 main.log.warn( json.dumps( json.loads( tmpIntents ),
901 sort_keys=True,
902 indent=4,
903 separators=( ',', ': ' ) ) )
904 except ( ValueError, TypeError ):
905 main.log.warn( repr( tmpIntents ) )
906 utilities.assert_equals(
907 expect=main.TRUE,
908 actual=PingResult,
909 onpass="Intents have been installed correctly and pings work",
910 onfail="Intents have not been installed correctly, pings failed." )
911
912 main.step( "Check Intent state" )
913 installedCheck = False
914 loopCount = 0
915 while not installedCheck and loopCount < 40:
916 installedCheck = True
917 # Print the intent states
918 intents = main.ONOScli1.intents()
919 intentStates = []
920 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700921 count = 0
Jon Hall5cf14d52015-07-16 12:15:19 -0700922 # Iter through intents of a node
923 try:
924 for intent in json.loads( intents ):
925 state = intent.get( 'state', None )
926 if "INSTALLED" not in state:
927 installedCheck = False
928 intentId = intent.get( 'id', None )
929 intentStates.append( ( intentId, state ) )
930 except ( ValueError, TypeError ):
931 main.log.exception( "Error parsing intents." )
932 # Print states
933 intentStates.sort()
934 for i, s in intentStates:
935 count += 1
936 main.log.info( "%-6s%-15s%-15s" %
937 ( str( count ), str( i ), str( s ) ) )
938 if not installedCheck:
939 time.sleep( 1 )
940 loopCount += 1
941 utilities.assert_equals( expect=True, actual=installedCheck,
942 onpass="Intents are all INSTALLED",
943 onfail="Intents are not all in " +
944 "INSTALLED state" )
945
946 main.step( "Check leadership of topics" )
947 leaders = main.ONOScli1.leaders()
948 topicCheck = main.TRUE
949 try:
950 if leaders:
951 parsedLeaders = json.loads( leaders )
952 main.log.warn( json.dumps( parsedLeaders,
953 sort_keys=True,
954 indent=4,
955 separators=( ',', ': ' ) ) )
956 # check for all intent partitions
957 # check for election
958 # TODO: Look at Devices as topics now that it uses this system
959 topics = []
960 for i in range( 14 ):
961 topics.append( "intent-partition-" + str( i ) )
962 # FIXME: this should only be after we start the app
963 # FIXME: topics.append( "org.onosproject.election" )
964 # Print leaders output
965 main.log.debug( topics )
966 ONOStopics = [ j['topic'] for j in parsedLeaders ]
967 for topic in topics:
968 if topic not in ONOStopics:
969 main.log.error( "Error: " + topic +
970 " not in leaders" )
971 topicCheck = main.FALSE
972 else:
973 main.log.error( "leaders() returned None" )
974 topicCheck = main.FALSE
975 except ( ValueError, TypeError ):
976 topicCheck = main.FALSE
977 main.log.exception( "Error parsing leaders" )
978 main.log.error( repr( leaders ) )
979 # TODO: Check for a leader of these topics
980 # Check all nodes
981 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700982 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700983 response = node.leaders( jsonFormat=False)
984 main.log.warn( str( node.name ) + " leaders output: \n" +
985 str( response ) )
986
987 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
988 onpass="intent Partitions is in leaders",
989 onfail="Some topics were lost " )
990 # Print partitions
991 partitions = main.ONOScli1.partitions()
992 try:
993 if partitions :
994 parsedPartitions = json.loads( partitions )
995 main.log.warn( json.dumps( parsedPartitions,
996 sort_keys=True,
997 indent=4,
998 separators=( ',', ': ' ) ) )
999 # TODO check for a leader in all paritions
1000 # TODO check for consistency among nodes
1001 else:
1002 main.log.error( "partitions() returned None" )
1003 except ( ValueError, TypeError ):
1004 main.log.exception( "Error parsing partitions" )
1005 main.log.error( repr( partitions ) )
1006 # Print Pending Map
1007 pendingMap = main.ONOScli1.pendingMap()
1008 try:
1009 if pendingMap :
1010 parsedPending = json.loads( pendingMap )
1011 main.log.warn( json.dumps( parsedPending,
1012 sort_keys=True,
1013 indent=4,
1014 separators=( ',', ': ' ) ) )
1015 # TODO check something here?
1016 else:
1017 main.log.error( "pendingMap() returned None" )
1018 except ( ValueError, TypeError ):
1019 main.log.exception( "Error parsing pending map" )
1020 main.log.error( repr( pendingMap ) )
1021
1022 if not installedCheck:
1023 main.log.info( "Waiting 60 seconds to see if the state of " +
1024 "intents change" )
1025 time.sleep( 60 )
1026 # Print the intent states
1027 intents = main.ONOScli1.intents()
1028 intentStates = []
1029 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1030 count = 0
1031 # Iter through intents of a node
1032 try:
1033 for intent in json.loads( intents ):
1034 state = intent.get( 'state', None )
1035 if "INSTALLED" not in state:
1036 installedCheck = False
1037 intentId = intent.get( 'id', None )
1038 intentStates.append( ( intentId, state ) )
1039 except ( ValueError, TypeError ):
1040 main.log.exception( "Error parsing intents." )
1041 intentStates.sort()
1042 for i, s in intentStates:
1043 count += 1
1044 main.log.info( "%-6s%-15s%-15s" %
1045 ( str( count ), str( i ), str( s ) ) )
1046 leaders = main.ONOScli1.leaders()
1047 try:
1048 missing = False
1049 if leaders:
1050 parsedLeaders = json.loads( leaders )
1051 main.log.warn( json.dumps( parsedLeaders,
1052 sort_keys=True,
1053 indent=4,
1054 separators=( ',', ': ' ) ) )
1055 # check for all intent partitions
1056 # check for election
1057 topics = []
1058 for i in range( 14 ):
1059 topics.append( "intent-partition-" + str( i ) )
1060 # FIXME: this should only be after we start the app
1061 topics.append( "org.onosproject.election" )
1062 main.log.debug( topics )
1063 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1064 for topic in topics:
1065 if topic not in ONOStopics:
1066 main.log.error( "Error: " + topic +
1067 " not in leaders" )
1068 missing = True
1069 else:
1070 main.log.error( "leaders() returned None" )
1071 except ( ValueError, TypeError ):
1072 main.log.exception( "Error parsing leaders" )
1073 main.log.error( repr( leaders ) )
1074 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001075 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001076 response = node.leaders( jsonFormat=False)
1077 main.log.warn( str( node.name ) + " leaders output: \n" +
1078 str( response ) )
1079
1080 partitions = main.ONOScli1.partitions()
1081 try:
1082 if partitions :
1083 parsedPartitions = json.loads( partitions )
1084 main.log.warn( json.dumps( parsedPartitions,
1085 sort_keys=True,
1086 indent=4,
1087 separators=( ',', ': ' ) ) )
1088 # TODO check for a leader in all paritions
1089 # TODO check for consistency among nodes
1090 else:
1091 main.log.error( "partitions() returned None" )
1092 except ( ValueError, TypeError ):
1093 main.log.exception( "Error parsing partitions" )
1094 main.log.error( repr( partitions ) )
1095 pendingMap = main.ONOScli1.pendingMap()
1096 try:
1097 if pendingMap :
1098 parsedPending = json.loads( pendingMap )
1099 main.log.warn( json.dumps( parsedPending,
1100 sort_keys=True,
1101 indent=4,
1102 separators=( ',', ': ' ) ) )
1103 # TODO check something here?
1104 else:
1105 main.log.error( "pendingMap() returned None" )
1106 except ( ValueError, TypeError ):
1107 main.log.exception( "Error parsing pending map" )
1108 main.log.error( repr( pendingMap ) )
1109 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001110 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001111 main.step( "Wait a minute then ping again" )
1112 # the wait is above
1113 PingResult = main.TRUE
1114 for i in range( 8, 18 ):
1115 ping = main.Mininet1.pingHost( src="h" + str( i ),
1116 target="h" + str( i + 10 ) )
1117 PingResult = PingResult and ping
1118 if ping == main.FALSE:
1119 main.log.warn( "Ping failed between h" + str( i ) +
1120 " and h" + str( i + 10 ) )
1121 elif ping == main.TRUE:
1122 main.log.info( "Ping test passed!" )
1123 # Don't set PingResult or you'd override failures
1124 if PingResult == main.FALSE:
1125 main.log.error(
1126 "Intents have not been installed correctly, pings failed." )
1127 # TODO: pretty print
1128 main.log.warn( "ONOS1 intents: " )
1129 try:
1130 tmpIntents = main.ONOScli1.intents()
1131 main.log.warn( json.dumps( json.loads( tmpIntents ),
1132 sort_keys=True,
1133 indent=4,
1134 separators=( ',', ': ' ) ) )
1135 except ( ValueError, TypeError ):
1136 main.log.warn( repr( tmpIntents ) )
1137 utilities.assert_equals(
1138 expect=main.TRUE,
1139 actual=PingResult,
1140 onpass="Intents have been installed correctly and pings work",
1141 onfail="Intents have not been installed correctly, pings failed." )
1142
1143 def CASE5( self, main ):
1144 """
1145 Reading state of ONOS
1146 """
1147 import json
1148 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001149 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001150 assert main, "main not defined"
1151 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001152 assert main.CLIs, "main.CLIs not defined"
1153 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001154
1155 main.case( "Setting up and gathering data for current state" )
1156 # The general idea for this test case is to pull the state of
1157 # ( intents,flows, topology,... ) from each ONOS node
1158 # We can then compare them with each other and also with past states
1159
1160 main.step( "Check that each switch has a master" )
1161 global mastershipState
1162 mastershipState = '[]'
1163
1164 # Assert that each device has a master
1165 rolesNotNull = main.TRUE
1166 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001167 for i in range( main.numCtrls ):
1168 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001169 name="rolesNotNull-" + str( i ),
1170 args=[] )
1171 threads.append( t )
1172 t.start()
1173
1174 for t in threads:
1175 t.join()
1176 rolesNotNull = rolesNotNull and t.result
1177 utilities.assert_equals(
1178 expect=main.TRUE,
1179 actual=rolesNotNull,
1180 onpass="Each device has a master",
1181 onfail="Some devices don't have a master assigned" )
1182
1183 main.step( "Get the Mastership of each switch from each controller" )
1184 ONOSMastership = []
1185 mastershipCheck = main.FALSE
1186 consistentMastership = True
1187 rolesResults = True
1188 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001189 for i in range( main.numCtrls ):
1190 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001191 name="roles-" + str( i ),
1192 args=[] )
1193 threads.append( t )
1194 t.start()
1195
1196 for t in threads:
1197 t.join()
1198 ONOSMastership.append( t.result )
1199
Jon Halle1a3b752015-07-22 13:02:46 -07001200 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001201 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1202 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1203 " roles" )
1204 main.log.warn(
1205 "ONOS" + str( i + 1 ) + " mastership response: " +
1206 repr( ONOSMastership[i] ) )
1207 rolesResults = False
1208 utilities.assert_equals(
1209 expect=True,
1210 actual=rolesResults,
1211 onpass="No error in reading roles output",
1212 onfail="Error in reading roles from ONOS" )
1213
1214 main.step( "Check for consistency in roles from each controller" )
1215 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1216 main.log.info(
1217 "Switch roles are consistent across all ONOS nodes" )
1218 else:
1219 consistentMastership = False
1220 utilities.assert_equals(
1221 expect=True,
1222 actual=consistentMastership,
1223 onpass="Switch roles are consistent across all ONOS nodes",
1224 onfail="ONOS nodes have different views of switch roles" )
1225
1226 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001227 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001228 try:
1229 main.log.warn(
1230 "ONOS" + str( i + 1 ) + " roles: ",
1231 json.dumps(
1232 json.loads( ONOSMastership[ i ] ),
1233 sort_keys=True,
1234 indent=4,
1235 separators=( ',', ': ' ) ) )
1236 except ( ValueError, TypeError ):
1237 main.log.warn( repr( ONOSMastership[ i ] ) )
1238 elif rolesResults and consistentMastership:
1239 mastershipCheck = main.TRUE
1240 mastershipState = ONOSMastership[ 0 ]
1241
1242 main.step( "Get the intents from each controller" )
1243 global intentState
1244 intentState = []
1245 ONOSIntents = []
1246 intentCheck = main.FALSE
1247 consistentIntents = True
1248 intentsResults = True
1249 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001250 for i in range( main.numCtrls ):
1251 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001252 name="intents-" + str( i ),
1253 args=[],
1254 kwargs={ 'jsonFormat': True } )
1255 threads.append( t )
1256 t.start()
1257
1258 for t in threads:
1259 t.join()
1260 ONOSIntents.append( t.result )
1261
Jon Halle1a3b752015-07-22 13:02:46 -07001262 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08001263 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001264 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hall6e709752016-02-01 13:38:46 -08001265 main.log.error( "Error in getting ONOS" + node + " intents" )
1266 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001267 repr( ONOSIntents[ i ] ) )
1268 intentsResults = False
1269 utilities.assert_equals(
1270 expect=True,
1271 actual=intentsResults,
1272 onpass="No error in reading intents output",
1273 onfail="Error in reading intents from ONOS" )
1274
1275 main.step( "Check for consistency in Intents from each controller" )
1276 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1277 main.log.info( "Intents are consistent across all ONOS " +
1278 "nodes" )
1279 else:
1280 consistentIntents = False
1281 main.log.error( "Intents not consistent" )
1282 utilities.assert_equals(
1283 expect=True,
1284 actual=consistentIntents,
1285 onpass="Intents are consistent across all ONOS nodes",
1286 onfail="ONOS nodes have different views of intents" )
1287
1288 if intentsResults:
1289 # Try to make it easy to figure out what is happening
1290 #
1291 # Intent ONOS1 ONOS2 ...
1292 # 0x01 INSTALLED INSTALLING
1293 # ... ... ...
1294 # ... ... ...
1295 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001296 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001297 title += " " * 10 + "ONOS" + str( n + 1 )
1298 main.log.warn( title )
1299 # get all intent keys in the cluster
1300 keys = []
1301 for nodeStr in ONOSIntents:
1302 node = json.loads( nodeStr )
1303 for intent in node:
1304 keys.append( intent.get( 'id' ) )
1305 keys = set( keys )
1306 for key in keys:
1307 row = "%-13s" % key
1308 for nodeStr in ONOSIntents:
1309 node = json.loads( nodeStr )
1310 for intent in node:
1311 if intent.get( 'id', "Error" ) == key:
1312 row += "%-15s" % intent.get( 'state' )
1313 main.log.warn( row )
1314 # End table view
1315
1316 if intentsResults and not consistentIntents:
1317 # print the json objects
1318 n = len(ONOSIntents)
1319 main.log.debug( "ONOS" + str( n ) + " intents: " )
1320 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1321 sort_keys=True,
1322 indent=4,
1323 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001324 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001325 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1326 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1327 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1328 sort_keys=True,
1329 indent=4,
1330 separators=( ',', ': ' ) ) )
1331 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001332 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001333 str( n ) + " intents" )
1334 elif intentsResults and consistentIntents:
1335 intentCheck = main.TRUE
1336 intentState = ONOSIntents[ 0 ]
1337
1338 main.step( "Get the flows from each controller" )
1339 global flowState
1340 flowState = []
1341 ONOSFlows = []
1342 ONOSFlowsJson = []
1343 flowCheck = main.FALSE
1344 consistentFlows = True
1345 flowsResults = True
1346 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001347 for i in range( main.numCtrls ):
1348 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001349 name="flows-" + str( i ),
1350 args=[],
1351 kwargs={ 'jsonFormat': True } )
1352 threads.append( t )
1353 t.start()
1354
1355 # NOTE: Flows command can take some time to run
1356 time.sleep(30)
1357 for t in threads:
1358 t.join()
1359 result = t.result
1360 ONOSFlows.append( result )
1361
Jon Halle1a3b752015-07-22 13:02:46 -07001362 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001363 num = str( i + 1 )
1364 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1365 main.log.error( "Error in getting ONOS" + num + " flows" )
1366 main.log.warn( "ONOS" + num + " flows response: " +
1367 repr( ONOSFlows[ i ] ) )
1368 flowsResults = False
1369 ONOSFlowsJson.append( None )
1370 else:
1371 try:
1372 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1373 except ( ValueError, TypeError ):
1374 # FIXME: change this to log.error?
1375 main.log.exception( "Error in parsing ONOS" + num +
1376 " response as json." )
1377 main.log.error( repr( ONOSFlows[ i ] ) )
1378 ONOSFlowsJson.append( None )
1379 flowsResults = False
1380 utilities.assert_equals(
1381 expect=True,
1382 actual=flowsResults,
1383 onpass="No error in reading flows output",
1384 onfail="Error in reading flows from ONOS" )
1385
1386 main.step( "Check for consistency in Flows from each controller" )
1387 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1388 if all( tmp ):
1389 main.log.info( "Flow count is consistent across all ONOS nodes" )
1390 else:
1391 consistentFlows = False
1392 utilities.assert_equals(
1393 expect=True,
1394 actual=consistentFlows,
1395 onpass="The flow count is consistent across all ONOS nodes",
1396 onfail="ONOS nodes have different flow counts" )
1397
1398 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001399 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001400 try:
1401 main.log.warn(
1402 "ONOS" + str( i + 1 ) + " flows: " +
1403 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1404 indent=4, separators=( ',', ': ' ) ) )
1405 except ( ValueError, TypeError ):
1406 main.log.warn(
1407 "ONOS" + str( i + 1 ) + " flows: " +
1408 repr( ONOSFlows[ i ] ) )
1409 elif flowsResults and consistentFlows:
1410 flowCheck = main.TRUE
1411 flowState = ONOSFlows[ 0 ]
1412
1413 main.step( "Get the OF Table entries" )
1414 global flows
1415 flows = []
1416 for i in range( 1, 29 ):
Jon Hallca7ac292015-11-11 09:28:12 -08001417 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3" ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001418 if flowCheck == main.FALSE:
1419 for table in flows:
1420 main.log.warn( table )
1421 # TODO: Compare switch flow tables with ONOS flow tables
1422
1423 main.step( "Start continuous pings" )
1424 main.Mininet2.pingLong(
1425 src=main.params[ 'PING' ][ 'source1' ],
1426 target=main.params[ 'PING' ][ 'target1' ],
1427 pingTime=500 )
1428 main.Mininet2.pingLong(
1429 src=main.params[ 'PING' ][ 'source2' ],
1430 target=main.params[ 'PING' ][ 'target2' ],
1431 pingTime=500 )
1432 main.Mininet2.pingLong(
1433 src=main.params[ 'PING' ][ 'source3' ],
1434 target=main.params[ 'PING' ][ 'target3' ],
1435 pingTime=500 )
1436 main.Mininet2.pingLong(
1437 src=main.params[ 'PING' ][ 'source4' ],
1438 target=main.params[ 'PING' ][ 'target4' ],
1439 pingTime=500 )
1440 main.Mininet2.pingLong(
1441 src=main.params[ 'PING' ][ 'source5' ],
1442 target=main.params[ 'PING' ][ 'target5' ],
1443 pingTime=500 )
1444 main.Mininet2.pingLong(
1445 src=main.params[ 'PING' ][ 'source6' ],
1446 target=main.params[ 'PING' ][ 'target6' ],
1447 pingTime=500 )
1448 main.Mininet2.pingLong(
1449 src=main.params[ 'PING' ][ 'source7' ],
1450 target=main.params[ 'PING' ][ 'target7' ],
1451 pingTime=500 )
1452 main.Mininet2.pingLong(
1453 src=main.params[ 'PING' ][ 'source8' ],
1454 target=main.params[ 'PING' ][ 'target8' ],
1455 pingTime=500 )
1456 main.Mininet2.pingLong(
1457 src=main.params[ 'PING' ][ 'source9' ],
1458 target=main.params[ 'PING' ][ 'target9' ],
1459 pingTime=500 )
1460 main.Mininet2.pingLong(
1461 src=main.params[ 'PING' ][ 'source10' ],
1462 target=main.params[ 'PING' ][ 'target10' ],
1463 pingTime=500 )
1464
1465 main.step( "Collecting topology information from ONOS" )
1466 devices = []
1467 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001468 for i in range( main.numCtrls ):
1469 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001470 name="devices-" + str( i ),
1471 args=[ ] )
1472 threads.append( t )
1473 t.start()
1474
1475 for t in threads:
1476 t.join()
1477 devices.append( t.result )
1478 hosts = []
1479 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001480 for i in range( main.numCtrls ):
1481 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001482 name="hosts-" + str( i ),
1483 args=[ ] )
1484 threads.append( t )
1485 t.start()
1486
1487 for t in threads:
1488 t.join()
1489 try:
1490 hosts.append( json.loads( t.result ) )
1491 except ( ValueError, TypeError ):
1492 # FIXME: better handling of this, print which node
1493 # Maybe use thread name?
1494 main.log.exception( "Error parsing json output of hosts" )
Jon Hall3afe4c92015-12-14 19:30:38 -08001495 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001496 hosts.append( None )
1497
1498 ports = []
1499 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001500 for i in range( main.numCtrls ):
1501 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001502 name="ports-" + str( i ),
1503 args=[ ] )
1504 threads.append( t )
1505 t.start()
1506
1507 for t in threads:
1508 t.join()
1509 ports.append( t.result )
1510 links = []
1511 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001512 for i in range( main.numCtrls ):
1513 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001514 name="links-" + str( i ),
1515 args=[ ] )
1516 threads.append( t )
1517 t.start()
1518
1519 for t in threads:
1520 t.join()
1521 links.append( t.result )
1522 clusters = []
1523 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001524 for i in range( main.numCtrls ):
1525 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001526 name="clusters-" + str( i ),
1527 args=[ ] )
1528 threads.append( t )
1529 t.start()
1530
1531 for t in threads:
1532 t.join()
1533 clusters.append( t.result )
1534 # Compare json objects for hosts and dataplane clusters
1535
1536 # hosts
1537 main.step( "Host view is consistent across ONOS nodes" )
1538 consistentHostsResult = main.TRUE
1539 for controller in range( len( hosts ) ):
1540 controllerStr = str( controller + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001541 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001542 if hosts[ controller ] == hosts[ 0 ]:
1543 continue
1544 else: # hosts not consistent
1545 main.log.error( "hosts from ONOS" +
1546 controllerStr +
1547 " is inconsistent with ONOS1" )
1548 main.log.warn( repr( hosts[ controller ] ) )
1549 consistentHostsResult = main.FALSE
1550
1551 else:
1552 main.log.error( "Error in getting ONOS hosts from ONOS" +
1553 controllerStr )
1554 consistentHostsResult = main.FALSE
1555 main.log.warn( "ONOS" + controllerStr +
1556 " hosts response: " +
1557 repr( hosts[ controller ] ) )
1558 utilities.assert_equals(
1559 expect=main.TRUE,
1560 actual=consistentHostsResult,
1561 onpass="Hosts view is consistent across all ONOS nodes",
1562 onfail="ONOS nodes have different views of hosts" )
1563
1564 main.step( "Each host has an IP address" )
1565 ipResult = main.TRUE
1566 for controller in range( 0, len( hosts ) ):
1567 controllerStr = str( controller + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001568 if hosts[ controller ]:
1569 for host in hosts[ controller ]:
1570 if not host.get( 'ipAddresses', [ ] ):
Jon Hallf3d16e72015-12-16 17:45:08 -08001571 main.log.error( "Error with host ips on controller" +
Jon Hall3afe4c92015-12-14 19:30:38 -08001572 controllerStr + ": " + str( host ) )
1573 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001574 utilities.assert_equals(
1575 expect=main.TRUE,
1576 actual=ipResult,
1577 onpass="The ips of the hosts aren't empty",
1578 onfail="The ip of at least one host is missing" )
1579
1580 # Strongly connected clusters of devices
1581 main.step( "Cluster view is consistent across ONOS nodes" )
1582 consistentClustersResult = main.TRUE
1583 for controller in range( len( clusters ) ):
1584 controllerStr = str( controller + 1 )
1585 if "Error" not in clusters[ controller ]:
1586 if clusters[ controller ] == clusters[ 0 ]:
1587 continue
1588 else: # clusters not consistent
1589 main.log.error( "clusters from ONOS" + controllerStr +
1590 " is inconsistent with ONOS1" )
1591 consistentClustersResult = main.FALSE
1592
1593 else:
1594 main.log.error( "Error in getting dataplane clusters " +
1595 "from ONOS" + controllerStr )
1596 consistentClustersResult = main.FALSE
1597 main.log.warn( "ONOS" + controllerStr +
1598 " clusters response: " +
1599 repr( clusters[ controller ] ) )
1600 utilities.assert_equals(
1601 expect=main.TRUE,
1602 actual=consistentClustersResult,
1603 onpass="Clusters view is consistent across all ONOS nodes",
1604 onfail="ONOS nodes have different views of clusters" )
1605 # there should always only be one cluster
1606 main.step( "Cluster view correct across ONOS nodes" )
1607 try:
1608 numClusters = len( json.loads( clusters[ 0 ] ) )
1609 except ( ValueError, TypeError ):
1610 main.log.exception( "Error parsing clusters[0]: " +
1611 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001612 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001613 clusterResults = main.FALSE
1614 if numClusters == 1:
1615 clusterResults = main.TRUE
1616 utilities.assert_equals(
1617 expect=1,
1618 actual=numClusters,
1619 onpass="ONOS shows 1 SCC",
1620 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1621
1622 main.step( "Comparing ONOS topology to MN" )
1623 devicesResults = main.TRUE
1624 linksResults = main.TRUE
1625 hostsResults = main.TRUE
1626 mnSwitches = main.Mininet1.getSwitches()
1627 mnLinks = main.Mininet1.getLinks()
1628 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001629 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001630 controllerStr = str( controller + 1 )
1631 if devices[ controller ] and ports[ controller ] and\
1632 "Error" not in devices[ controller ] and\
1633 "Error" not in ports[ controller ]:
Jon Halle1a3b752015-07-22 13:02:46 -07001634 currentDevicesResult = main.Mininet1.compareSwitches(
1635 mnSwitches,
1636 json.loads( devices[ controller ] ),
1637 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001638 else:
1639 currentDevicesResult = main.FALSE
1640 utilities.assert_equals( expect=main.TRUE,
1641 actual=currentDevicesResult,
1642 onpass="ONOS" + controllerStr +
1643 " Switches view is correct",
1644 onfail="ONOS" + controllerStr +
1645 " Switches view is incorrect" )
1646 if links[ controller ] and "Error" not in links[ controller ]:
1647 currentLinksResult = main.Mininet1.compareLinks(
1648 mnSwitches, mnLinks,
1649 json.loads( links[ controller ] ) )
1650 else:
1651 currentLinksResult = main.FALSE
1652 utilities.assert_equals( expect=main.TRUE,
1653 actual=currentLinksResult,
1654 onpass="ONOS" + controllerStr +
1655 " links view is correct",
1656 onfail="ONOS" + controllerStr +
1657 " links view is incorrect" )
1658
Jon Hall657cdf62015-12-17 14:40:51 -08001659 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001660 currentHostsResult = main.Mininet1.compareHosts(
1661 mnHosts,
1662 hosts[ controller ] )
1663 else:
1664 currentHostsResult = main.FALSE
1665 utilities.assert_equals( expect=main.TRUE,
1666 actual=currentHostsResult,
1667 onpass="ONOS" + controllerStr +
1668 " hosts exist in Mininet",
1669 onfail="ONOS" + controllerStr +
1670 " hosts don't match Mininet" )
1671
1672 devicesResults = devicesResults and currentDevicesResult
1673 linksResults = linksResults and currentLinksResult
1674 hostsResults = hostsResults and currentHostsResult
1675
1676 main.step( "Device information is correct" )
1677 utilities.assert_equals(
1678 expect=main.TRUE,
1679 actual=devicesResults,
1680 onpass="Device information is correct",
1681 onfail="Device information is incorrect" )
1682
1683 main.step( "Links are correct" )
1684 utilities.assert_equals(
1685 expect=main.TRUE,
1686 actual=linksResults,
1687 onpass="Link are correct",
1688 onfail="Links are incorrect" )
1689
1690 main.step( "Hosts are correct" )
1691 utilities.assert_equals(
1692 expect=main.TRUE,
1693 actual=hostsResults,
1694 onpass="Hosts are correct",
1695 onfail="Hosts are incorrect" )
1696
1697 def CASE6( self, main ):
1698 """
1699 The Failure case.
1700 """
1701 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001702 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001703 assert main, "main not defined"
1704 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001705 assert main.CLIs, "main.CLIs not defined"
1706 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001707 try:
1708 labels
1709 except NameError:
1710 main.log.error( "labels not defined, setting to []" )
1711 global labels
1712 labels = []
1713 try:
1714 data
1715 except NameError:
1716 main.log.error( "data not defined, setting to []" )
1717 global data
1718 data = []
1719 # Reset non-persistent variables
1720 try:
1721 iCounterValue = 0
1722 except NameError:
1723 main.log.error( "iCounterValue not defined, setting to 0" )
1724 iCounterValue = 0
1725
1726 main.case( "Restart entire ONOS cluster" )
1727
Jon Hall5ec6b1b2015-09-17 18:20:14 -07001728 main.step( "Checking ONOS Logs for errors" )
1729 for node in main.nodes:
1730 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1731 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1732
Jon Hall5cf14d52015-07-16 12:15:19 -07001733 main.step( "Killing ONOS nodes" )
1734 killResults = main.TRUE
1735 killTime = time.time()
Jon Halle1a3b752015-07-22 13:02:46 -07001736 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001737 killed = main.ONOSbench.onosKill( node.ip_address )
1738 killResults = killResults and killed
1739 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1740 onpass="ONOS nodes killed",
1741 onfail="ONOS kill unsuccessful" )
1742
1743 main.step( "Checking if ONOS is up yet" )
1744 for i in range( 2 ):
1745 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001746 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001747 started = main.ONOSbench.isup( node.ip_address )
1748 if not started:
1749 main.log.error( node.name + " didn't start!" )
1750 onosIsupResult = onosIsupResult and started
1751 if onosIsupResult == main.TRUE:
1752 break
1753 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1754 onpass="ONOS restarted",
1755 onfail="ONOS restart NOT successful" )
1756
1757 main.log.step( "Starting ONOS CLI sessions" )
1758 cliResults = main.TRUE
1759 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001760 for i in range( main.numCtrls ):
1761 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -07001762 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -07001763 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001764 threads.append( t )
1765 t.start()
1766
1767 for t in threads:
1768 t.join()
1769 cliResults = cliResults and t.result
1770 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1771 onpass="ONOS cli started",
1772 onfail="ONOS clis did not restart" )
1773
Jon Hall6e709752016-02-01 13:38:46 -08001774 for i in range( 10 ):
1775 ready = True
1776 for cli in main.CLIs:
1777 output = cli.summary()
1778 if not output:
1779 ready = False
1780 time.sleep( 30 )
1781 utilities.assert_equals( expect=True, actual=ready,
1782 onpass="ONOS summary command succeded",
1783 onfail="ONOS summary command failed" )
1784 if not ready:
1785 main.cleanup()
1786 main.exit()
1787
Jon Hall5cf14d52015-07-16 12:15:19 -07001788 # Grab the time of restart so we chan check how long the gossip
1789 # protocol has had time to work
1790 main.restartTime = time.time() - killTime
1791 main.log.debug( "Restart time: " + str( main.restartTime ) )
1792 labels.append( "Restart" )
1793 data.append( str( main.restartTime ) )
1794
1795 # FIXME: revisit test plan for election with madan
1796 # Rerun for election on restarted nodes
1797 runResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001798 for cli in main.CLIs:
1799 run = main.CLIs[0].electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001800 if run != main.TRUE:
1801 main.log.error( "Error running for election on " + cli.name )
1802 runResults = runResults and run
1803 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1804 onpass="Reran for election",
1805 onfail="Failed to rerun for election" )
1806
1807 # TODO: Make this configurable
1808 time.sleep( 60 )
Jon Halle1a3b752015-07-22 13:02:46 -07001809 main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) )
1810 main.log.debug( main.CLIs[0].leaders( jsonFormat=False ) )
1811 main.log.debug( main.CLIs[0].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001812
1813 def CASE7( self, main ):
1814 """
1815 Check state after ONOS failure
1816 """
1817 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001818 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001819 assert main, "main not defined"
1820 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001821 assert main.CLIs, "main.CLIs not defined"
1822 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001823 main.case( "Running ONOS Constant State Tests" )
1824
1825 main.step( "Check that each switch has a master" )
1826 # Assert that each device has a master
1827 rolesNotNull = main.TRUE
1828 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001829 for i in range( main.numCtrls ):
1830 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001831 name="rolesNotNull-" + str( i ),
1832 args=[ ] )
1833 threads.append( t )
1834 t.start()
1835
1836 for t in threads:
1837 t.join()
1838 rolesNotNull = rolesNotNull and t.result
1839 utilities.assert_equals(
1840 expect=main.TRUE,
1841 actual=rolesNotNull,
1842 onpass="Each device has a master",
1843 onfail="Some devices don't have a master assigned" )
1844
1845 main.step( "Read device roles from ONOS" )
1846 ONOSMastership = []
1847 mastershipCheck = main.FALSE
1848 consistentMastership = True
1849 rolesResults = True
1850 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001851 for i in range( main.numCtrls ):
1852 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001853 name="roles-" + str( i ),
1854 args=[] )
1855 threads.append( t )
1856 t.start()
1857
1858 for t in threads:
1859 t.join()
1860 ONOSMastership.append( t.result )
1861
Jon Halle1a3b752015-07-22 13:02:46 -07001862 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001863 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1864 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1865 " roles" )
1866 main.log.warn(
1867 "ONOS" + str( i + 1 ) + " mastership response: " +
1868 repr( ONOSMastership[i] ) )
1869 rolesResults = False
1870 utilities.assert_equals(
1871 expect=True,
1872 actual=rolesResults,
1873 onpass="No error in reading roles output",
1874 onfail="Error in reading roles from ONOS" )
1875
1876 main.step( "Check for consistency in roles from each controller" )
1877 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1878 main.log.info(
1879 "Switch roles are consistent across all ONOS nodes" )
1880 else:
1881 consistentMastership = False
1882 utilities.assert_equals(
1883 expect=True,
1884 actual=consistentMastership,
1885 onpass="Switch roles are consistent across all ONOS nodes",
1886 onfail="ONOS nodes have different views of switch roles" )
1887
1888 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001889 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08001890 main.log.warn( "ONOS" + str( i + 1 ) + " roles: ",
1891 json.dumps( json.loads( ONOSMastership[ i ] ),
1892 sort_keys=True,
1893 indent=4,
1894 separators=( ',', ': ' ) ) )
1895 elif rolesResults and consistentMastership:
Jon Hall5cf14d52015-07-16 12:15:19 -07001896 mastershipCheck = main.TRUE
1897
Jon Hall5cf14d52015-07-16 12:15:19 -07001898 # NOTE: we expect mastership to change on controller failure
1899
1900 main.step( "Get the intents and compare across all nodes" )
1901 ONOSIntents = []
1902 intentCheck = main.FALSE
1903 consistentIntents = True
1904 intentsResults = True
1905 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001906 for i in range( main.numCtrls ):
1907 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001908 name="intents-" + str( i ),
1909 args=[],
1910 kwargs={ 'jsonFormat': True } )
1911 threads.append( t )
1912 t.start()
1913
1914 for t in threads:
1915 t.join()
1916 ONOSIntents.append( t.result )
1917
Jon Halle1a3b752015-07-22 13:02:46 -07001918 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001919 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1920 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1921 " intents" )
1922 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1923 repr( ONOSIntents[ i ] ) )
1924 intentsResults = False
1925 utilities.assert_equals(
1926 expect=True,
1927 actual=intentsResults,
1928 onpass="No error in reading intents output",
1929 onfail="Error in reading intents from ONOS" )
1930
1931 main.step( "Check for consistency in Intents from each controller" )
1932 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1933 main.log.info( "Intents are consistent across all ONOS " +
1934 "nodes" )
1935 else:
1936 consistentIntents = False
1937
1938 # Try to make it easy to figure out what is happening
1939 #
1940 # Intent ONOS1 ONOS2 ...
1941 # 0x01 INSTALLED INSTALLING
1942 # ... ... ...
1943 # ... ... ...
1944 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001945 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001946 title += " " * 10 + "ONOS" + str( n + 1 )
1947 main.log.warn( title )
1948 # get all intent keys in the cluster
1949 keys = []
1950 for nodeStr in ONOSIntents:
1951 node = json.loads( nodeStr )
1952 for intent in node:
1953 keys.append( intent.get( 'id' ) )
1954 keys = set( keys )
1955 for key in keys:
1956 row = "%-13s" % key
1957 for nodeStr in ONOSIntents:
1958 node = json.loads( nodeStr )
1959 for intent in node:
1960 if intent.get( 'id' ) == key:
1961 row += "%-15s" % intent.get( 'state' )
1962 main.log.warn( row )
1963 # End table view
1964
1965 utilities.assert_equals(
1966 expect=True,
1967 actual=consistentIntents,
1968 onpass="Intents are consistent across all ONOS nodes",
1969 onfail="ONOS nodes have different views of intents" )
1970 intentStates = []
1971 for node in ONOSIntents: # Iter through ONOS nodes
1972 nodeStates = []
1973 # Iter through intents of a node
1974 try:
1975 for intent in json.loads( node ):
1976 nodeStates.append( intent[ 'state' ] )
1977 except ( ValueError, TypeError ):
1978 main.log.exception( "Error in parsing intents" )
1979 main.log.error( repr( node ) )
1980 intentStates.append( nodeStates )
1981 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1982 main.log.info( dict( out ) )
1983
1984 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001985 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001986 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1987 main.log.warn( json.dumps(
1988 json.loads( ONOSIntents[ i ] ),
1989 sort_keys=True,
1990 indent=4,
1991 separators=( ',', ': ' ) ) )
1992 elif intentsResults and consistentIntents:
1993 intentCheck = main.TRUE
1994
1995 # NOTE: Store has no durability, so intents are lost across system
1996 # restarts
1997 """
1998 main.step( "Compare current intents with intents before the failure" )
1999 # NOTE: this requires case 5 to pass for intentState to be set.
2000 # maybe we should stop the test if that fails?
2001 sameIntents = main.FALSE
2002 if intentState and intentState == ONOSIntents[ 0 ]:
2003 sameIntents = main.TRUE
2004 main.log.info( "Intents are consistent with before failure" )
2005 # TODO: possibly the states have changed? we may need to figure out
2006 # what the acceptable states are
2007 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2008 sameIntents = main.TRUE
2009 try:
2010 before = json.loads( intentState )
2011 after = json.loads( ONOSIntents[ 0 ] )
2012 for intent in before:
2013 if intent not in after:
2014 sameIntents = main.FALSE
2015 main.log.debug( "Intent is not currently in ONOS " +
2016 "(at least in the same form):" )
2017 main.log.debug( json.dumps( intent ) )
2018 except ( ValueError, TypeError ):
2019 main.log.exception( "Exception printing intents" )
2020 main.log.debug( repr( ONOSIntents[0] ) )
2021 main.log.debug( repr( intentState ) )
2022 if sameIntents == main.FALSE:
2023 try:
2024 main.log.debug( "ONOS intents before: " )
2025 main.log.debug( json.dumps( json.loads( intentState ),
2026 sort_keys=True, indent=4,
2027 separators=( ',', ': ' ) ) )
2028 main.log.debug( "Current ONOS intents: " )
2029 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2030 sort_keys=True, indent=4,
2031 separators=( ',', ': ' ) ) )
2032 except ( ValueError, TypeError ):
2033 main.log.exception( "Exception printing intents" )
2034 main.log.debug( repr( ONOSIntents[0] ) )
2035 main.log.debug( repr( intentState ) )
2036 utilities.assert_equals(
2037 expect=main.TRUE,
2038 actual=sameIntents,
2039 onpass="Intents are consistent with before failure",
2040 onfail="The Intents changed during failure" )
2041 intentCheck = intentCheck and sameIntents
2042 """
2043 main.step( "Get the OF Table entries and compare to before " +
2044 "component failure" )
2045 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002046 for i in range( 28 ):
2047 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002048 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
2049 FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
Jon Hall5cf14d52015-07-16 12:15:19 -07002050 if FlowTables == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002051 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002052 utilities.assert_equals(
2053 expect=main.TRUE,
2054 actual=FlowTables,
2055 onpass="No changes were found in the flow tables",
2056 onfail="Changes were found in the flow tables" )
2057
2058 main.Mininet2.pingLongKill()
2059 '''
2060 # main.step( "Check the continuous pings to ensure that no packets " +
2061 # "were dropped during component failure" )
2062 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2063 main.params[ 'TESTONIP' ] )
2064 LossInPings = main.FALSE
2065 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2066 for i in range( 8, 18 ):
2067 main.log.info(
2068 "Checking for a loss in pings along flow from s" +
2069 str( i ) )
2070 LossInPings = main.Mininet2.checkForLoss(
2071 "/tmp/ping.h" +
2072 str( i ) ) or LossInPings
2073 if LossInPings == main.TRUE:
2074 main.log.info( "Loss in ping detected" )
2075 elif LossInPings == main.ERROR:
2076 main.log.info( "There are multiple mininet process running" )
2077 elif LossInPings == main.FALSE:
2078 main.log.info( "No Loss in the pings" )
2079 main.log.info( "No loss of dataplane connectivity" )
2080 # utilities.assert_equals(
2081 # expect=main.FALSE,
2082 # actual=LossInPings,
2083 # onpass="No Loss of connectivity",
2084 # onfail="Loss of dataplane connectivity detected" )
2085
2086 # NOTE: Since intents are not persisted with IntnentStore,
2087 # we expect loss in dataplane connectivity
2088 LossInPings = main.FALSE
2089 '''
2090
2091 main.step( "Leadership Election is still functional" )
2092 # Test of LeadershipElection
2093 leaderList = []
2094 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002095 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002096 leaderN = cli.electionTestLeader()
2097 leaderList.append( leaderN )
2098 if leaderN == main.FALSE:
2099 # error in response
2100 main.log.error( "Something is wrong with " +
2101 "electionTestLeader function, check the" +
2102 " error logs" )
2103 leaderResult = main.FALSE
2104 elif leaderN is None:
2105 main.log.error( cli.name +
2106 " shows no leader for the election-app." )
2107 leaderResult = main.FALSE
2108 if len( set( leaderList ) ) != 1:
2109 leaderResult = main.FALSE
2110 main.log.error(
2111 "Inconsistent view of leader for the election test app" )
2112 # TODO: print the list
2113 utilities.assert_equals(
2114 expect=main.TRUE,
2115 actual=leaderResult,
2116 onpass="Leadership election passed",
2117 onfail="Something went wrong with Leadership election" )
2118
2119 def CASE8( self, main ):
2120 """
2121 Compare topo
2122 """
2123 import json
2124 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002125 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002126 assert main, "main not defined"
2127 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002128 assert main.CLIs, "main.CLIs not defined"
2129 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002130
2131 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002132 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002133 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002134 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002135 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002136 elapsed = 0
2137 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002138 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002139 startTime = time.time()
2140 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002141 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hallba609822015-09-18 12:00:21 -07002142 devicesResults = main.TRUE
2143 linksResults = main.TRUE
2144 hostsResults = main.TRUE
2145 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002146 count += 1
2147 cliStart = time.time()
2148 devices = []
2149 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002150 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08002151 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002152 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002153 args=[ main.CLIs[i].devices, [ None ] ],
2154 kwargs= { 'sleep': 5, 'attempts': 5,
2155 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002156 threads.append( t )
2157 t.start()
2158
2159 for t in threads:
2160 t.join()
2161 devices.append( t.result )
2162 hosts = []
2163 ipResult = main.TRUE
2164 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002165 for i in range( main.numCtrls ):
Jon Halld8f6de82015-12-17 17:04:34 -08002166 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002167 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002168 args=[ main.CLIs[i].hosts, [ None ] ],
2169 kwargs= { 'sleep': 5, 'attempts': 5,
2170 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002171 threads.append( t )
2172 t.start()
2173
2174 for t in threads:
2175 t.join()
2176 try:
2177 hosts.append( json.loads( t.result ) )
2178 except ( ValueError, TypeError ):
2179 main.log.exception( "Error parsing hosts results" )
2180 main.log.error( repr( t.result ) )
Jon Hall3afe4c92015-12-14 19:30:38 -08002181 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002182 for controller in range( 0, len( hosts ) ):
2183 controllerStr = str( controller + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002184 if hosts[ controller ]:
2185 for host in hosts[ controller ]:
2186 if host is None or host.get( 'ipAddresses', [] ) == []:
2187 main.log.error(
2188 "Error with host ipAddresses on controller" +
2189 controllerStr + ": " + str( host ) )
2190 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002191 ports = []
2192 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002193 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08002194 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002195 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002196 args=[ main.CLIs[i].ports, [ None ] ],
2197 kwargs= { 'sleep': 5, 'attempts': 5,
2198 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002199 threads.append( t )
2200 t.start()
2201
2202 for t in threads:
2203 t.join()
2204 ports.append( t.result )
2205 links = []
2206 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002207 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08002208 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002209 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002210 args=[ main.CLIs[i].links, [ None ] ],
2211 kwargs= { 'sleep': 5, 'attempts': 5,
2212 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002213 threads.append( t )
2214 t.start()
2215
2216 for t in threads:
2217 t.join()
2218 links.append( t.result )
2219 clusters = []
2220 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002221 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08002222 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002223 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002224 args=[ main.CLIs[i].clusters, [ None ] ],
2225 kwargs= { 'sleep': 5, 'attempts': 5,
2226 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002227 threads.append( t )
2228 t.start()
2229
2230 for t in threads:
2231 t.join()
2232 clusters.append( t.result )
2233
2234 elapsed = time.time() - startTime
2235 cliTime = time.time() - cliStart
2236 print "Elapsed time: " + str( elapsed )
2237 print "CLI time: " + str( cliTime )
2238
Jon Hall6e709752016-02-01 13:38:46 -08002239 if all( e is None for e in devices ) and\
2240 all( e is None for e in hosts ) and\
2241 all( e is None for e in ports ) and\
2242 all( e is None for e in links ) and\
2243 all( e is None for e in clusters ):
2244 topoFailMsg = "Could not get topology from ONOS"
2245 main.log.error( topoFailMsg )
2246 continue # Try again, No use trying to compare
2247
Jon Hall5cf14d52015-07-16 12:15:19 -07002248 mnSwitches = main.Mininet1.getSwitches()
2249 mnLinks = main.Mininet1.getLinks()
2250 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002251 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002252 controllerStr = str( controller + 1 )
2253 if devices[ controller ] and ports[ controller ] and\
2254 "Error" not in devices[ controller ] and\
2255 "Error" not in ports[ controller ]:
2256
Jon Hallc6793552016-01-19 14:18:37 -08002257 try:
2258 currentDevicesResult = main.Mininet1.compareSwitches(
2259 mnSwitches,
2260 json.loads( devices[ controller ] ),
2261 json.loads( ports[ controller ] ) )
2262 except ( TypeError, ValueError ) as e:
2263 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2264 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002265 else:
2266 currentDevicesResult = main.FALSE
2267 utilities.assert_equals( expect=main.TRUE,
2268 actual=currentDevicesResult,
2269 onpass="ONOS" + controllerStr +
2270 " Switches view is correct",
2271 onfail="ONOS" + controllerStr +
2272 " Switches view is incorrect" )
2273
2274 if links[ controller ] and "Error" not in links[ controller ]:
2275 currentLinksResult = main.Mininet1.compareLinks(
2276 mnSwitches, mnLinks,
2277 json.loads( links[ controller ] ) )
2278 else:
2279 currentLinksResult = main.FALSE
2280 utilities.assert_equals( expect=main.TRUE,
2281 actual=currentLinksResult,
2282 onpass="ONOS" + controllerStr +
2283 " links view is correct",
2284 onfail="ONOS" + controllerStr +
2285 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002286 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002287 currentHostsResult = main.Mininet1.compareHosts(
2288 mnHosts,
2289 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002290 elif hosts[ controller ] == []:
2291 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002292 else:
2293 currentHostsResult = main.FALSE
2294 utilities.assert_equals( expect=main.TRUE,
2295 actual=currentHostsResult,
2296 onpass="ONOS" + controllerStr +
2297 " hosts exist in Mininet",
2298 onfail="ONOS" + controllerStr +
2299 " hosts don't match Mininet" )
2300 # CHECKING HOST ATTACHMENT POINTS
2301 hostAttachment = True
2302 noHosts = False
2303 # FIXME: topo-HA/obelisk specific mappings:
2304 # key is mac and value is dpid
2305 mappings = {}
2306 for i in range( 1, 29 ): # hosts 1 through 28
2307 # set up correct variables:
2308 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2309 if i == 1:
2310 deviceId = "1000".zfill(16)
2311 elif i == 2:
2312 deviceId = "2000".zfill(16)
2313 elif i == 3:
2314 deviceId = "3000".zfill(16)
2315 elif i == 4:
2316 deviceId = "3004".zfill(16)
2317 elif i == 5:
2318 deviceId = "5000".zfill(16)
2319 elif i == 6:
2320 deviceId = "6000".zfill(16)
2321 elif i == 7:
2322 deviceId = "6007".zfill(16)
2323 elif i >= 8 and i <= 17:
2324 dpid = '3' + str( i ).zfill( 3 )
2325 deviceId = dpid.zfill(16)
2326 elif i >= 18 and i <= 27:
2327 dpid = '6' + str( i ).zfill( 3 )
2328 deviceId = dpid.zfill(16)
2329 elif i == 28:
2330 deviceId = "2800".zfill(16)
2331 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002332 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002333 if hosts[ controller ] == []:
2334 main.log.warn( "There are no hosts discovered" )
2335 noHosts = True
2336 else:
2337 for host in hosts[ controller ]:
2338 mac = None
2339 location = None
2340 device = None
2341 port = None
2342 try:
2343 mac = host.get( 'mac' )
2344 assert mac, "mac field could not be found for this host object"
2345
2346 location = host.get( 'location' )
2347 assert location, "location field could not be found for this host object"
2348
2349 # Trim the protocol identifier off deviceId
2350 device = str( location.get( 'elementId' ) ).split(':')[1]
2351 assert device, "elementId field could not be found for this host location object"
2352
2353 port = location.get( 'port' )
2354 assert port, "port field could not be found for this host location object"
2355
2356 # Now check if this matches where they should be
2357 if mac and device and port:
2358 if str( port ) != "1":
2359 main.log.error( "The attachment port is incorrect for " +
2360 "host " + str( mac ) +
2361 ". Expected: 1 Actual: " + str( port) )
2362 hostAttachment = False
2363 if device != mappings[ str( mac ) ]:
2364 main.log.error( "The attachment device is incorrect for " +
2365 "host " + str( mac ) +
2366 ". Expected: " + mappings[ str( mac ) ] +
2367 " Actual: " + device )
2368 hostAttachment = False
2369 else:
2370 hostAttachment = False
2371 except AssertionError:
2372 main.log.exception( "Json object not as expected" )
2373 main.log.error( repr( host ) )
2374 hostAttachment = False
2375 else:
2376 main.log.error( "No hosts json output or \"Error\"" +
2377 " in output. hosts = " +
2378 repr( hosts[ controller ] ) )
2379 if noHosts is False:
2380 # TODO: Find a way to know if there should be hosts in a
2381 # given point of the test
2382 hostAttachment = True
2383
2384 # END CHECKING HOST ATTACHMENT POINTS
2385 devicesResults = devicesResults and currentDevicesResult
2386 linksResults = linksResults and currentLinksResult
2387 hostsResults = hostsResults and currentHostsResult
2388 hostAttachmentResults = hostAttachmentResults and\
2389 hostAttachment
2390 topoResult = ( devicesResults and linksResults
2391 and hostsResults and ipResult and
2392 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002393 utilities.assert_equals( expect=True,
2394 actual=topoResult,
2395 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002396 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002397 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002398
2399 # Compare json objects for hosts and dataplane clusters
2400
2401 # hosts
2402 main.step( "Hosts view is consistent across all ONOS nodes" )
2403 consistentHostsResult = main.TRUE
2404 for controller in range( len( hosts ) ):
2405 controllerStr = str( controller + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002406 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002407 if hosts[ controller ] == hosts[ 0 ]:
2408 continue
2409 else: # hosts not consistent
2410 main.log.error( "hosts from ONOS" + controllerStr +
2411 " is inconsistent with ONOS1" )
2412 main.log.warn( repr( hosts[ controller ] ) )
2413 consistentHostsResult = main.FALSE
2414
2415 else:
2416 main.log.error( "Error in getting ONOS hosts from ONOS" +
2417 controllerStr )
2418 consistentHostsResult = main.FALSE
2419 main.log.warn( "ONOS" + controllerStr +
2420 " hosts response: " +
2421 repr( hosts[ controller ] ) )
2422 utilities.assert_equals(
2423 expect=main.TRUE,
2424 actual=consistentHostsResult,
2425 onpass="Hosts view is consistent across all ONOS nodes",
2426 onfail="ONOS nodes have different views of hosts" )
2427
2428 main.step( "Hosts information is correct" )
2429 hostsResults = hostsResults and ipResult
2430 utilities.assert_equals(
2431 expect=main.TRUE,
2432 actual=hostsResults,
2433 onpass="Host information is correct",
2434 onfail="Host information is incorrect" )
2435
2436 main.step( "Host attachment points to the network" )
2437 utilities.assert_equals(
2438 expect=True,
2439 actual=hostAttachmentResults,
2440 onpass="Hosts are correctly attached to the network",
2441 onfail="ONOS did not correctly attach hosts to the network" )
2442
2443 # Strongly connected clusters of devices
2444 main.step( "Clusters view is consistent across all ONOS nodes" )
2445 consistentClustersResult = main.TRUE
2446 for controller in range( len( clusters ) ):
2447 controllerStr = str( controller + 1 )
2448 if "Error" not in clusters[ controller ]:
2449 if clusters[ controller ] == clusters[ 0 ]:
2450 continue
2451 else: # clusters not consistent
2452 main.log.error( "clusters from ONOS" +
2453 controllerStr +
2454 " is inconsistent with ONOS1" )
2455 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002456 else:
2457 main.log.error( "Error in getting dataplane clusters " +
2458 "from ONOS" + controllerStr )
2459 consistentClustersResult = main.FALSE
2460 main.log.warn( "ONOS" + controllerStr +
2461 " clusters response: " +
2462 repr( clusters[ controller ] ) )
2463 utilities.assert_equals(
2464 expect=main.TRUE,
2465 actual=consistentClustersResult,
2466 onpass="Clusters view is consistent across all ONOS nodes",
2467 onfail="ONOS nodes have different views of clusters" )
2468
2469 main.step( "There is only one SCC" )
2470 # there should always only be one cluster
2471 try:
2472 numClusters = len( json.loads( clusters[ 0 ] ) )
2473 except ( ValueError, TypeError ):
2474 main.log.exception( "Error parsing clusters[0]: " +
2475 repr( clusters[0] ) )
2476 clusterResults = main.FALSE
2477 if numClusters == 1:
2478 clusterResults = main.TRUE
2479 utilities.assert_equals(
2480 expect=1,
2481 actual=numClusters,
2482 onpass="ONOS shows 1 SCC",
2483 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2484
2485 topoResult = ( devicesResults and linksResults
2486 and hostsResults and consistentHostsResult
2487 and consistentClustersResult and clusterResults
2488 and ipResult and hostAttachmentResults )
2489
2490 topoResult = topoResult and int( count <= 2 )
2491 note = "note it takes about " + str( int( cliTime ) ) + \
2492 " seconds for the test to make all the cli calls to fetch " +\
2493 "the topology from each ONOS instance"
2494 main.log.info(
2495 "Very crass estimate for topology discovery/convergence( " +
2496 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2497 str( count ) + " tries" )
2498
2499 main.step( "Device information is correct" )
2500 utilities.assert_equals(
2501 expect=main.TRUE,
2502 actual=devicesResults,
2503 onpass="Device information is correct",
2504 onfail="Device information is incorrect" )
2505
2506 main.step( "Links are correct" )
2507 utilities.assert_equals(
2508 expect=main.TRUE,
2509 actual=linksResults,
2510 onpass="Link are correct",
2511 onfail="Links are incorrect" )
2512
2513 # FIXME: move this to an ONOS state case
2514 main.step( "Checking ONOS nodes" )
2515 nodesOutput = []
2516 nodeResults = main.TRUE
2517 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002518 for i in range( main.numCtrls ):
2519 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002520 name="nodes-" + str( i ),
2521 args=[ ] )
2522 threads.append( t )
2523 t.start()
2524
2525 for t in threads:
2526 t.join()
2527 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002528 ips = [ node.ip_address for node in main.nodes ]
Jon Halle9b1fa32015-12-08 15:32:21 -08002529 ips.sort()
Jon Hall5cf14d52015-07-16 12:15:19 -07002530 for i in nodesOutput:
2531 try:
2532 current = json.loads( i )
Jon Halle9b1fa32015-12-08 15:32:21 -08002533 activeIps = []
2534 currentResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002535 for node in current:
Jon Hallbd182782016-03-28 16:42:22 -07002536 if node['state'] == 'READY':
Jon Halle9b1fa32015-12-08 15:32:21 -08002537 activeIps.append( node['ip'] )
2538 activeIps.sort()
2539 if ips == activeIps:
2540 currentResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002541 except ( ValueError, TypeError ):
2542 main.log.error( "Error parsing nodes output" )
2543 main.log.warn( repr( i ) )
Jon Halle9b1fa32015-12-08 15:32:21 -08002544 currentResult = main.FALSE
2545 nodeResults = nodeResults and currentResult
Jon Hall5cf14d52015-07-16 12:15:19 -07002546 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2547 onpass="Nodes check successful",
2548 onfail="Nodes check NOT successful" )
2549
2550 def CASE9( self, main ):
2551 """
2552 Link s3-s28 down
2553 """
2554 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002555 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002556 assert main, "main not defined"
2557 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002558 assert main.CLIs, "main.CLIs not defined"
2559 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002560 # NOTE: You should probably run a topology check after this
2561
2562 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2563
2564 description = "Turn off a link to ensure that Link Discovery " +\
2565 "is working properly"
2566 main.case( description )
2567
2568 main.step( "Kill Link between s3 and s28" )
2569 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2570 main.log.info( "Waiting " + str( linkSleep ) +
2571 " seconds for link down to be discovered" )
2572 time.sleep( linkSleep )
2573 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2574 onpass="Link down successful",
2575 onfail="Failed to bring link down" )
2576 # TODO do some sort of check here
2577
2578 def CASE10( self, main ):
2579 """
2580 Link s3-s28 up
2581 """
2582 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002583 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002584 assert main, "main not defined"
2585 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002586 assert main.CLIs, "main.CLIs not defined"
2587 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002588 # NOTE: You should probably run a topology check after this
2589
2590 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2591
2592 description = "Restore a link to ensure that Link Discovery is " + \
2593 "working properly"
2594 main.case( description )
2595
2596 main.step( "Bring link between s3 and s28 back up" )
2597 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2598 main.log.info( "Waiting " + str( linkSleep ) +
2599 " seconds for link up to be discovered" )
2600 time.sleep( linkSleep )
2601 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2602 onpass="Link up successful",
2603 onfail="Failed to bring link up" )
2604 # TODO do some sort of check here
2605
2606 def CASE11( self, main ):
2607 """
2608 Switch Down
2609 """
2610 # NOTE: You should probably run a topology check after this
2611 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002612 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002613 assert main, "main not defined"
2614 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002615 assert main.CLIs, "main.CLIs not defined"
2616 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002617
2618 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2619
2620 description = "Killing a switch to ensure it is discovered correctly"
2621 main.case( description )
2622 switch = main.params[ 'kill' ][ 'switch' ]
2623 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2624
2625 # TODO: Make this switch parameterizable
2626 main.step( "Kill " + switch )
2627 main.log.info( "Deleting " + switch )
2628 main.Mininet1.delSwitch( switch )
2629 main.log.info( "Waiting " + str( switchSleep ) +
2630 " seconds for switch down to be discovered" )
2631 time.sleep( switchSleep )
2632 device = main.ONOScli1.getDevice( dpid=switchDPID )
2633 # Peek at the deleted switch
2634 main.log.warn( str( device ) )
2635 result = main.FALSE
2636 if device and device[ 'available' ] is False:
2637 result = main.TRUE
2638 utilities.assert_equals( expect=main.TRUE, actual=result,
2639 onpass="Kill switch successful",
2640 onfail="Failed to kill switch?" )
2641
2642 def CASE12( self, main ):
2643 """
2644 Switch Up
2645 """
2646 # NOTE: You should probably run a topology check after this
2647 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002648 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002649 assert main, "main not defined"
2650 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002651 assert main.CLIs, "main.CLIs not defined"
2652 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002653 assert ONOS1Port, "ONOS1Port not defined"
2654 assert ONOS2Port, "ONOS2Port not defined"
2655 assert ONOS3Port, "ONOS3Port not defined"
2656 assert ONOS4Port, "ONOS4Port not defined"
2657 assert ONOS5Port, "ONOS5Port not defined"
2658 assert ONOS6Port, "ONOS6Port not defined"
2659 assert ONOS7Port, "ONOS7Port not defined"
2660
2661 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2662 switch = main.params[ 'kill' ][ 'switch' ]
2663 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2664 links = main.params[ 'kill' ][ 'links' ].split()
2665 description = "Adding a switch to ensure it is discovered correctly"
2666 main.case( description )
2667
2668 main.step( "Add back " + switch )
2669 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2670 for peer in links:
2671 main.Mininet1.addLink( switch, peer )
2672 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002673 for i in range( main.numCtrls ):
2674 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002675 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2676 main.log.info( "Waiting " + str( switchSleep ) +
2677 " seconds for switch up to be discovered" )
2678 time.sleep( switchSleep )
2679 device = main.ONOScli1.getDevice( dpid=switchDPID )
2680 # Peek at the deleted switch
2681 main.log.warn( str( device ) )
2682 result = main.FALSE
2683 if device and device[ 'available' ]:
2684 result = main.TRUE
2685 utilities.assert_equals( expect=main.TRUE, actual=result,
2686 onpass="add switch successful",
2687 onfail="Failed to add switch?" )
2688
2689 def CASE13( self, main ):
2690 """
2691 Clean up
2692 """
2693 import os
2694 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002695 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002696 assert main, "main not defined"
2697 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002698 assert main.CLIs, "main.CLIs not defined"
2699 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002700
2701 # printing colors to terminal
2702 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2703 'blue': '\033[94m', 'green': '\033[92m',
2704 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2705 main.case( "Test Cleanup" )
2706 main.step( "Killing tcpdumps" )
2707 main.Mininet2.stopTcpdump()
2708
2709 testname = main.TEST
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002710 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002711 main.step( "Copying MN pcap and ONOS log files to test station" )
2712 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2713 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002714 # NOTE: MN Pcap file is being saved to logdir.
2715 # We scp this file as MN and TestON aren't necessarily the same vm
2716
2717 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002718 # TODO: Load these from params
2719 # NOTE: must end in /
2720 logFolder = "/opt/onos/log/"
2721 logFiles = [ "karaf.log", "karaf.log.1" ]
2722 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002723 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002724 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002725 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002726 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2727 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002728 # std*.log's
2729 # NOTE: must end in /
2730 logFolder = "/opt/onos/var/"
2731 logFiles = [ "stderr.log", "stdout.log" ]
2732 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002733 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002734 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002735 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002736 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2737 logFolder + f, dstName )
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002738 else:
2739 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002740
2741 main.step( "Stopping Mininet" )
2742 mnResult = main.Mininet1.stopNet()
2743 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2744 onpass="Mininet stopped",
2745 onfail="MN cleanup NOT successful" )
2746
2747 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002748 for node in main.nodes:
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002749 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2750 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002751
2752 try:
2753 timerLog = open( main.logdir + "/Timers.csv", 'w')
2754 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2755 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2756 timerLog.close()
2757 except NameError, e:
2758 main.log.exception(e)
2759
2760 def CASE14( self, main ):
2761 """
2762 start election app on all onos nodes
2763 """
Jon Halle1a3b752015-07-22 13:02:46 -07002764 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002765 assert main, "main not defined"
2766 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002767 assert main.CLIs, "main.CLIs not defined"
2768 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002769
2770 main.case("Start Leadership Election app")
2771 main.step( "Install leadership election app" )
2772 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2773 utilities.assert_equals(
2774 expect=main.TRUE,
2775 actual=appResult,
2776 onpass="Election app installed",
2777 onfail="Something went wrong with installing Leadership election" )
2778
2779 main.step( "Run for election on each node" )
2780 leaderResult = main.TRUE
2781 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002782 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002783 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002784 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002785 leader = cli.electionTestLeader()
2786 if leader is None or leader == main.FALSE:
2787 main.log.error( cli.name + ": Leader for the election app " +
2788 "should be an ONOS node, instead got '" +
2789 str( leader ) + "'" )
2790 leaderResult = main.FALSE
2791 leaders.append( leader )
2792 utilities.assert_equals(
2793 expect=main.TRUE,
2794 actual=leaderResult,
2795 onpass="Successfully ran for leadership",
2796 onfail="Failed to run for leadership" )
2797
2798 main.step( "Check that each node shows the same leader" )
2799 sameLeader = main.TRUE
2800 if len( set( leaders ) ) != 1:
2801 sameLeader = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002802 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002803 str( leaders ) )
2804 utilities.assert_equals(
2805 expect=main.TRUE,
2806 actual=sameLeader,
2807 onpass="Leadership is consistent for the election topic",
2808 onfail="Nodes have different leaders" )
2809
2810 def CASE15( self, main ):
2811 """
2812 Check that Leadership Election is still functional
acsmars9475b1c2015-08-28 18:02:08 -07002813 15.1 Run election on each node
2814 15.2 Check that each node has the same leaders and candidates
2815 15.3 Find current leader and withdraw
2816 15.4 Check that a new node was elected leader
2817 15.5 Check that that new leader was the candidate of old leader
2818 15.6 Run for election on old leader
2819 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2820 15.8 Make sure that the old leader was added to the candidate list
2821
2822 old and new variable prefixes refer to data from before vs after
2823 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002824 """
2825 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002826 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002827 assert main, "main not defined"
2828 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002829 assert main.CLIs, "main.CLIs not defined"
2830 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002831
Jon Hall5cf14d52015-07-16 12:15:19 -07002832 description = "Check that Leadership Election is still functional"
2833 main.case( description )
2834 # NOTE: Need to re-run since being a canidate is not persistant
acsmars9475b1c2015-08-28 18:02:08 -07002835 # TODO: add check for "Command not found:" in the driver, this
2836 # means the election test app isn't loaded
2837
2838 oldLeaders = [] # leaders by node before withdrawl from candidates
2839 newLeaders = [] # leaders by node after withdrawl from candidates
2840 oldAllCandidates = [] # list of lists of each nodes' candidates before
2841 newAllCandidates = [] # list of lists of each nodes' candidates after
2842 oldCandidates = [] # list of candidates from node 0 before withdrawl
2843 newCandidates = [] # list of candidates from node 0 after withdrawl
2844 oldLeader = '' # the old leader from oldLeaders, None if not same
2845 newLeader = '' # the new leaders fron newLoeaders, None if not same
2846 oldLeaderCLI = None # the CLI of the old leader used for re-electing
acsmars71adceb2015-08-31 15:09:26 -07002847 expectNoLeader = False # True when there is only one leader
2848 if main.numCtrls == 1:
2849 expectNoLeader = True
acsmars9475b1c2015-08-28 18:02:08 -07002850
Jon Hall5cf14d52015-07-16 12:15:19 -07002851 main.step( "Run for election on each node" )
acsmars9475b1c2015-08-28 18:02:08 -07002852 electionResult = main.TRUE
2853
2854 for cli in main.CLIs: # run test election on each node
2855 if cli.electionTestRun() == main.FALSE:
2856 electionResult = main.FALSE
2857
Jon Hall5cf14d52015-07-16 12:15:19 -07002858 utilities.assert_equals(
2859 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002860 actual=electionResult,
2861 onpass="All nodes successfully ran for leadership",
2862 onfail="At least one node failed to run for leadership" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002863
acsmars3a72bde2015-09-02 14:16:22 -07002864 if electionResult == main.FALSE:
2865 main.log.error(
2866 "Skipping Test Case because Election Test App isn't loaded" )
2867 main.skipCase()
2868
acsmars9475b1c2015-08-28 18:02:08 -07002869 main.step( "Check that each node shows the same leader and candidates" )
2870 sameResult = main.TRUE
2871 failMessage = "Nodes have different leaders"
2872 for cli in main.CLIs:
2873 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2874 oldAllCandidates.append( node )
Jon Hall6e709752016-02-01 13:38:46 -08002875 if node:
2876 oldLeaders.append( node[ 0 ] )
2877 else:
2878 oldLeaders.append( None )
acsmars9475b1c2015-08-28 18:02:08 -07002879 oldCandidates = oldAllCandidates[ 0 ]
Jon Hall6e709752016-02-01 13:38:46 -08002880 if oldCandidates is None:
2881 oldCandidates = [ None ]
acsmars9475b1c2015-08-28 18:02:08 -07002882
2883 # Check that each node has the same leader. Defines oldLeader
2884 if len( set( oldLeaders ) ) != 1:
2885 sameResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002886 main.log.error( "More than one leader present:" + str( oldLeaders ) )
acsmars9475b1c2015-08-28 18:02:08 -07002887 oldLeader = None
2888 else:
2889 oldLeader = oldLeaders[ 0 ]
2890
2891 # Check that each node's candidate list is the same
acsmars29233db2015-11-04 11:15:00 -08002892 candidateDiscrepancy = False # Boolean of candidate mismatches
acsmars9475b1c2015-08-28 18:02:08 -07002893 for candidates in oldAllCandidates:
Jon Hall6e709752016-02-01 13:38:46 -08002894 if candidates is None:
2895 main.log.warn( "Error getting candidates" )
2896 candidates = [ None ]
acsmars9475b1c2015-08-28 18:02:08 -07002897 if set( candidates ) != set( oldCandidates ):
2898 sameResult = main.FALSE
acsmars29233db2015-11-04 11:15:00 -08002899 candidateDiscrepancy = True
acsmars29233db2015-11-04 11:15:00 -08002900 if candidateDiscrepancy:
2901 failMessage += " and candidates"
Jon Hall5cf14d52015-07-16 12:15:19 -07002902 utilities.assert_equals(
2903 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002904 actual=sameResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002905 onpass="Leadership is consistent for the election topic",
acsmars9475b1c2015-08-28 18:02:08 -07002906 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002907
2908 main.step( "Find current leader and withdraw" )
acsmars9475b1c2015-08-28 18:02:08 -07002909 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002910 # do some sanity checking on leader before using it
acsmars9475b1c2015-08-28 18:02:08 -07002911 if oldLeader is None:
2912 main.log.error( "Leadership isn't consistent." )
2913 withdrawResult = main.FALSE
2914 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002915 for i in range( len( main.CLIs ) ):
acsmars9475b1c2015-08-28 18:02:08 -07002916 if oldLeader == main.nodes[ i ].ip_address:
2917 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002918 break
2919 else: # FOR/ELSE statement
2920 main.log.error( "Leader election, could not find current leader" )
2921 if oldLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002922 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002923 utilities.assert_equals(
2924 expect=main.TRUE,
2925 actual=withdrawResult,
2926 onpass="Node was withdrawn from election",
2927 onfail="Node was not withdrawn from election" )
2928
acsmars9475b1c2015-08-28 18:02:08 -07002929 main.step( "Check that a new node was elected leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002930 # FIXME: use threads
acsmars9475b1c2015-08-28 18:02:08 -07002931 newLeaderResult = main.TRUE
2932 failMessage = "Nodes have different leaders"
2933
2934 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002935 for cli in main.CLIs:
acsmars9475b1c2015-08-28 18:02:08 -07002936 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
acsmars71adceb2015-08-31 15:09:26 -07002937 # elections might no have finished yet
2938 if node[ 0 ] == 'none' and not expectNoLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002939 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2940 "sure elections are complete." )
2941 time.sleep(5)
2942 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
acsmars71adceb2015-08-31 15:09:26 -07002943 # election still isn't done or there is a problem
2944 if node[ 0 ] == 'none':
2945 main.log.error( "No leader was elected on at least 1 node" )
2946 newLeaderResult = main.FALSE
acsmars9475b1c2015-08-28 18:02:08 -07002947 newAllCandidates.append( node )
2948 newLeaders.append( node[ 0 ] )
2949 newCandidates = newAllCandidates[ 0 ]
2950
2951 # Check that each node has the same leader. Defines newLeader
2952 if len( set( newLeaders ) ) != 1:
2953 newLeaderResult = main.FALSE
2954 main.log.error( "Nodes have different leaders: " +
2955 str( newLeaders ) )
2956 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002957 else:
acsmars9475b1c2015-08-28 18:02:08 -07002958 newLeader = newLeaders[ 0 ]
2959
acsmars71adceb2015-08-31 15:09:26 -07002960 # Check that each node's candidate list is the same
2961 for candidates in newAllCandidates:
2962 if set( candidates ) != set( newCandidates ):
2963 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002964 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002965
acsmars9475b1c2015-08-28 18:02:08 -07002966 # Check that the new leader is not the older leader, which was withdrawn
2967 if newLeader == oldLeader:
2968 newLeaderResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002969 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars9475b1c2015-08-28 18:02:08 -07002970 " as the current leader" )
2971
Jon Hall5cf14d52015-07-16 12:15:19 -07002972 utilities.assert_equals(
2973 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002974 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002975 onpass="Leadership election passed",
2976 onfail="Something went wrong with Leadership election" )
2977
acsmars9475b1c2015-08-28 18:02:08 -07002978 main.step( "Check that that new leader was the candidate of old leader")
Jon Hall6e709752016-02-01 13:38:46 -08002979 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars9475b1c2015-08-28 18:02:08 -07002980 correctCandidateResult = main.TRUE
acsmars71adceb2015-08-31 15:09:26 -07002981 if expectNoLeader:
2982 if newLeader == 'none':
2983 main.log.info( "No leader expected. None found. Pass" )
2984 correctCandidateResult = main.TRUE
2985 else:
2986 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2987 correctCandidateResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002988 elif len( oldCandidates ) >= 3 and newLeader != oldCandidates[ 2 ]:
acsmars9475b1c2015-08-28 18:02:08 -07002989 correctCandidateResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002990 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
2991 newLeader, oldCandidates[ 2 ] ) )
2992 else:
2993 main.log.warn( "Could not determine who should be the correct leader" )
2994 correctCandidateResult = main.FALSE
acsmars9475b1c2015-08-28 18:02:08 -07002995 utilities.assert_equals(
2996 expect=main.TRUE,
2997 actual=correctCandidateResult,
2998 onpass="Correct Candidate Elected",
2999 onfail="Incorrect Candidate Elected" )
3000
Jon Hall5cf14d52015-07-16 12:15:19 -07003001 main.step( "Run for election on old leader( just so everyone " +
3002 "is in the hat )" )
acsmars9475b1c2015-08-28 18:02:08 -07003003 if oldLeaderCLI is not None:
3004 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003005 else:
acsmars9475b1c2015-08-28 18:02:08 -07003006 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003007 runResult = main.FALSE
3008 utilities.assert_equals(
3009 expect=main.TRUE,
3010 actual=runResult,
3011 onpass="App re-ran for election",
3012 onfail="App failed to run for election" )
acsmars9475b1c2015-08-28 18:02:08 -07003013 main.step(
3014 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003015 # verify leader didn't just change
acsmars9475b1c2015-08-28 18:02:08 -07003016 positionResult = main.TRUE
3017 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
3018
3019 # Reset and reuse the new candidate and leaders lists
3020 newAllCandidates = []
3021 newCandidates = []
3022 newLeaders = []
3023 for cli in main.CLIs:
3024 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
3025 if oldLeader not in node: # election might no have finished yet
3026 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
3027 "be sure elections are complete" )
3028 time.sleep(5)
3029 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
3030 if oldLeader not in node: # election still isn't done, errors
3031 main.log.error(
3032 "Old leader was not elected on at least one node" )
3033 positionResult = main.FALSE
3034 newAllCandidates.append( node )
3035 newLeaders.append( node[ 0 ] )
3036 newCandidates = newAllCandidates[ 0 ]
3037
3038 # Check that each node has the same leader. Defines newLeader
3039 if len( set( newLeaders ) ) != 1:
3040 positionResult = main.FALSE
3041 main.log.error( "Nodes have different leaders: " +
3042 str( newLeaders ) )
3043 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07003044 else:
acsmars9475b1c2015-08-28 18:02:08 -07003045 newLeader = newLeaders[ 0 ]
3046
acsmars71adceb2015-08-31 15:09:26 -07003047 # Check that each node's candidate list is the same
3048 for candidates in newAllCandidates:
3049 if set( candidates ) != set( newCandidates ):
3050 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07003051 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07003052
acsmars9475b1c2015-08-28 18:02:08 -07003053 # Check that the re-elected node is last on the candidate List
3054 if oldLeader != newCandidates[ -1 ]:
Jon Hall6e709752016-02-01 13:38:46 -08003055 main.log.error( "Old Leader (" + str( oldLeader ) + ") not in the proper position " +
acsmars9475b1c2015-08-28 18:02:08 -07003056 str( newCandidates ) )
3057 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003058
3059 utilities.assert_equals(
3060 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07003061 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003062 onpass="Old leader successfully re-ran for election",
3063 onfail="Something went wrong with Leadership election after " +
3064 "the old leader re-ran for election" )
3065
3066 def CASE16( self, main ):
3067 """
3068 Install Distributed Primitives app
3069 """
3070 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003071 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003072 assert main, "main not defined"
3073 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003074 assert main.CLIs, "main.CLIs not defined"
3075 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003076
3077 # Variables for the distributed primitives tests
3078 global pCounterName
3079 global iCounterName
3080 global pCounterValue
3081 global iCounterValue
3082 global onosSet
3083 global onosSetName
3084 pCounterName = "TestON-Partitions"
3085 iCounterName = "TestON-inMemory"
3086 pCounterValue = 0
3087 iCounterValue = 0
3088 onosSet = set([])
3089 onosSetName = "TestON-set"
3090
3091 description = "Install Primitives app"
3092 main.case( description )
3093 main.step( "Install Primitives app" )
3094 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07003095 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003096 utilities.assert_equals( expect=main.TRUE,
3097 actual=appResults,
3098 onpass="Primitives app activated",
3099 onfail="Primitives app not activated" )
3100 time.sleep( 5 ) # To allow all nodes to activate
3101
3102 def CASE17( self, main ):
3103 """
3104 Check for basic functionality with distributed primitives
3105 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003106 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003107 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003108 assert main, "main not defined"
3109 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003110 assert main.CLIs, "main.CLIs not defined"
3111 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003112 assert pCounterName, "pCounterName not defined"
3113 assert iCounterName, "iCounterName not defined"
3114 assert onosSetName, "onosSetName not defined"
3115 # NOTE: assert fails if value is 0/None/Empty/False
3116 try:
3117 pCounterValue
3118 except NameError:
3119 main.log.error( "pCounterValue not defined, setting to 0" )
3120 pCounterValue = 0
3121 try:
3122 iCounterValue
3123 except NameError:
3124 main.log.error( "iCounterValue not defined, setting to 0" )
3125 iCounterValue = 0
3126 try:
3127 onosSet
3128 except NameError:
3129 main.log.error( "onosSet not defined, setting to empty Set" )
3130 onosSet = set([])
3131 # Variables for the distributed primitives tests. These are local only
3132 addValue = "a"
3133 addAllValue = "a b c d e f"
3134 retainValue = "c d e f"
3135
3136 description = "Check for basic functionality with distributed " +\
3137 "primitives"
3138 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003139 main.caseExplanation = "Test the methods of the distributed " +\
3140 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003141 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003142 # Partitioned counters
3143 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003144 pCounters = []
3145 threads = []
3146 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003147 for i in range( main.numCtrls ):
3148 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3149 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003150 args=[ pCounterName ] )
3151 pCounterValue += 1
3152 addedPValues.append( pCounterValue )
3153 threads.append( t )
3154 t.start()
3155
3156 for t in threads:
3157 t.join()
3158 pCounters.append( t.result )
3159 # Check that counter incremented numController times
3160 pCounterResults = True
3161 for i in addedPValues:
3162 tmpResult = i in pCounters
3163 pCounterResults = pCounterResults and tmpResult
3164 if not tmpResult:
3165 main.log.error( str( i ) + " is not in partitioned "
3166 "counter incremented results" )
3167 utilities.assert_equals( expect=True,
3168 actual=pCounterResults,
3169 onpass="Default counter incremented",
3170 onfail="Error incrementing default" +
3171 " counter" )
3172
Jon Halle1a3b752015-07-22 13:02:46 -07003173 main.step( "Get then Increment a default counter on each node" )
3174 pCounters = []
3175 threads = []
3176 addedPValues = []
3177 for i in range( main.numCtrls ):
3178 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3179 name="counterGetAndAdd-" + str( i ),
3180 args=[ pCounterName ] )
3181 addedPValues.append( pCounterValue )
3182 pCounterValue += 1
3183 threads.append( t )
3184 t.start()
3185
3186 for t in threads:
3187 t.join()
3188 pCounters.append( t.result )
3189 # Check that counter incremented numController times
3190 pCounterResults = True
3191 for i in addedPValues:
3192 tmpResult = i in pCounters
3193 pCounterResults = pCounterResults and tmpResult
3194 if not tmpResult:
3195 main.log.error( str( i ) + " is not in partitioned "
3196 "counter incremented results" )
3197 utilities.assert_equals( expect=True,
3198 actual=pCounterResults,
3199 onpass="Default counter incremented",
3200 onfail="Error incrementing default" +
3201 " counter" )
3202
3203 main.step( "Counters we added have the correct values" )
3204 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3205 utilities.assert_equals( expect=main.TRUE,
3206 actual=incrementCheck,
3207 onpass="Added counters are correct",
3208 onfail="Added counters are incorrect" )
3209
3210 main.step( "Add -8 to then get a default counter on each node" )
3211 pCounters = []
3212 threads = []
3213 addedPValues = []
3214 for i in range( main.numCtrls ):
3215 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3216 name="counterIncrement-" + str( i ),
3217 args=[ pCounterName ],
3218 kwargs={ "delta": -8 } )
3219 pCounterValue += -8
3220 addedPValues.append( pCounterValue )
3221 threads.append( t )
3222 t.start()
3223
3224 for t in threads:
3225 t.join()
3226 pCounters.append( t.result )
3227 # Check that counter incremented numController times
3228 pCounterResults = True
3229 for i in addedPValues:
3230 tmpResult = i in pCounters
3231 pCounterResults = pCounterResults and tmpResult
3232 if not tmpResult:
3233 main.log.error( str( i ) + " is not in partitioned "
3234 "counter incremented results" )
3235 utilities.assert_equals( expect=True,
3236 actual=pCounterResults,
3237 onpass="Default counter incremented",
3238 onfail="Error incrementing default" +
3239 " counter" )
3240
3241 main.step( "Add 5 to then get a default counter on each node" )
3242 pCounters = []
3243 threads = []
3244 addedPValues = []
3245 for i in range( main.numCtrls ):
3246 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3247 name="counterIncrement-" + str( i ),
3248 args=[ pCounterName ],
3249 kwargs={ "delta": 5 } )
3250 pCounterValue += 5
3251 addedPValues.append( pCounterValue )
3252 threads.append( t )
3253 t.start()
3254
3255 for t in threads:
3256 t.join()
3257 pCounters.append( t.result )
3258 # Check that counter incremented numController times
3259 pCounterResults = True
3260 for i in addedPValues:
3261 tmpResult = i in pCounters
3262 pCounterResults = pCounterResults and tmpResult
3263 if not tmpResult:
3264 main.log.error( str( i ) + " is not in partitioned "
3265 "counter incremented results" )
3266 utilities.assert_equals( expect=True,
3267 actual=pCounterResults,
3268 onpass="Default counter incremented",
3269 onfail="Error incrementing default" +
3270 " counter" )
3271
3272 main.step( "Get then add 5 to a default counter on each node" )
3273 pCounters = []
3274 threads = []
3275 addedPValues = []
3276 for i in range( main.numCtrls ):
3277 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3278 name="counterIncrement-" + str( i ),
3279 args=[ pCounterName ],
3280 kwargs={ "delta": 5 } )
3281 addedPValues.append( pCounterValue )
3282 pCounterValue += 5
3283 threads.append( t )
3284 t.start()
3285
3286 for t in threads:
3287 t.join()
3288 pCounters.append( t.result )
3289 # Check that counter incremented numController times
3290 pCounterResults = True
3291 for i in addedPValues:
3292 tmpResult = i in pCounters
3293 pCounterResults = pCounterResults and tmpResult
3294 if not tmpResult:
3295 main.log.error( str( i ) + " is not in partitioned "
3296 "counter incremented results" )
3297 utilities.assert_equals( expect=True,
3298 actual=pCounterResults,
3299 onpass="Default counter incremented",
3300 onfail="Error incrementing default" +
3301 " counter" )
3302
3303 main.step( "Counters we added have the correct values" )
3304 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3305 utilities.assert_equals( expect=main.TRUE,
3306 actual=incrementCheck,
3307 onpass="Added counters are correct",
3308 onfail="Added counters are incorrect" )
3309
3310 # In-Memory counters
3311 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003312 iCounters = []
3313 addedIValues = []
3314 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003315 for i in range( main.numCtrls ):
3316 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003317 name="icounterIncrement-" + str( i ),
3318 args=[ iCounterName ],
3319 kwargs={ "inMemory": True } )
3320 iCounterValue += 1
3321 addedIValues.append( iCounterValue )
3322 threads.append( t )
3323 t.start()
3324
3325 for t in threads:
3326 t.join()
3327 iCounters.append( t.result )
3328 # Check that counter incremented numController times
3329 iCounterResults = True
3330 for i in addedIValues:
3331 tmpResult = i in iCounters
3332 iCounterResults = iCounterResults and tmpResult
3333 if not tmpResult:
3334 main.log.error( str( i ) + " is not in the in-memory "
3335 "counter incremented results" )
3336 utilities.assert_equals( expect=True,
3337 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003338 onpass="In-memory counter incremented",
3339 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003340 " counter" )
3341
Jon Halle1a3b752015-07-22 13:02:46 -07003342 main.step( "Get then Increment a in-memory counter on each node" )
3343 iCounters = []
3344 threads = []
3345 addedIValues = []
3346 for i in range( main.numCtrls ):
3347 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3348 name="counterGetAndAdd-" + str( i ),
3349 args=[ iCounterName ],
3350 kwargs={ "inMemory": True } )
3351 addedIValues.append( iCounterValue )
3352 iCounterValue += 1
3353 threads.append( t )
3354 t.start()
3355
3356 for t in threads:
3357 t.join()
3358 iCounters.append( t.result )
3359 # Check that counter incremented numController times
3360 iCounterResults = True
3361 for i in addedIValues:
3362 tmpResult = i in iCounters
3363 iCounterResults = iCounterResults and tmpResult
3364 if not tmpResult:
3365 main.log.error( str( i ) + " is not in in-memory "
3366 "counter incremented results" )
3367 utilities.assert_equals( expect=True,
3368 actual=iCounterResults,
3369 onpass="In-memory counter incremented",
3370 onfail="Error incrementing in-memory" +
3371 " counter" )
3372
3373 main.step( "Counters we added have the correct values" )
3374 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3375 utilities.assert_equals( expect=main.TRUE,
3376 actual=incrementCheck,
3377 onpass="Added counters are correct",
3378 onfail="Added counters are incorrect" )
3379
3380 main.step( "Add -8 to then get a in-memory counter on each node" )
3381 iCounters = []
3382 threads = []
3383 addedIValues = []
3384 for i in range( main.numCtrls ):
3385 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3386 name="counterIncrement-" + str( i ),
3387 args=[ iCounterName ],
3388 kwargs={ "delta": -8, "inMemory": True } )
3389 iCounterValue += -8
3390 addedIValues.append( iCounterValue )
3391 threads.append( t )
3392 t.start()
3393
3394 for t in threads:
3395 t.join()
3396 iCounters.append( t.result )
3397 # Check that counter incremented numController times
3398 iCounterResults = True
3399 for i in addedIValues:
3400 tmpResult = i in iCounters
3401 iCounterResults = iCounterResults and tmpResult
3402 if not tmpResult:
3403 main.log.error( str( i ) + " is not in in-memory "
3404 "counter incremented results" )
3405 utilities.assert_equals( expect=True,
3406 actual=pCounterResults,
3407 onpass="In-memory counter incremented",
3408 onfail="Error incrementing in-memory" +
3409 " counter" )
3410
3411 main.step( "Add 5 to then get a in-memory counter on each node" )
3412 iCounters = []
3413 threads = []
3414 addedIValues = []
3415 for i in range( main.numCtrls ):
3416 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3417 name="counterIncrement-" + str( i ),
3418 args=[ iCounterName ],
3419 kwargs={ "delta": 5, "inMemory": True } )
3420 iCounterValue += 5
3421 addedIValues.append( iCounterValue )
3422 threads.append( t )
3423 t.start()
3424
3425 for t in threads:
3426 t.join()
3427 iCounters.append( t.result )
3428 # Check that counter incremented numController times
3429 iCounterResults = True
3430 for i in addedIValues:
3431 tmpResult = i in iCounters
3432 iCounterResults = iCounterResults and tmpResult
3433 if not tmpResult:
3434 main.log.error( str( i ) + " is not in in-memory "
3435 "counter incremented results" )
3436 utilities.assert_equals( expect=True,
3437 actual=pCounterResults,
3438 onpass="In-memory counter incremented",
3439 onfail="Error incrementing in-memory" +
3440 " counter" )
3441
3442 main.step( "Get then add 5 to a in-memory counter on each node" )
3443 iCounters = []
3444 threads = []
3445 addedIValues = []
3446 for i in range( main.numCtrls ):
3447 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3448 name="counterIncrement-" + str( i ),
3449 args=[ iCounterName ],
3450 kwargs={ "delta": 5, "inMemory": True } )
3451 addedIValues.append( iCounterValue )
3452 iCounterValue += 5
3453 threads.append( t )
3454 t.start()
3455
3456 for t in threads:
3457 t.join()
3458 iCounters.append( t.result )
3459 # Check that counter incremented numController times
3460 iCounterResults = True
3461 for i in addedIValues:
3462 tmpResult = i in iCounters
3463 iCounterResults = iCounterResults and tmpResult
3464 if not tmpResult:
3465 main.log.error( str( i ) + " is not in in-memory "
3466 "counter incremented results" )
3467 utilities.assert_equals( expect=True,
3468 actual=iCounterResults,
3469 onpass="In-memory counter incremented",
3470 onfail="Error incrementing in-memory" +
3471 " counter" )
3472
3473 main.step( "Counters we added have the correct values" )
3474 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3475 utilities.assert_equals( expect=main.TRUE,
3476 actual=incrementCheck,
3477 onpass="Added counters are correct",
3478 onfail="Added counters are incorrect" )
3479
Jon Hall5cf14d52015-07-16 12:15:19 -07003480 main.step( "Check counters are consistant across nodes" )
Jon Hall57b50432015-10-22 10:20:10 -07003481 onosCounters, consistentCounterResults = main.Counters.consistentCheck()
Jon Hall5cf14d52015-07-16 12:15:19 -07003482 utilities.assert_equals( expect=main.TRUE,
3483 actual=consistentCounterResults,
3484 onpass="ONOS counters are consistent " +
3485 "across nodes",
3486 onfail="ONOS Counters are inconsistent " +
3487 "across nodes" )
3488
3489 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003490 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3491 incrementCheck = incrementCheck and \
3492 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003493 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003494 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003495 onpass="Added counters are correct",
3496 onfail="Added counters are incorrect" )
3497 # DISTRIBUTED SETS
3498 main.step( "Distributed Set get" )
3499 size = len( onosSet )
3500 getResponses = []
3501 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003502 for i in range( main.numCtrls ):
3503 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003504 name="setTestGet-" + str( i ),
3505 args=[ onosSetName ] )
3506 threads.append( t )
3507 t.start()
3508 for t in threads:
3509 t.join()
3510 getResponses.append( t.result )
3511
3512 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003513 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003514 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003515 if isinstance( getResponses[ i ], list):
3516 current = set( getResponses[ i ] )
3517 if len( current ) == len( getResponses[ i ] ):
3518 # no repeats
3519 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003520 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003521 " has incorrect view" +
3522 " of set " + onosSetName + ":\n" +
3523 str( getResponses[ i ] ) )
3524 main.log.debug( "Expected: " + str( onosSet ) )
3525 main.log.debug( "Actual: " + str( current ) )
3526 getResults = main.FALSE
3527 else:
3528 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003529 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003530 " has repeat elements in" +
3531 " set " + onosSetName + ":\n" +
3532 str( getResponses[ i ] ) )
3533 getResults = main.FALSE
3534 elif getResponses[ i ] == main.ERROR:
3535 getResults = main.FALSE
3536 utilities.assert_equals( expect=main.TRUE,
3537 actual=getResults,
3538 onpass="Set elements are correct",
3539 onfail="Set elements are incorrect" )
3540
3541 main.step( "Distributed Set size" )
3542 sizeResponses = []
3543 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003544 for i in range( main.numCtrls ):
3545 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003546 name="setTestSize-" + str( i ),
3547 args=[ onosSetName ] )
3548 threads.append( t )
3549 t.start()
3550 for t in threads:
3551 t.join()
3552 sizeResponses.append( t.result )
3553
3554 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003555 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003556 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003557 if size != sizeResponses[ i ]:
3558 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003559 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003560 " expected a size of " + str( size ) +
3561 " for set " + onosSetName +
3562 " but got " + str( sizeResponses[ i ] ) )
3563 utilities.assert_equals( expect=main.TRUE,
3564 actual=sizeResults,
3565 onpass="Set sizes are correct",
3566 onfail="Set sizes are incorrect" )
3567
3568 main.step( "Distributed Set add()" )
3569 onosSet.add( addValue )
3570 addResponses = []
3571 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003572 for i in range( main.numCtrls ):
3573 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003574 name="setTestAdd-" + str( i ),
3575 args=[ onosSetName, addValue ] )
3576 threads.append( t )
3577 t.start()
3578 for t in threads:
3579 t.join()
3580 addResponses.append( t.result )
3581
3582 # main.TRUE = successfully changed the set
3583 # main.FALSE = action resulted in no change in set
3584 # main.ERROR - Some error in executing the function
3585 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003586 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003587 if addResponses[ i ] == main.TRUE:
3588 # All is well
3589 pass
3590 elif addResponses[ i ] == main.FALSE:
3591 # Already in set, probably fine
3592 pass
3593 elif addResponses[ i ] == main.ERROR:
3594 # Error in execution
3595 addResults = main.FALSE
3596 else:
3597 # unexpected result
3598 addResults = main.FALSE
3599 if addResults != main.TRUE:
3600 main.log.error( "Error executing set add" )
3601
3602 # Check if set is still correct
3603 size = len( onosSet )
3604 getResponses = []
3605 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003606 for i in range( main.numCtrls ):
3607 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003608 name="setTestGet-" + str( i ),
3609 args=[ onosSetName ] )
3610 threads.append( t )
3611 t.start()
3612 for t in threads:
3613 t.join()
3614 getResponses.append( t.result )
3615 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003616 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003617 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003618 if isinstance( getResponses[ i ], list):
3619 current = set( getResponses[ i ] )
3620 if len( current ) == len( getResponses[ i ] ):
3621 # no repeats
3622 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003623 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003624 " of set " + onosSetName + ":\n" +
3625 str( getResponses[ i ] ) )
3626 main.log.debug( "Expected: " + str( onosSet ) )
3627 main.log.debug( "Actual: " + str( current ) )
3628 getResults = main.FALSE
3629 else:
3630 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003631 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003632 " set " + onosSetName + ":\n" +
3633 str( getResponses[ i ] ) )
3634 getResults = main.FALSE
3635 elif getResponses[ i ] == main.ERROR:
3636 getResults = main.FALSE
3637 sizeResponses = []
3638 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003639 for i in range( main.numCtrls ):
3640 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003641 name="setTestSize-" + str( i ),
3642 args=[ onosSetName ] )
3643 threads.append( t )
3644 t.start()
3645 for t in threads:
3646 t.join()
3647 sizeResponses.append( t.result )
3648 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003649 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003650 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003651 if size != sizeResponses[ i ]:
3652 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003653 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003654 " expected a size of " + str( size ) +
3655 " for set " + onosSetName +
3656 " but got " + str( sizeResponses[ i ] ) )
3657 addResults = addResults and getResults and sizeResults
3658 utilities.assert_equals( expect=main.TRUE,
3659 actual=addResults,
3660 onpass="Set add correct",
3661 onfail="Set add was incorrect" )
3662
3663 main.step( "Distributed Set addAll()" )
3664 onosSet.update( addAllValue.split() )
3665 addResponses = []
3666 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003667 for i in range( main.numCtrls ):
3668 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003669 name="setTestAddAll-" + str( i ),
3670 args=[ onosSetName, addAllValue ] )
3671 threads.append( t )
3672 t.start()
3673 for t in threads:
3674 t.join()
3675 addResponses.append( t.result )
3676
3677 # main.TRUE = successfully changed the set
3678 # main.FALSE = action resulted in no change in set
3679 # main.ERROR - Some error in executing the function
3680 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003681 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003682 if addResponses[ i ] == main.TRUE:
3683 # All is well
3684 pass
3685 elif addResponses[ i ] == main.FALSE:
3686 # Already in set, probably fine
3687 pass
3688 elif addResponses[ i ] == main.ERROR:
3689 # Error in execution
3690 addAllResults = main.FALSE
3691 else:
3692 # unexpected result
3693 addAllResults = main.FALSE
3694 if addAllResults != main.TRUE:
3695 main.log.error( "Error executing set addAll" )
3696
3697 # Check if set is still correct
3698 size = len( onosSet )
3699 getResponses = []
3700 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003701 for i in range( main.numCtrls ):
3702 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003703 name="setTestGet-" + str( i ),
3704 args=[ onosSetName ] )
3705 threads.append( t )
3706 t.start()
3707 for t in threads:
3708 t.join()
3709 getResponses.append( t.result )
3710 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003711 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003712 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003713 if isinstance( getResponses[ i ], list):
3714 current = set( getResponses[ i ] )
3715 if len( current ) == len( getResponses[ i ] ):
3716 # no repeats
3717 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003718 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003719 " has incorrect view" +
3720 " of set " + onosSetName + ":\n" +
3721 str( getResponses[ i ] ) )
3722 main.log.debug( "Expected: " + str( onosSet ) )
3723 main.log.debug( "Actual: " + str( current ) )
3724 getResults = main.FALSE
3725 else:
3726 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003727 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003728 " has repeat elements in" +
3729 " set " + onosSetName + ":\n" +
3730 str( getResponses[ i ] ) )
3731 getResults = main.FALSE
3732 elif getResponses[ i ] == main.ERROR:
3733 getResults = main.FALSE
3734 sizeResponses = []
3735 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003736 for i in range( main.numCtrls ):
3737 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003738 name="setTestSize-" + str( i ),
3739 args=[ onosSetName ] )
3740 threads.append( t )
3741 t.start()
3742 for t in threads:
3743 t.join()
3744 sizeResponses.append( t.result )
3745 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003746 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003747 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003748 if size != sizeResponses[ i ]:
3749 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003750 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003751 " expected a size of " + str( size ) +
3752 " for set " + onosSetName +
3753 " but got " + str( sizeResponses[ i ] ) )
3754 addAllResults = addAllResults and getResults and sizeResults
3755 utilities.assert_equals( expect=main.TRUE,
3756 actual=addAllResults,
3757 onpass="Set addAll correct",
3758 onfail="Set addAll was incorrect" )
3759
3760 main.step( "Distributed Set contains()" )
3761 containsResponses = []
3762 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003763 for i in range( main.numCtrls ):
3764 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003765 name="setContains-" + str( i ),
3766 args=[ onosSetName ],
3767 kwargs={ "values": addValue } )
3768 threads.append( t )
3769 t.start()
3770 for t in threads:
3771 t.join()
3772 # NOTE: This is the tuple
3773 containsResponses.append( t.result )
3774
3775 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003776 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003777 if containsResponses[ i ] == main.ERROR:
3778 containsResults = main.FALSE
3779 else:
3780 containsResults = containsResults and\
3781 containsResponses[ i ][ 1 ]
3782 utilities.assert_equals( expect=main.TRUE,
3783 actual=containsResults,
3784 onpass="Set contains is functional",
3785 onfail="Set contains failed" )
3786
3787 main.step( "Distributed Set containsAll()" )
3788 containsAllResponses = []
3789 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003790 for i in range( main.numCtrls ):
3791 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003792 name="setContainsAll-" + str( i ),
3793 args=[ onosSetName ],
3794 kwargs={ "values": addAllValue } )
3795 threads.append( t )
3796 t.start()
3797 for t in threads:
3798 t.join()
3799 # NOTE: This is the tuple
3800 containsAllResponses.append( t.result )
3801
3802 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003803 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003804 if containsResponses[ i ] == main.ERROR:
3805 containsResults = main.FALSE
3806 else:
3807 containsResults = containsResults and\
3808 containsResponses[ i ][ 1 ]
3809 utilities.assert_equals( expect=main.TRUE,
3810 actual=containsAllResults,
3811 onpass="Set containsAll is functional",
3812 onfail="Set containsAll failed" )
3813
3814 main.step( "Distributed Set remove()" )
3815 onosSet.remove( addValue )
3816 removeResponses = []
3817 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003818 for i in range( main.numCtrls ):
3819 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003820 name="setTestRemove-" + str( i ),
3821 args=[ onosSetName, addValue ] )
3822 threads.append( t )
3823 t.start()
3824 for t in threads:
3825 t.join()
3826 removeResponses.append( t.result )
3827
3828 # main.TRUE = successfully changed the set
3829 # main.FALSE = action resulted in no change in set
3830 # main.ERROR - Some error in executing the function
3831 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003832 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003833 if removeResponses[ i ] == main.TRUE:
3834 # All is well
3835 pass
3836 elif removeResponses[ i ] == main.FALSE:
3837 # not in set, probably fine
3838 pass
3839 elif removeResponses[ i ] == main.ERROR:
3840 # Error in execution
3841 removeResults = main.FALSE
3842 else:
3843 # unexpected result
3844 removeResults = main.FALSE
3845 if removeResults != main.TRUE:
3846 main.log.error( "Error executing set remove" )
3847
3848 # Check if set is still correct
3849 size = len( onosSet )
3850 getResponses = []
3851 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003852 for i in range( main.numCtrls ):
3853 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003854 name="setTestGet-" + str( i ),
3855 args=[ onosSetName ] )
3856 threads.append( t )
3857 t.start()
3858 for t in threads:
3859 t.join()
3860 getResponses.append( t.result )
3861 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003862 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003863 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003864 if isinstance( getResponses[ i ], list):
3865 current = set( getResponses[ i ] )
3866 if len( current ) == len( getResponses[ i ] ):
3867 # no repeats
3868 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003869 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003870 " has incorrect view" +
3871 " of set " + onosSetName + ":\n" +
3872 str( getResponses[ i ] ) )
3873 main.log.debug( "Expected: " + str( onosSet ) )
3874 main.log.debug( "Actual: " + str( current ) )
3875 getResults = main.FALSE
3876 else:
3877 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003878 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003879 " has repeat elements in" +
3880 " set " + onosSetName + ":\n" +
3881 str( getResponses[ i ] ) )
3882 getResults = main.FALSE
3883 elif getResponses[ i ] == main.ERROR:
3884 getResults = main.FALSE
3885 sizeResponses = []
3886 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003887 for i in range( main.numCtrls ):
3888 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003889 name="setTestSize-" + str( i ),
3890 args=[ onosSetName ] )
3891 threads.append( t )
3892 t.start()
3893 for t in threads:
3894 t.join()
3895 sizeResponses.append( t.result )
3896 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003897 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003898 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003899 if size != sizeResponses[ i ]:
3900 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003901 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003902 " expected a size of " + str( size ) +
3903 " for set " + onosSetName +
3904 " but got " + str( sizeResponses[ i ] ) )
3905 removeResults = removeResults and getResults and sizeResults
3906 utilities.assert_equals( expect=main.TRUE,
3907 actual=removeResults,
3908 onpass="Set remove correct",
3909 onfail="Set remove was incorrect" )
3910
3911 main.step( "Distributed Set removeAll()" )
3912 onosSet.difference_update( addAllValue.split() )
3913 removeAllResponses = []
3914 threads = []
3915 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003916 for i in range( main.numCtrls ):
3917 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003918 name="setTestRemoveAll-" + str( i ),
3919 args=[ onosSetName, addAllValue ] )
3920 threads.append( t )
3921 t.start()
3922 for t in threads:
3923 t.join()
3924 removeAllResponses.append( t.result )
3925 except Exception, e:
3926 main.log.exception(e)
3927
3928 # main.TRUE = successfully changed the set
3929 # main.FALSE = action resulted in no change in set
3930 # main.ERROR - Some error in executing the function
3931 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003932 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003933 if removeAllResponses[ i ] == main.TRUE:
3934 # All is well
3935 pass
3936 elif removeAllResponses[ i ] == main.FALSE:
3937 # not in set, probably fine
3938 pass
3939 elif removeAllResponses[ i ] == main.ERROR:
3940 # Error in execution
3941 removeAllResults = main.FALSE
3942 else:
3943 # unexpected result
3944 removeAllResults = main.FALSE
3945 if removeAllResults != main.TRUE:
3946 main.log.error( "Error executing set removeAll" )
3947
3948 # Check if set is still correct
3949 size = len( onosSet )
3950 getResponses = []
3951 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003952 for i in range( main.numCtrls ):
3953 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003954 name="setTestGet-" + str( i ),
3955 args=[ onosSetName ] )
3956 threads.append( t )
3957 t.start()
3958 for t in threads:
3959 t.join()
3960 getResponses.append( t.result )
3961 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003962 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003963 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003964 if isinstance( getResponses[ i ], list):
3965 current = set( getResponses[ i ] )
3966 if len( current ) == len( getResponses[ i ] ):
3967 # no repeats
3968 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003969 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003970 " has incorrect view" +
3971 " of set " + onosSetName + ":\n" +
3972 str( getResponses[ i ] ) )
3973 main.log.debug( "Expected: " + str( onosSet ) )
3974 main.log.debug( "Actual: " + str( current ) )
3975 getResults = main.FALSE
3976 else:
3977 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003978 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003979 " has repeat elements in" +
3980 " set " + onosSetName + ":\n" +
3981 str( getResponses[ i ] ) )
3982 getResults = main.FALSE
3983 elif getResponses[ i ] == main.ERROR:
3984 getResults = main.FALSE
3985 sizeResponses = []
3986 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003987 for i in range( main.numCtrls ):
3988 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003989 name="setTestSize-" + str( i ),
3990 args=[ onosSetName ] )
3991 threads.append( t )
3992 t.start()
3993 for t in threads:
3994 t.join()
3995 sizeResponses.append( t.result )
3996 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003997 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08003998 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003999 if size != sizeResponses[ i ]:
4000 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004001 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004002 " expected a size of " + str( size ) +
4003 " for set " + onosSetName +
4004 " but got " + str( sizeResponses[ i ] ) )
4005 removeAllResults = removeAllResults and getResults and sizeResults
4006 utilities.assert_equals( expect=main.TRUE,
4007 actual=removeAllResults,
4008 onpass="Set removeAll correct",
4009 onfail="Set removeAll was incorrect" )
4010
4011 main.step( "Distributed Set addAll()" )
4012 onosSet.update( addAllValue.split() )
4013 addResponses = []
4014 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004015 for i in range( main.numCtrls ):
4016 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004017 name="setTestAddAll-" + str( i ),
4018 args=[ onosSetName, addAllValue ] )
4019 threads.append( t )
4020 t.start()
4021 for t in threads:
4022 t.join()
4023 addResponses.append( t.result )
4024
4025 # main.TRUE = successfully changed the set
4026 # main.FALSE = action resulted in no change in set
4027 # main.ERROR - Some error in executing the function
4028 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004029 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004030 if addResponses[ i ] == main.TRUE:
4031 # All is well
4032 pass
4033 elif addResponses[ i ] == main.FALSE:
4034 # Already in set, probably fine
4035 pass
4036 elif addResponses[ i ] == main.ERROR:
4037 # Error in execution
4038 addAllResults = main.FALSE
4039 else:
4040 # unexpected result
4041 addAllResults = main.FALSE
4042 if addAllResults != main.TRUE:
4043 main.log.error( "Error executing set addAll" )
4044
4045 # Check if set is still correct
4046 size = len( onosSet )
4047 getResponses = []
4048 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004049 for i in range( main.numCtrls ):
4050 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004051 name="setTestGet-" + str( i ),
4052 args=[ onosSetName ] )
4053 threads.append( t )
4054 t.start()
4055 for t in threads:
4056 t.join()
4057 getResponses.append( t.result )
4058 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004059 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08004060 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004061 if isinstance( getResponses[ i ], list):
4062 current = set( getResponses[ i ] )
4063 if len( current ) == len( getResponses[ i ] ):
4064 # no repeats
4065 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004066 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004067 " has incorrect view" +
4068 " of set " + onosSetName + ":\n" +
4069 str( getResponses[ i ] ) )
4070 main.log.debug( "Expected: " + str( onosSet ) )
4071 main.log.debug( "Actual: " + str( current ) )
4072 getResults = main.FALSE
4073 else:
4074 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004075 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004076 " has repeat elements in" +
4077 " set " + onosSetName + ":\n" +
4078 str( getResponses[ i ] ) )
4079 getResults = main.FALSE
4080 elif getResponses[ i ] == main.ERROR:
4081 getResults = main.FALSE
4082 sizeResponses = []
4083 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004084 for i in range( main.numCtrls ):
4085 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004086 name="setTestSize-" + str( i ),
4087 args=[ onosSetName ] )
4088 threads.append( t )
4089 t.start()
4090 for t in threads:
4091 t.join()
4092 sizeResponses.append( t.result )
4093 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004094 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08004095 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004096 if size != sizeResponses[ i ]:
4097 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004098 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004099 " expected a size of " + str( size ) +
4100 " for set " + onosSetName +
4101 " but got " + str( sizeResponses[ i ] ) )
4102 addAllResults = addAllResults and getResults and sizeResults
4103 utilities.assert_equals( expect=main.TRUE,
4104 actual=addAllResults,
4105 onpass="Set addAll correct",
4106 onfail="Set addAll was incorrect" )
4107
4108 main.step( "Distributed Set clear()" )
4109 onosSet.clear()
4110 clearResponses = []
4111 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004112 for i in range( main.numCtrls ):
4113 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004114 name="setTestClear-" + str( i ),
4115 args=[ onosSetName, " "], # Values doesn't matter
4116 kwargs={ "clear": True } )
4117 threads.append( t )
4118 t.start()
4119 for t in threads:
4120 t.join()
4121 clearResponses.append( t.result )
4122
4123 # main.TRUE = successfully changed the set
4124 # main.FALSE = action resulted in no change in set
4125 # main.ERROR - Some error in executing the function
4126 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004127 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004128 if clearResponses[ i ] == main.TRUE:
4129 # All is well
4130 pass
4131 elif clearResponses[ i ] == main.FALSE:
4132 # Nothing set, probably fine
4133 pass
4134 elif clearResponses[ i ] == main.ERROR:
4135 # Error in execution
4136 clearResults = main.FALSE
4137 else:
4138 # unexpected result
4139 clearResults = main.FALSE
4140 if clearResults != main.TRUE:
4141 main.log.error( "Error executing set clear" )
4142
4143 # Check if set is still correct
4144 size = len( onosSet )
4145 getResponses = []
4146 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004147 for i in range( main.numCtrls ):
4148 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004149 name="setTestGet-" + str( i ),
4150 args=[ onosSetName ] )
4151 threads.append( t )
4152 t.start()
4153 for t in threads:
4154 t.join()
4155 getResponses.append( t.result )
4156 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004157 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08004158 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004159 if isinstance( getResponses[ i ], list):
4160 current = set( getResponses[ i ] )
4161 if len( current ) == len( getResponses[ i ] ):
4162 # no repeats
4163 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004164 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004165 " has incorrect view" +
4166 " of set " + onosSetName + ":\n" +
4167 str( getResponses[ i ] ) )
4168 main.log.debug( "Expected: " + str( onosSet ) )
4169 main.log.debug( "Actual: " + str( current ) )
4170 getResults = main.FALSE
4171 else:
4172 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004173 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004174 " has repeat elements in" +
4175 " set " + onosSetName + ":\n" +
4176 str( getResponses[ i ] ) )
4177 getResults = main.FALSE
4178 elif getResponses[ i ] == main.ERROR:
4179 getResults = main.FALSE
4180 sizeResponses = []
4181 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004182 for i in range( main.numCtrls ):
4183 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004184 name="setTestSize-" + str( i ),
4185 args=[ onosSetName ] )
4186 threads.append( t )
4187 t.start()
4188 for t in threads:
4189 t.join()
4190 sizeResponses.append( t.result )
4191 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004192 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08004193 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004194 if size != sizeResponses[ i ]:
4195 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004196 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004197 " expected a size of " + str( size ) +
4198 " for set " + onosSetName +
4199 " but got " + str( sizeResponses[ i ] ) )
4200 clearResults = clearResults and getResults and sizeResults
4201 utilities.assert_equals( expect=main.TRUE,
4202 actual=clearResults,
4203 onpass="Set clear correct",
4204 onfail="Set clear was incorrect" )
4205
4206 main.step( "Distributed Set addAll()" )
4207 onosSet.update( addAllValue.split() )
4208 addResponses = []
4209 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004210 for i in range( main.numCtrls ):
4211 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004212 name="setTestAddAll-" + str( i ),
4213 args=[ onosSetName, addAllValue ] )
4214 threads.append( t )
4215 t.start()
4216 for t in threads:
4217 t.join()
4218 addResponses.append( t.result )
4219
4220 # main.TRUE = successfully changed the set
4221 # main.FALSE = action resulted in no change in set
4222 # main.ERROR - Some error in executing the function
4223 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004224 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004225 if addResponses[ i ] == main.TRUE:
4226 # All is well
4227 pass
4228 elif addResponses[ i ] == main.FALSE:
4229 # Already in set, probably fine
4230 pass
4231 elif addResponses[ i ] == main.ERROR:
4232 # Error in execution
4233 addAllResults = main.FALSE
4234 else:
4235 # unexpected result
4236 addAllResults = main.FALSE
4237 if addAllResults != main.TRUE:
4238 main.log.error( "Error executing set addAll" )
4239
4240 # Check if set is still correct
4241 size = len( onosSet )
4242 getResponses = []
4243 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004244 for i in range( main.numCtrls ):
4245 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004246 name="setTestGet-" + str( i ),
4247 args=[ onosSetName ] )
4248 threads.append( t )
4249 t.start()
4250 for t in threads:
4251 t.join()
4252 getResponses.append( t.result )
4253 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004254 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08004255 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004256 if isinstance( getResponses[ i ], list):
4257 current = set( getResponses[ i ] )
4258 if len( current ) == len( getResponses[ i ] ):
4259 # no repeats
4260 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004261 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004262 " has incorrect view" +
4263 " of set " + onosSetName + ":\n" +
4264 str( getResponses[ i ] ) )
4265 main.log.debug( "Expected: " + str( onosSet ) )
4266 main.log.debug( "Actual: " + str( current ) )
4267 getResults = main.FALSE
4268 else:
4269 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004270 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004271 " has repeat elements in" +
4272 " set " + onosSetName + ":\n" +
4273 str( getResponses[ i ] ) )
4274 getResults = main.FALSE
4275 elif getResponses[ i ] == main.ERROR:
4276 getResults = main.FALSE
4277 sizeResponses = []
4278 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004279 for i in range( main.numCtrls ):
4280 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004281 name="setTestSize-" + str( i ),
4282 args=[ onosSetName ] )
4283 threads.append( t )
4284 t.start()
4285 for t in threads:
4286 t.join()
4287 sizeResponses.append( t.result )
4288 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004289 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08004290 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004291 if size != sizeResponses[ i ]:
4292 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004293 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004294 " expected a size of " + str( size ) +
4295 " for set " + onosSetName +
4296 " but got " + str( sizeResponses[ i ] ) )
4297 addAllResults = addAllResults and getResults and sizeResults
4298 utilities.assert_equals( expect=main.TRUE,
4299 actual=addAllResults,
4300 onpass="Set addAll correct",
4301 onfail="Set addAll was incorrect" )
4302
4303 main.step( "Distributed Set retain()" )
4304 onosSet.intersection_update( retainValue.split() )
4305 retainResponses = []
4306 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004307 for i in range( main.numCtrls ):
4308 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004309 name="setTestRetain-" + str( i ),
4310 args=[ onosSetName, retainValue ],
4311 kwargs={ "retain": True } )
4312 threads.append( t )
4313 t.start()
4314 for t in threads:
4315 t.join()
4316 retainResponses.append( t.result )
4317
4318 # main.TRUE = successfully changed the set
4319 # main.FALSE = action resulted in no change in set
4320 # main.ERROR - Some error in executing the function
4321 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004322 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004323 if retainResponses[ i ] == main.TRUE:
4324 # All is well
4325 pass
4326 elif retainResponses[ i ] == main.FALSE:
4327 # Already in set, probably fine
4328 pass
4329 elif retainResponses[ i ] == main.ERROR:
4330 # Error in execution
4331 retainResults = main.FALSE
4332 else:
4333 # unexpected result
4334 retainResults = main.FALSE
4335 if retainResults != main.TRUE:
4336 main.log.error( "Error executing set retain" )
4337
4338 # Check if set is still correct
4339 size = len( onosSet )
4340 getResponses = []
4341 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004342 for i in range( main.numCtrls ):
4343 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004344 name="setTestGet-" + str( i ),
4345 args=[ onosSetName ] )
4346 threads.append( t )
4347 t.start()
4348 for t in threads:
4349 t.join()
4350 getResponses.append( t.result )
4351 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004352 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08004353 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004354 if isinstance( getResponses[ i ], list):
4355 current = set( getResponses[ i ] )
4356 if len( current ) == len( getResponses[ i ] ):
4357 # no repeats
4358 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004359 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004360 " has incorrect view" +
4361 " of set " + onosSetName + ":\n" +
4362 str( getResponses[ i ] ) )
4363 main.log.debug( "Expected: " + str( onosSet ) )
4364 main.log.debug( "Actual: " + str( current ) )
4365 getResults = main.FALSE
4366 else:
4367 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004368 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004369 " has repeat elements in" +
4370 " set " + onosSetName + ":\n" +
4371 str( getResponses[ i ] ) )
4372 getResults = main.FALSE
4373 elif getResponses[ i ] == main.ERROR:
4374 getResults = main.FALSE
4375 sizeResponses = []
4376 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004377 for i in range( main.numCtrls ):
4378 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004379 name="setTestSize-" + str( i ),
4380 args=[ onosSetName ] )
4381 threads.append( t )
4382 t.start()
4383 for t in threads:
4384 t.join()
4385 sizeResponses.append( t.result )
4386 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004387 for i in range( main.numCtrls ):
Jon Hall6e709752016-02-01 13:38:46 -08004388 node = str( i + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004389 if size != sizeResponses[ i ]:
4390 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004391 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004392 str( size ) + " for set " + onosSetName +
4393 " but got " + str( sizeResponses[ i ] ) )
4394 retainResults = retainResults and getResults and sizeResults
4395 utilities.assert_equals( expect=main.TRUE,
4396 actual=retainResults,
4397 onpass="Set retain correct",
4398 onfail="Set retain was incorrect" )
4399
Jon Hall2a5002c2015-08-21 16:49:11 -07004400 # Transactional maps
4401 main.step( "Partitioned Transactional maps put" )
4402 tMapValue = "Testing"
4403 numKeys = 100
4404 putResult = True
4405 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004406 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004407 for i in putResponses:
4408 if putResponses[ i ][ 'value' ] != tMapValue:
4409 putResult = False
4410 else:
4411 putResult = False
4412 if not putResult:
4413 main.log.debug( "Put response values: " + str( putResponses ) )
4414 utilities.assert_equals( expect=True,
4415 actual=putResult,
4416 onpass="Partitioned Transactional Map put successful",
4417 onfail="Partitioned Transactional Map put values are incorrect" )
4418
4419 main.step( "Partitioned Transactional maps get" )
4420 getCheck = True
4421 for n in range( 1, numKeys + 1 ):
4422 getResponses = []
4423 threads = []
4424 valueCheck = True
4425 for i in range( main.numCtrls ):
4426 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4427 name="TMap-get-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08004428 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004429 threads.append( t )
4430 t.start()
4431 for t in threads:
4432 t.join()
4433 getResponses.append( t.result )
4434 for node in getResponses:
4435 if node != tMapValue:
4436 valueCheck = False
4437 if not valueCheck:
4438 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4439 main.log.warn( getResponses )
4440 getCheck = getCheck and valueCheck
4441 utilities.assert_equals( expect=True,
4442 actual=getCheck,
4443 onpass="Partitioned Transactional Map get values were correct",
4444 onfail="Partitioned Transactional Map values incorrect" )
4445
4446 main.step( "In-memory Transactional maps put" )
4447 tMapValue = "Testing"
4448 numKeys = 100
4449 putResult = True
4450 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4451 if len( putResponses ) == 100:
4452 for i in putResponses:
4453 if putResponses[ i ][ 'value' ] != tMapValue:
4454 putResult = False
4455 else:
4456 putResult = False
4457 if not putResult:
4458 main.log.debug( "Put response values: " + str( putResponses ) )
4459 utilities.assert_equals( expect=True,
4460 actual=putResult,
4461 onpass="In-Memory Transactional Map put successful",
4462 onfail="In-Memory Transactional Map put values are incorrect" )
4463
4464 main.step( "In-Memory Transactional maps get" )
4465 getCheck = True
4466 for n in range( 1, numKeys + 1 ):
4467 getResponses = []
4468 threads = []
4469 valueCheck = True
4470 for i in range( main.numCtrls ):
4471 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4472 name="TMap-get-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08004473 args=[ "Key" + str( n ) ],
Jon Hall2a5002c2015-08-21 16:49:11 -07004474 kwargs={ "inMemory": True } )
4475 threads.append( t )
4476 t.start()
4477 for t in threads:
4478 t.join()
4479 getResponses.append( t.result )
4480 for node in getResponses:
4481 if node != tMapValue:
4482 valueCheck = False
4483 if not valueCheck:
4484 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4485 main.log.warn( getResponses )
4486 getCheck = getCheck and valueCheck
4487 utilities.assert_equals( expect=True,
4488 actual=getCheck,
4489 onpass="In-Memory Transactional Map get values were correct",
4490 onfail="In-Memory Transactional Map values incorrect" )