blob: b52943cdb33a1e5c969453bd6ada02b6a709a39b [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
28class HAsanity:
29
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hall5cf14d52015-07-16 12:15:19 -070052 main.log.info( "ONOS HA Sanity test - initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
57 # TODO: save all the timers and output them for plotting
58
59 # load some variables from the params file
60 PULLCODE = False
61 if main.params[ 'Git' ] == 'True':
62 PULLCODE = True
63 gitBranch = main.params[ 'branch' ]
64 cellName = main.params[ 'ENV' ][ 'cellName' ]
65
Jon Halle1a3b752015-07-22 13:02:46 -070066 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070067 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070068 if main.ONOSbench.maxNodes < main.numCtrls:
69 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070070 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070071 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79
80 # FIXME: just get controller port from params?
81 # TODO: do we really need all these?
82 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
83 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
84 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
85 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
86 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
87 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
88 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
89
Jon Halle1a3b752015-07-22 13:02:46 -070090 try:
91 fileName = "Counters"
92 path = main.params[ 'imports' ][ 'path' ]
93 main.Counters = imp.load_source( fileName,
94 path + fileName + ".py" )
95 except Exception as e:
96 main.log.exception( e )
97 main.cleanup()
98 main.exit()
99
100 main.CLIs = []
101 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700102 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700103 for i in range( 1, main.numCtrls + 1 ):
104 try:
105 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
106 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
107 ipList.append( main.nodes[ -1 ].ip_address )
108 except AttributeError:
109 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700110
111 main.step( "Create cell file" )
112 cellAppString = main.params[ 'ENV' ][ 'appString' ]
113 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
114 main.Mininet1.ip_address,
115 cellAppString, ipList )
116 main.step( "Applying cell variable to environment" )
117 cellResult = main.ONOSbench.setCell( cellName )
118 verifyResult = main.ONOSbench.verifyCell()
119
120 # FIXME:this is short term fix
121 main.log.info( "Removing raft logs" )
122 main.ONOSbench.onosRemoveRaftLogs()
123
124 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700125 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700126 main.ONOSbench.onosUninstall( node.ip_address )
127
128 # Make sure ONOS is DEAD
129 main.log.info( "Killing any ONOS processes" )
130 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700131 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700132 killed = main.ONOSbench.onosKill( node.ip_address )
133 killResults = killResults and killed
134
135 cleanInstallResult = main.TRUE
136 gitPullResult = main.TRUE
137
138 main.step( "Starting Mininet" )
139 # scp topo file to mininet
140 # TODO: move to params?
141 topoName = "obelisk.py"
142 filePath = main.ONOSbench.home + "/tools/test/topos/"
143 main.ONOSbench.copyMininetFile( topoName, filePath,
144 main.Mininet1.user_name,
145 main.Mininet1.ip_address )
146 mnResult = main.Mininet1.startNet( )
147 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
148 onpass="Mininet Started",
149 onfail="Error starting Mininet" )
150
151 main.step( "Git checkout and pull " + gitBranch )
152 if PULLCODE:
153 main.ONOSbench.gitCheckout( gitBranch )
154 gitPullResult = main.ONOSbench.gitPull()
155 # values of 1 or 3 are good
156 utilities.assert_lesser( expect=0, actual=gitPullResult,
157 onpass="Git pull successful",
158 onfail="Git pull failed" )
159 main.ONOSbench.getVersion( report=True )
160
161 main.step( "Using mvn clean install" )
162 cleanInstallResult = main.TRUE
163 if PULLCODE and gitPullResult == main.TRUE:
164 cleanInstallResult = main.ONOSbench.cleanInstall()
165 else:
166 main.log.warn( "Did not pull new code so skipping mvn " +
167 "clean install" )
168 utilities.assert_equals( expect=main.TRUE,
169 actual=cleanInstallResult,
170 onpass="MCI successful",
171 onfail="MCI failed" )
172 # GRAPHS
173 # NOTE: important params here:
174 # job = name of Jenkins job
175 # Plot Name = Plot-HA, only can be used if multiple plots
176 # index = The number of the graph under plot name
177 job = "HAsanity"
178 plotName = "Plot-HA"
179 graphs = '<ac:structured-macro ac:name="html">\n'
180 graphs += '<ac:plain-text-body><![CDATA[\n'
181 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
182 '/plot/' + plotName + '/getPlot?index=0' +\
183 '&width=500&height=300"' +\
184 'noborder="0" width="500" height="300" scrolling="yes" ' +\
185 'seamless="seamless"></iframe>\n'
186 graphs += ']]></ac:plain-text-body>\n'
187 graphs += '</ac:structured-macro>\n'
188 main.log.wiki(graphs)
189
190 main.step( "Creating ONOS package" )
191 packageResult = main.ONOSbench.onosPackage()
192 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
193 onpass="ONOS package successful",
194 onfail="ONOS package failed" )
195
196 main.step( "Installing ONOS package" )
197 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700198 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700199 tmpResult = main.ONOSbench.onosInstall( options="-f",
200 node=node.ip_address )
201 onosInstallResult = onosInstallResult and tmpResult
202 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
203 onpass="ONOS install successful",
204 onfail="ONOS install failed" )
205
206 main.step( "Checking if ONOS is up yet" )
207 for i in range( 2 ):
208 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700209 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700210 started = main.ONOSbench.isup( node.ip_address )
211 if not started:
212 main.log.error( node.name + " didn't start!" )
213 main.ONOSbench.onosStop( node.ip_address )
214 main.ONOSbench.onosStart( node.ip_address )
215 onosIsupResult = onosIsupResult and started
216 if onosIsupResult == main.TRUE:
217 break
218 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
219 onpass="ONOS startup successful",
220 onfail="ONOS startup failed" )
221
222 main.log.step( "Starting ONOS CLI sessions" )
223 cliResults = main.TRUE
224 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700225 for i in range( main.numCtrls ):
226 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700227 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700228 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700229 threads.append( t )
230 t.start()
231
232 for t in threads:
233 t.join()
234 cliResults = cliResults and t.result
235 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
236 onpass="ONOS cli startup successful",
237 onfail="ONOS cli startup failed" )
238
239 if main.params[ 'tcpdump' ].lower() == "true":
240 main.step( "Start Packet Capture MN" )
241 main.Mininet2.startTcpdump(
242 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
243 + "-MN.pcap",
244 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
245 port=main.params[ 'MNtcpdump' ][ 'port' ] )
246
247 main.step( "App Ids check" )
248 appCheck = main.TRUE
249 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700250 for i in range( main.numCtrls ):
251 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700252 name="appToIDCheck-" + str( i ),
253 args=[] )
254 threads.append( t )
255 t.start()
256
257 for t in threads:
258 t.join()
259 appCheck = appCheck and t.result
260 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700261 main.log.warn( main.CLIs[0].apps() )
262 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700263 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
264 onpass="App Ids seem to be correct",
265 onfail="Something is wrong with app Ids" )
266
267 if cliResults == main.FALSE:
268 main.log.error( "Failed to start ONOS, stopping test" )
269 main.cleanup()
270 main.exit()
271
272 def CASE2( self, main ):
273 """
274 Assign devices to controllers
275 """
276 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700277 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700278 assert main, "main not defined"
279 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700280 assert main.CLIs, "main.CLIs not defined"
281 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700282 assert ONOS1Port, "ONOS1Port not defined"
283 assert ONOS2Port, "ONOS2Port not defined"
284 assert ONOS3Port, "ONOS3Port not defined"
285 assert ONOS4Port, "ONOS4Port not defined"
286 assert ONOS5Port, "ONOS5Port not defined"
287 assert ONOS6Port, "ONOS6Port not defined"
288 assert ONOS7Port, "ONOS7Port not defined"
289
290 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700291 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700292 "and check that an ONOS node becomes the " +\
293 "master of the device."
294 main.step( "Assign switches to controllers" )
295
296 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700297 for i in range( main.numCtrls ):
298 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700299 swList = []
300 for i in range( 1, 29 ):
301 swList.append( "s" + str( i ) )
302 main.Mininet1.assignSwController( sw=swList, ip=ipList )
303
304 mastershipCheck = main.TRUE
305 for i in range( 1, 29 ):
306 response = main.Mininet1.getSwController( "s" + str( i ) )
307 try:
308 main.log.info( str( response ) )
309 except Exception:
310 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700311 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700312 if re.search( "tcp:" + node.ip_address, response ):
313 mastershipCheck = mastershipCheck and main.TRUE
314 else:
315 main.log.error( "Error, node " + node.ip_address + " is " +
316 "not in the list of controllers s" +
317 str( i ) + " is connecting to." )
318 mastershipCheck = main.FALSE
319 utilities.assert_equals(
320 expect=main.TRUE,
321 actual=mastershipCheck,
322 onpass="Switch mastership assigned correctly",
323 onfail="Switches not assigned correctly to controllers" )
324
325 def CASE21( self, main ):
326 """
327 Assign mastership to controllers
328 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700329 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700330 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700331 assert main, "main not defined"
332 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700333 assert main.CLIs, "main.CLIs not defined"
334 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700335 assert ONOS1Port, "ONOS1Port not defined"
336 assert ONOS2Port, "ONOS2Port not defined"
337 assert ONOS3Port, "ONOS3Port not defined"
338 assert ONOS4Port, "ONOS4Port not defined"
339 assert ONOS5Port, "ONOS5Port not defined"
340 assert ONOS6Port, "ONOS6Port not defined"
341 assert ONOS7Port, "ONOS7Port not defined"
342
343 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700344 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700345 "device. Then manually assign" +\
346 " mastership to specific ONOS nodes using" +\
347 " 'device-role'"
348 main.step( "Assign mastership of switches to specific controllers" )
349 # Manually assign mastership to the controller we want
350 roleCall = main.TRUE
351
352 ipList = [ ]
353 deviceList = []
354 try:
355 # Assign mastership to specific controllers. This assignment was
356 # determined for a 7 node cluser, but will work with any sized
357 # cluster
358 for i in range( 1, 29 ): # switches 1 through 28
359 # set up correct variables:
360 if i == 1:
361 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700362 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700363 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
364 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700365 c = 1 % main.numCtrls
366 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700367 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
368 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700369 c = 1 % main.numCtrls
370 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700371 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
372 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700373 c = 3 % main.numCtrls
374 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700375 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
376 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700377 c = 2 % main.numCtrls
378 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700379 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
380 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700381 c = 2 % main.numCtrls
382 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700383 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
384 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700385 c = 5 % main.numCtrls
386 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700387 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
388 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700389 c = 4 % main.numCtrls
390 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700391 dpid = '3' + str( i ).zfill( 3 )
392 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
393 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700394 c = 6 % main.numCtrls
395 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700396 dpid = '6' + str( i ).zfill( 3 )
397 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
398 elif i == 28:
399 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700400 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700401 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
402 else:
403 main.log.error( "You didn't write an else statement for " +
404 "switch s" + str( i ) )
405 roleCall = main.FALSE
406 # Assign switch
407 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
408 # TODO: make this controller dynamic
409 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
410 ip )
411 ipList.append( ip )
412 deviceList.append( deviceId )
413 except ( AttributeError, AssertionError ):
414 main.log.exception( "Something is wrong with ONOS device view" )
415 main.log.info( main.ONOScli1.devices() )
416 utilities.assert_equals(
417 expect=main.TRUE,
418 actual=roleCall,
419 onpass="Re-assigned switch mastership to designated controller",
420 onfail="Something wrong with deviceRole calls" )
421
422 main.step( "Check mastership was correctly assigned" )
423 roleCheck = main.TRUE
424 # NOTE: This is due to the fact that device mastership change is not
425 # atomic and is actually a multi step process
426 time.sleep( 5 )
427 for i in range( len( ipList ) ):
428 ip = ipList[i]
429 deviceId = deviceList[i]
430 # Check assignment
431 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
432 if ip in master:
433 roleCheck = roleCheck and main.TRUE
434 else:
435 roleCheck = roleCheck and main.FALSE
436 main.log.error( "Error, controller " + ip + " is not" +
437 " master " + "of device " +
438 str( deviceId ) + ". Master is " +
439 repr( master ) + "." )
440 utilities.assert_equals(
441 expect=main.TRUE,
442 actual=roleCheck,
443 onpass="Switches were successfully reassigned to designated " +
444 "controller",
445 onfail="Switches were not successfully reassigned" )
446
447 def CASE3( self, main ):
448 """
449 Assign intents
450 """
451 import time
452 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700453 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700454 assert main, "main not defined"
455 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700456 assert main.CLIs, "main.CLIs not defined"
457 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700458 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700459 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700460 "assign predetermined host-to-host intents." +\
461 " After installation, check that the intent" +\
462 " is distributed to all nodes and the state" +\
463 " is INSTALLED"
464
465 # install onos-app-fwd
466 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700467 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700468 utilities.assert_equals( expect=main.TRUE, actual=installResults,
469 onpass="Install fwd successful",
470 onfail="Install fwd failed" )
471
472 main.step( "Check app ids" )
473 appCheck = main.TRUE
474 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700475 for i in range( main.numCtrls ):
476 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700477 name="appToIDCheck-" + str( i ),
478 args=[] )
479 threads.append( t )
480 t.start()
481
482 for t in threads:
483 t.join()
484 appCheck = appCheck and t.result
485 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700486 main.log.warn( main.CLIs[0].apps() )
487 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700488 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
489 onpass="App Ids seem to be correct",
490 onfail="Something is wrong with app Ids" )
491
492 main.step( "Discovering Hosts( Via pingall for now )" )
493 # FIXME: Once we have a host discovery mechanism, use that instead
494 # REACTIVE FWD test
495 pingResult = main.FALSE
496 for i in range(2): # Retry if pingall fails first time
497 time1 = time.time()
498 pingResult = main.Mininet1.pingall()
499 if i == 0:
500 utilities.assert_equals(
501 expect=main.TRUE,
502 actual=pingResult,
503 onpass="Reactive Pingall test passed",
504 onfail="Reactive Pingall failed, " +
505 "one or more ping pairs failed" )
506 time2 = time.time()
507 main.log.info( "Time for pingall: %2f seconds" %
508 ( time2 - time1 ) )
509 # timeout for fwd flows
510 time.sleep( 11 )
511 # uninstall onos-app-fwd
512 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700513 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700514 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
515 onpass="Uninstall fwd successful",
516 onfail="Uninstall fwd failed" )
517 '''
518 main.Mininet1.handle.sendline( "py [ h.cmd( \"arping -c 1 10.1.1.1 \" ) for h in net.hosts ] ")
519 import time
520 time.sleep(60)
521 '''
522
523 main.step( "Check app ids" )
524 threads = []
525 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700526 for i in range( main.numCtrls ):
527 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700528 name="appToIDCheck-" + str( i ),
529 args=[] )
530 threads.append( t )
531 t.start()
532
533 for t in threads:
534 t.join()
535 appCheck2 = appCheck2 and t.result
536 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700537 main.log.warn( main.CLIs[0].apps() )
538 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700539 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
540 onpass="App Ids seem to be correct",
541 onfail="Something is wrong with app Ids" )
542
543 main.step( "Add host intents via cli" )
544 intentIds = []
545 # TODO: move the host numbers to params
546 # Maybe look at all the paths we ping?
547 intentAddResult = True
548 hostResult = main.TRUE
549 for i in range( 8, 18 ):
550 main.log.info( "Adding host intent between h" + str( i ) +
551 " and h" + str( i + 10 ) )
552 host1 = "00:00:00:00:00:" + \
553 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
554 host2 = "00:00:00:00:00:" + \
555 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
556 # NOTE: getHost can return None
557 host1Dict = main.ONOScli1.getHost( host1 )
558 host2Dict = main.ONOScli1.getHost( host2 )
559 host1Id = None
560 host2Id = None
561 if host1Dict and host2Dict:
562 host1Id = host1Dict.get( 'id', None )
563 host2Id = host2Dict.get( 'id', None )
564 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700565 nodeNum = ( i % main.numCtrls )
566 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700567 if tmpId:
568 main.log.info( "Added intent with id: " + tmpId )
569 intentIds.append( tmpId )
570 else:
571 main.log.error( "addHostIntent returned: " +
572 repr( tmpId ) )
573 else:
574 main.log.error( "Error, getHost() failed for h" + str( i ) +
575 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700576 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700577 main.log.warn( "Hosts output: " )
578 try:
579 main.log.warn( json.dumps( json.loads( hosts ),
580 sort_keys=True,
581 indent=4,
582 separators=( ',', ': ' ) ) )
583 except ( ValueError, TypeError ):
584 main.log.warn( repr( hosts ) )
585 hostResult = main.FALSE
586 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
587 onpass="Found a host id for each host",
588 onfail="Error looking up host ids" )
589
590 intentStart = time.time()
591 onosIds = main.ONOScli1.getAllIntentsId()
592 main.log.info( "Submitted intents: " + str( intentIds ) )
593 main.log.info( "Intents in ONOS: " + str( onosIds ) )
594 for intent in intentIds:
595 if intent in onosIds:
596 pass # intent submitted is in onos
597 else:
598 intentAddResult = False
599 if intentAddResult:
600 intentStop = time.time()
601 else:
602 intentStop = None
603 # Print the intent states
604 intents = main.ONOScli1.intents()
605 intentStates = []
606 installedCheck = True
607 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
608 count = 0
609 try:
610 for intent in json.loads( intents ):
611 state = intent.get( 'state', None )
612 if "INSTALLED" not in state:
613 installedCheck = False
614 intentId = intent.get( 'id', None )
615 intentStates.append( ( intentId, state ) )
616 except ( ValueError, TypeError ):
617 main.log.exception( "Error parsing intents" )
618 # add submitted intents not in the store
619 tmplist = [ i for i, s in intentStates ]
620 missingIntents = False
621 for i in intentIds:
622 if i not in tmplist:
623 intentStates.append( ( i, " - " ) )
624 missingIntents = True
625 intentStates.sort()
626 for i, s in intentStates:
627 count += 1
628 main.log.info( "%-6s%-15s%-15s" %
629 ( str( count ), str( i ), str( s ) ) )
630 leaders = main.ONOScli1.leaders()
631 try:
632 missing = False
633 if leaders:
634 parsedLeaders = json.loads( leaders )
635 main.log.warn( json.dumps( parsedLeaders,
636 sort_keys=True,
637 indent=4,
638 separators=( ',', ': ' ) ) )
639 # check for all intent partitions
640 topics = []
641 for i in range( 14 ):
642 topics.append( "intent-partition-" + str( i ) )
643 main.log.debug( topics )
644 ONOStopics = [ j['topic'] for j in parsedLeaders ]
645 for topic in topics:
646 if topic not in ONOStopics:
647 main.log.error( "Error: " + topic +
648 " not in leaders" )
649 missing = True
650 else:
651 main.log.error( "leaders() returned None" )
652 except ( ValueError, TypeError ):
653 main.log.exception( "Error parsing leaders" )
654 main.log.error( repr( leaders ) )
655 # Check all nodes
656 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700657 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700658 response = node.leaders( jsonFormat=False)
659 main.log.warn( str( node.name ) + " leaders output: \n" +
660 str( response ) )
661
662 partitions = main.ONOScli1.partitions()
663 try:
664 if partitions :
665 parsedPartitions = json.loads( partitions )
666 main.log.warn( json.dumps( parsedPartitions,
667 sort_keys=True,
668 indent=4,
669 separators=( ',', ': ' ) ) )
670 # TODO check for a leader in all paritions
671 # TODO check for consistency among nodes
672 else:
673 main.log.error( "partitions() returned None" )
674 except ( ValueError, TypeError ):
675 main.log.exception( "Error parsing partitions" )
676 main.log.error( repr( partitions ) )
677 pendingMap = main.ONOScli1.pendingMap()
678 try:
679 if pendingMap :
680 parsedPending = json.loads( pendingMap )
681 main.log.warn( json.dumps( parsedPending,
682 sort_keys=True,
683 indent=4,
684 separators=( ',', ': ' ) ) )
685 # TODO check something here?
686 else:
687 main.log.error( "pendingMap() returned None" )
688 except ( ValueError, TypeError ):
689 main.log.exception( "Error parsing pending map" )
690 main.log.error( repr( pendingMap ) )
691
692 intentAddResult = bool( intentAddResult and not missingIntents and
693 installedCheck )
694 if not intentAddResult:
695 main.log.error( "Error in pushing host intents to ONOS" )
696
697 main.step( "Intent Anti-Entropy dispersion" )
698 for i in range(100):
699 correct = True
700 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700701 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700702 onosIds = []
703 ids = cli.getAllIntentsId()
704 onosIds.append( ids )
705 main.log.debug( "Intents in " + cli.name + ": " +
706 str( sorted( onosIds ) ) )
707 if sorted( ids ) != sorted( intentIds ):
708 main.log.warn( "Set of intent IDs doesn't match" )
709 correct = False
710 break
711 else:
712 intents = json.loads( cli.intents() )
713 for intent in intents:
714 if intent[ 'state' ] != "INSTALLED":
715 main.log.warn( "Intent " + intent[ 'id' ] +
716 " is " + intent[ 'state' ] )
717 correct = False
718 break
719 if correct:
720 break
721 else:
722 time.sleep(1)
723 if not intentStop:
724 intentStop = time.time()
725 global gossipTime
726 gossipTime = intentStop - intentStart
727 main.log.info( "It took about " + str( gossipTime ) +
728 " seconds for all intents to appear in each node" )
729 # FIXME: make this time configurable/calculate based off of number of
730 # nodes and gossip rounds
731 utilities.assert_greater_equals(
732 expect=40, actual=gossipTime,
733 onpass="ECM anti-entropy for intents worked within " +
734 "expected time",
735 onfail="Intent ECM anti-entropy took too long" )
736 if gossipTime <= 40:
737 intentAddResult = True
738
739 if not intentAddResult or "key" in pendingMap:
740 import time
741 installedCheck = True
742 main.log.info( "Sleeping 60 seconds to see if intents are found" )
743 time.sleep( 60 )
744 onosIds = main.ONOScli1.getAllIntentsId()
745 main.log.info( "Submitted intents: " + str( intentIds ) )
746 main.log.info( "Intents in ONOS: " + str( onosIds ) )
747 # Print the intent states
748 intents = main.ONOScli1.intents()
749 intentStates = []
750 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
751 count = 0
752 try:
753 for intent in json.loads( intents ):
754 # Iter through intents of a node
755 state = intent.get( 'state', None )
756 if "INSTALLED" not in state:
757 installedCheck = False
758 intentId = intent.get( 'id', None )
759 intentStates.append( ( intentId, state ) )
760 except ( ValueError, TypeError ):
761 main.log.exception( "Error parsing intents" )
762 # add submitted intents not in the store
763 tmplist = [ i for i, s in intentStates ]
764 for i in intentIds:
765 if i not in tmplist:
766 intentStates.append( ( i, " - " ) )
767 intentStates.sort()
768 for i, s in intentStates:
769 count += 1
770 main.log.info( "%-6s%-15s%-15s" %
771 ( str( count ), str( i ), str( s ) ) )
772 leaders = main.ONOScli1.leaders()
773 try:
774 missing = False
775 if leaders:
776 parsedLeaders = json.loads( leaders )
777 main.log.warn( json.dumps( parsedLeaders,
778 sort_keys=True,
779 indent=4,
780 separators=( ',', ': ' ) ) )
781 # check for all intent partitions
782 # check for election
783 topics = []
784 for i in range( 14 ):
785 topics.append( "intent-partition-" + str( i ) )
786 # FIXME: this should only be after we start the app
787 topics.append( "org.onosproject.election" )
788 main.log.debug( topics )
789 ONOStopics = [ j['topic'] for j in parsedLeaders ]
790 for topic in topics:
791 if topic not in ONOStopics:
792 main.log.error( "Error: " + topic +
793 " not in leaders" )
794 missing = True
795 else:
796 main.log.error( "leaders() returned None" )
797 except ( ValueError, TypeError ):
798 main.log.exception( "Error parsing leaders" )
799 main.log.error( repr( leaders ) )
800 # Check all nodes
801 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700802 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700803 response = node.leaders( jsonFormat=False)
804 main.log.warn( str( node.name ) + " leaders output: \n" +
805 str( response ) )
806
807 partitions = main.ONOScli1.partitions()
808 try:
809 if partitions :
810 parsedPartitions = json.loads( partitions )
811 main.log.warn( json.dumps( parsedPartitions,
812 sort_keys=True,
813 indent=4,
814 separators=( ',', ': ' ) ) )
815 # TODO check for a leader in all paritions
816 # TODO check for consistency among nodes
817 else:
818 main.log.error( "partitions() returned None" )
819 except ( ValueError, TypeError ):
820 main.log.exception( "Error parsing partitions" )
821 main.log.error( repr( partitions ) )
822 pendingMap = main.ONOScli1.pendingMap()
823 try:
824 if pendingMap :
825 parsedPending = json.loads( pendingMap )
826 main.log.warn( json.dumps( parsedPending,
827 sort_keys=True,
828 indent=4,
829 separators=( ',', ': ' ) ) )
830 # TODO check something here?
831 else:
832 main.log.error( "pendingMap() returned None" )
833 except ( ValueError, TypeError ):
834 main.log.exception( "Error parsing pending map" )
835 main.log.error( repr( pendingMap ) )
836
837 def CASE4( self, main ):
838 """
839 Ping across added host intents
840 """
841 import json
842 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700843 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700844 assert main, "main not defined"
845 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700846 assert main.CLIs, "main.CLIs not defined"
847 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700848 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700849 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700850 "functionality and check the state of " +\
851 "the intent"
852 main.step( "Ping across added host intents" )
853 PingResult = main.TRUE
854 for i in range( 8, 18 ):
855 ping = main.Mininet1.pingHost( src="h" + str( i ),
856 target="h" + str( i + 10 ) )
857 PingResult = PingResult and ping
858 if ping == main.FALSE:
859 main.log.warn( "Ping failed between h" + str( i ) +
860 " and h" + str( i + 10 ) )
861 elif ping == main.TRUE:
862 main.log.info( "Ping test passed!" )
863 # Don't set PingResult or you'd override failures
864 if PingResult == main.FALSE:
865 main.log.error(
866 "Intents have not been installed correctly, pings failed." )
867 # TODO: pretty print
868 main.log.warn( "ONOS1 intents: " )
869 try:
870 tmpIntents = main.ONOScli1.intents()
871 main.log.warn( json.dumps( json.loads( tmpIntents ),
872 sort_keys=True,
873 indent=4,
874 separators=( ',', ': ' ) ) )
875 except ( ValueError, TypeError ):
876 main.log.warn( repr( tmpIntents ) )
877 utilities.assert_equals(
878 expect=main.TRUE,
879 actual=PingResult,
880 onpass="Intents have been installed correctly and pings work",
881 onfail="Intents have not been installed correctly, pings failed." )
882
883 main.step( "Check Intent state" )
884 installedCheck = False
885 loopCount = 0
886 while not installedCheck and loopCount < 40:
887 installedCheck = True
888 # Print the intent states
889 intents = main.ONOScli1.intents()
890 intentStates = []
891 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
892 count = 0
893 # Iter through intents of a node
894 try:
895 for intent in json.loads( intents ):
896 state = intent.get( 'state', None )
897 if "INSTALLED" not in state:
898 installedCheck = False
899 intentId = intent.get( 'id', None )
900 intentStates.append( ( intentId, state ) )
901 except ( ValueError, TypeError ):
902 main.log.exception( "Error parsing intents." )
903 # Print states
904 intentStates.sort()
905 for i, s in intentStates:
906 count += 1
907 main.log.info( "%-6s%-15s%-15s" %
908 ( str( count ), str( i ), str( s ) ) )
909 if not installedCheck:
910 time.sleep( 1 )
911 loopCount += 1
912 utilities.assert_equals( expect=True, actual=installedCheck,
913 onpass="Intents are all INSTALLED",
914 onfail="Intents are not all in " +
915 "INSTALLED state" )
916
917 main.step( "Check leadership of topics" )
918 leaders = main.ONOScli1.leaders()
919 topicCheck = main.TRUE
920 try:
921 if leaders:
922 parsedLeaders = json.loads( leaders )
923 main.log.warn( json.dumps( parsedLeaders,
924 sort_keys=True,
925 indent=4,
926 separators=( ',', ': ' ) ) )
927 # check for all intent partitions
928 # check for election
929 # TODO: Look at Devices as topics now that it uses this system
930 topics = []
931 for i in range( 14 ):
932 topics.append( "intent-partition-" + str( i ) )
933 # FIXME: this should only be after we start the app
934 # FIXME: topics.append( "org.onosproject.election" )
935 # Print leaders output
936 main.log.debug( topics )
937 ONOStopics = [ j['topic'] for j in parsedLeaders ]
938 for topic in topics:
939 if topic not in ONOStopics:
940 main.log.error( "Error: " + topic +
941 " not in leaders" )
942 topicCheck = main.FALSE
943 else:
944 main.log.error( "leaders() returned None" )
945 topicCheck = main.FALSE
946 except ( ValueError, TypeError ):
947 topicCheck = main.FALSE
948 main.log.exception( "Error parsing leaders" )
949 main.log.error( repr( leaders ) )
950 # TODO: Check for a leader of these topics
951 # Check all nodes
952 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700953 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700954 response = node.leaders( jsonFormat=False)
955 main.log.warn( str( node.name ) + " leaders output: \n" +
956 str( response ) )
957
958 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
959 onpass="intent Partitions is in leaders",
960 onfail="Some topics were lost " )
961 # Print partitions
962 partitions = main.ONOScli1.partitions()
963 try:
964 if partitions :
965 parsedPartitions = json.loads( partitions )
966 main.log.warn( json.dumps( parsedPartitions,
967 sort_keys=True,
968 indent=4,
969 separators=( ',', ': ' ) ) )
970 # TODO check for a leader in all paritions
971 # TODO check for consistency among nodes
972 else:
973 main.log.error( "partitions() returned None" )
974 except ( ValueError, TypeError ):
975 main.log.exception( "Error parsing partitions" )
976 main.log.error( repr( partitions ) )
977 # Print Pending Map
978 pendingMap = main.ONOScli1.pendingMap()
979 try:
980 if pendingMap :
981 parsedPending = json.loads( pendingMap )
982 main.log.warn( json.dumps( parsedPending,
983 sort_keys=True,
984 indent=4,
985 separators=( ',', ': ' ) ) )
986 # TODO check something here?
987 else:
988 main.log.error( "pendingMap() returned None" )
989 except ( ValueError, TypeError ):
990 main.log.exception( "Error parsing pending map" )
991 main.log.error( repr( pendingMap ) )
992
993 if not installedCheck:
994 main.log.info( "Waiting 60 seconds to see if the state of " +
995 "intents change" )
996 time.sleep( 60 )
997 # Print the intent states
998 intents = main.ONOScli1.intents()
999 intentStates = []
1000 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1001 count = 0
1002 # Iter through intents of a node
1003 try:
1004 for intent in json.loads( intents ):
1005 state = intent.get( 'state', None )
1006 if "INSTALLED" not in state:
1007 installedCheck = False
1008 intentId = intent.get( 'id', None )
1009 intentStates.append( ( intentId, state ) )
1010 except ( ValueError, TypeError ):
1011 main.log.exception( "Error parsing intents." )
1012 intentStates.sort()
1013 for i, s in intentStates:
1014 count += 1
1015 main.log.info( "%-6s%-15s%-15s" %
1016 ( str( count ), str( i ), str( s ) ) )
1017 leaders = main.ONOScli1.leaders()
1018 try:
1019 missing = False
1020 if leaders:
1021 parsedLeaders = json.loads( leaders )
1022 main.log.warn( json.dumps( parsedLeaders,
1023 sort_keys=True,
1024 indent=4,
1025 separators=( ',', ': ' ) ) )
1026 # check for all intent partitions
1027 # check for election
1028 topics = []
1029 for i in range( 14 ):
1030 topics.append( "intent-partition-" + str( i ) )
1031 # FIXME: this should only be after we start the app
1032 topics.append( "org.onosproject.election" )
1033 main.log.debug( topics )
1034 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1035 for topic in topics:
1036 if topic not in ONOStopics:
1037 main.log.error( "Error: " + topic +
1038 " not in leaders" )
1039 missing = True
1040 else:
1041 main.log.error( "leaders() returned None" )
1042 except ( ValueError, TypeError ):
1043 main.log.exception( "Error parsing leaders" )
1044 main.log.error( repr( leaders ) )
1045 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001046 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001047 response = node.leaders( jsonFormat=False)
1048 main.log.warn( str( node.name ) + " leaders output: \n" +
1049 str( response ) )
1050
1051 partitions = main.ONOScli1.partitions()
1052 try:
1053 if partitions :
1054 parsedPartitions = json.loads( partitions )
1055 main.log.warn( json.dumps( parsedPartitions,
1056 sort_keys=True,
1057 indent=4,
1058 separators=( ',', ': ' ) ) )
1059 # TODO check for a leader in all paritions
1060 # TODO check for consistency among nodes
1061 else:
1062 main.log.error( "partitions() returned None" )
1063 except ( ValueError, TypeError ):
1064 main.log.exception( "Error parsing partitions" )
1065 main.log.error( repr( partitions ) )
1066 pendingMap = main.ONOScli1.pendingMap()
1067 try:
1068 if pendingMap :
1069 parsedPending = json.loads( pendingMap )
1070 main.log.warn( json.dumps( parsedPending,
1071 sort_keys=True,
1072 indent=4,
1073 separators=( ',', ': ' ) ) )
1074 # TODO check something here?
1075 else:
1076 main.log.error( "pendingMap() returned None" )
1077 except ( ValueError, TypeError ):
1078 main.log.exception( "Error parsing pending map" )
1079 main.log.error( repr( pendingMap ) )
1080 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001081 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001082 main.step( "Wait a minute then ping again" )
1083 # the wait is above
1084 PingResult = main.TRUE
1085 for i in range( 8, 18 ):
1086 ping = main.Mininet1.pingHost( src="h" + str( i ),
1087 target="h" + str( i + 10 ) )
1088 PingResult = PingResult and ping
1089 if ping == main.FALSE:
1090 main.log.warn( "Ping failed between h" + str( i ) +
1091 " and h" + str( i + 10 ) )
1092 elif ping == main.TRUE:
1093 main.log.info( "Ping test passed!" )
1094 # Don't set PingResult or you'd override failures
1095 if PingResult == main.FALSE:
1096 main.log.error(
1097 "Intents have not been installed correctly, pings failed." )
1098 # TODO: pretty print
1099 main.log.warn( "ONOS1 intents: " )
1100 try:
1101 tmpIntents = main.ONOScli1.intents()
1102 main.log.warn( json.dumps( json.loads( tmpIntents ),
1103 sort_keys=True,
1104 indent=4,
1105 separators=( ',', ': ' ) ) )
1106 except ( ValueError, TypeError ):
1107 main.log.warn( repr( tmpIntents ) )
1108 utilities.assert_equals(
1109 expect=main.TRUE,
1110 actual=PingResult,
1111 onpass="Intents have been installed correctly and pings work",
1112 onfail="Intents have not been installed correctly, pings failed." )
1113
1114 def CASE5( self, main ):
1115 """
1116 Reading state of ONOS
1117 """
1118 import json
1119 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001120 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001121 assert main, "main not defined"
1122 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001123 assert main.CLIs, "main.CLIs not defined"
1124 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001125
1126 main.case( "Setting up and gathering data for current state" )
1127 # The general idea for this test case is to pull the state of
1128 # ( intents,flows, topology,... ) from each ONOS node
1129 # We can then compare them with each other and also with past states
1130
1131 main.step( "Check that each switch has a master" )
1132 global mastershipState
1133 mastershipState = '[]'
1134
1135 # Assert that each device has a master
1136 rolesNotNull = main.TRUE
1137 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001138 for i in range( main.numCtrls ):
1139 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001140 name="rolesNotNull-" + str( i ),
1141 args=[] )
1142 threads.append( t )
1143 t.start()
1144
1145 for t in threads:
1146 t.join()
1147 rolesNotNull = rolesNotNull and t.result
1148 utilities.assert_equals(
1149 expect=main.TRUE,
1150 actual=rolesNotNull,
1151 onpass="Each device has a master",
1152 onfail="Some devices don't have a master assigned" )
1153
1154 main.step( "Get the Mastership of each switch from each controller" )
1155 ONOSMastership = []
1156 mastershipCheck = main.FALSE
1157 consistentMastership = True
1158 rolesResults = True
1159 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001160 for i in range( main.numCtrls ):
1161 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001162 name="roles-" + str( i ),
1163 args=[] )
1164 threads.append( t )
1165 t.start()
1166
1167 for t in threads:
1168 t.join()
1169 ONOSMastership.append( t.result )
1170
Jon Halle1a3b752015-07-22 13:02:46 -07001171 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001172 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1173 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1174 " roles" )
1175 main.log.warn(
1176 "ONOS" + str( i + 1 ) + " mastership response: " +
1177 repr( ONOSMastership[i] ) )
1178 rolesResults = False
1179 utilities.assert_equals(
1180 expect=True,
1181 actual=rolesResults,
1182 onpass="No error in reading roles output",
1183 onfail="Error in reading roles from ONOS" )
1184
1185 main.step( "Check for consistency in roles from each controller" )
1186 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1187 main.log.info(
1188 "Switch roles are consistent across all ONOS nodes" )
1189 else:
1190 consistentMastership = False
1191 utilities.assert_equals(
1192 expect=True,
1193 actual=consistentMastership,
1194 onpass="Switch roles are consistent across all ONOS nodes",
1195 onfail="ONOS nodes have different views of switch roles" )
1196
1197 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001198 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001199 try:
1200 main.log.warn(
1201 "ONOS" + str( i + 1 ) + " roles: ",
1202 json.dumps(
1203 json.loads( ONOSMastership[ i ] ),
1204 sort_keys=True,
1205 indent=4,
1206 separators=( ',', ': ' ) ) )
1207 except ( ValueError, TypeError ):
1208 main.log.warn( repr( ONOSMastership[ i ] ) )
1209 elif rolesResults and consistentMastership:
1210 mastershipCheck = main.TRUE
1211 mastershipState = ONOSMastership[ 0 ]
1212
1213 main.step( "Get the intents from each controller" )
1214 global intentState
1215 intentState = []
1216 ONOSIntents = []
1217 intentCheck = main.FALSE
1218 consistentIntents = True
1219 intentsResults = True
1220 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001221 for i in range( main.numCtrls ):
1222 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001223 name="intents-" + str( i ),
1224 args=[],
1225 kwargs={ 'jsonFormat': True } )
1226 threads.append( t )
1227 t.start()
1228
1229 for t in threads:
1230 t.join()
1231 ONOSIntents.append( t.result )
1232
Jon Halle1a3b752015-07-22 13:02:46 -07001233 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001234 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1235 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1236 " intents" )
1237 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1238 repr( ONOSIntents[ i ] ) )
1239 intentsResults = False
1240 utilities.assert_equals(
1241 expect=True,
1242 actual=intentsResults,
1243 onpass="No error in reading intents output",
1244 onfail="Error in reading intents from ONOS" )
1245
1246 main.step( "Check for consistency in Intents from each controller" )
1247 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1248 main.log.info( "Intents are consistent across all ONOS " +
1249 "nodes" )
1250 else:
1251 consistentIntents = False
1252 main.log.error( "Intents not consistent" )
1253 utilities.assert_equals(
1254 expect=True,
1255 actual=consistentIntents,
1256 onpass="Intents are consistent across all ONOS nodes",
1257 onfail="ONOS nodes have different views of intents" )
1258
1259 if intentsResults:
1260 # Try to make it easy to figure out what is happening
1261 #
1262 # Intent ONOS1 ONOS2 ...
1263 # 0x01 INSTALLED INSTALLING
1264 # ... ... ...
1265 # ... ... ...
1266 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001267 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001268 title += " " * 10 + "ONOS" + str( n + 1 )
1269 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001270 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001271 keys = []
1272 try:
1273 # Get the set of all intent keys
1274 for nodeStr in ONOSIntents:
1275 node = json.loads( nodeStr )
1276 for intent in node:
1277 keys.append( intent.get( 'id' ) )
1278 keys = set( keys )
1279 # For each intent key, print the state on each node
1280 for key in keys:
1281 row = "%-13s" % key
1282 for nodeStr in ONOSIntents:
1283 node = json.loads( nodeStr )
1284 for intent in node:
1285 if intent.get( 'id', "Error" ) == key:
1286 row += "%-15s" % intent.get( 'state' )
1287 main.log.warn( row )
1288 # End of intent state table
1289 except ValueError as e:
1290 main.log.exception( e )
1291 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1292
1293 if intentsResults and not consistentIntents:
1294 # print the json objects
1295 n = len(ONOSIntents)
1296 main.log.debug( "ONOS" + str( n ) + " intents: " )
1297 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1298 sort_keys=True,
1299 indent=4,
1300 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001301 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001302 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1303 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1304 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1305 sort_keys=True,
1306 indent=4,
1307 separators=( ',', ': ' ) ) )
1308 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001309 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001310 str( n ) + " intents" )
1311 elif intentsResults and consistentIntents:
1312 intentCheck = main.TRUE
1313 intentState = ONOSIntents[ 0 ]
1314
1315 main.step( "Get the flows from each controller" )
1316 global flowState
1317 flowState = []
1318 ONOSFlows = []
1319 ONOSFlowsJson = []
1320 flowCheck = main.FALSE
1321 consistentFlows = True
1322 flowsResults = True
1323 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001324 for i in range( main.numCtrls ):
1325 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001326 name="flows-" + str( i ),
1327 args=[],
1328 kwargs={ 'jsonFormat': True } )
1329 threads.append( t )
1330 t.start()
1331
1332 # NOTE: Flows command can take some time to run
1333 time.sleep(30)
1334 for t in threads:
1335 t.join()
1336 result = t.result
1337 ONOSFlows.append( result )
1338
Jon Halle1a3b752015-07-22 13:02:46 -07001339 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001340 num = str( i + 1 )
1341 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1342 main.log.error( "Error in getting ONOS" + num + " flows" )
1343 main.log.warn( "ONOS" + num + " flows response: " +
1344 repr( ONOSFlows[ i ] ) )
1345 flowsResults = False
1346 ONOSFlowsJson.append( None )
1347 else:
1348 try:
1349 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1350 except ( ValueError, TypeError ):
1351 # FIXME: change this to log.error?
1352 main.log.exception( "Error in parsing ONOS" + num +
1353 " response as json." )
1354 main.log.error( repr( ONOSFlows[ i ] ) )
1355 ONOSFlowsJson.append( None )
1356 flowsResults = False
1357 utilities.assert_equals(
1358 expect=True,
1359 actual=flowsResults,
1360 onpass="No error in reading flows output",
1361 onfail="Error in reading flows from ONOS" )
1362
1363 main.step( "Check for consistency in Flows from each controller" )
1364 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1365 if all( tmp ):
1366 main.log.info( "Flow count is consistent across all ONOS nodes" )
1367 else:
1368 consistentFlows = False
1369 utilities.assert_equals(
1370 expect=True,
1371 actual=consistentFlows,
1372 onpass="The flow count is consistent across all ONOS nodes",
1373 onfail="ONOS nodes have different flow counts" )
1374
1375 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001376 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001377 try:
1378 main.log.warn(
1379 "ONOS" + str( i + 1 ) + " flows: " +
1380 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1381 indent=4, separators=( ',', ': ' ) ) )
1382 except ( ValueError, TypeError ):
1383 main.log.warn(
1384 "ONOS" + str( i + 1 ) + " flows: " +
1385 repr( ONOSFlows[ i ] ) )
1386 elif flowsResults and consistentFlows:
1387 flowCheck = main.TRUE
1388 flowState = ONOSFlows[ 0 ]
1389
1390 main.step( "Get the OF Table entries" )
1391 global flows
1392 flows = []
1393 for i in range( 1, 29 ):
1394 flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
1395 if flowCheck == main.FALSE:
1396 for table in flows:
1397 main.log.warn( table )
1398 # TODO: Compare switch flow tables with ONOS flow tables
1399
1400 main.step( "Start continuous pings" )
1401 main.Mininet2.pingLong(
1402 src=main.params[ 'PING' ][ 'source1' ],
1403 target=main.params[ 'PING' ][ 'target1' ],
1404 pingTime=500 )
1405 main.Mininet2.pingLong(
1406 src=main.params[ 'PING' ][ 'source2' ],
1407 target=main.params[ 'PING' ][ 'target2' ],
1408 pingTime=500 )
1409 main.Mininet2.pingLong(
1410 src=main.params[ 'PING' ][ 'source3' ],
1411 target=main.params[ 'PING' ][ 'target3' ],
1412 pingTime=500 )
1413 main.Mininet2.pingLong(
1414 src=main.params[ 'PING' ][ 'source4' ],
1415 target=main.params[ 'PING' ][ 'target4' ],
1416 pingTime=500 )
1417 main.Mininet2.pingLong(
1418 src=main.params[ 'PING' ][ 'source5' ],
1419 target=main.params[ 'PING' ][ 'target5' ],
1420 pingTime=500 )
1421 main.Mininet2.pingLong(
1422 src=main.params[ 'PING' ][ 'source6' ],
1423 target=main.params[ 'PING' ][ 'target6' ],
1424 pingTime=500 )
1425 main.Mininet2.pingLong(
1426 src=main.params[ 'PING' ][ 'source7' ],
1427 target=main.params[ 'PING' ][ 'target7' ],
1428 pingTime=500 )
1429 main.Mininet2.pingLong(
1430 src=main.params[ 'PING' ][ 'source8' ],
1431 target=main.params[ 'PING' ][ 'target8' ],
1432 pingTime=500 )
1433 main.Mininet2.pingLong(
1434 src=main.params[ 'PING' ][ 'source9' ],
1435 target=main.params[ 'PING' ][ 'target9' ],
1436 pingTime=500 )
1437 main.Mininet2.pingLong(
1438 src=main.params[ 'PING' ][ 'source10' ],
1439 target=main.params[ 'PING' ][ 'target10' ],
1440 pingTime=500 )
1441
1442 main.step( "Collecting topology information from ONOS" )
1443 devices = []
1444 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001445 for i in range( main.numCtrls ):
1446 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001447 name="devices-" + str( i ),
1448 args=[ ] )
1449 threads.append( t )
1450 t.start()
1451
1452 for t in threads:
1453 t.join()
1454 devices.append( t.result )
1455 hosts = []
1456 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001457 for i in range( main.numCtrls ):
1458 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001459 name="hosts-" + str( i ),
1460 args=[ ] )
1461 threads.append( t )
1462 t.start()
1463
1464 for t in threads:
1465 t.join()
1466 try:
1467 hosts.append( json.loads( t.result ) )
1468 except ( ValueError, TypeError ):
1469 # FIXME: better handling of this, print which node
1470 # Maybe use thread name?
1471 main.log.exception( "Error parsing json output of hosts" )
1472 # FIXME: should this be an empty json object instead?
1473 hosts.append( None )
1474
1475 ports = []
1476 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001477 for i in range( main.numCtrls ):
1478 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001479 name="ports-" + str( i ),
1480 args=[ ] )
1481 threads.append( t )
1482 t.start()
1483
1484 for t in threads:
1485 t.join()
1486 ports.append( t.result )
1487 links = []
1488 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001489 for i in range( main.numCtrls ):
1490 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001491 name="links-" + str( i ),
1492 args=[ ] )
1493 threads.append( t )
1494 t.start()
1495
1496 for t in threads:
1497 t.join()
1498 links.append( t.result )
1499 clusters = []
1500 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001501 for i in range( main.numCtrls ):
1502 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001503 name="clusters-" + str( i ),
1504 args=[ ] )
1505 threads.append( t )
1506 t.start()
1507
1508 for t in threads:
1509 t.join()
1510 clusters.append( t.result )
1511 # Compare json objects for hosts and dataplane clusters
1512
1513 # hosts
1514 main.step( "Host view is consistent across ONOS nodes" )
1515 consistentHostsResult = main.TRUE
1516 for controller in range( len( hosts ) ):
1517 controllerStr = str( controller + 1 )
1518 if "Error" not in hosts[ controller ]:
1519 if hosts[ controller ] == hosts[ 0 ]:
1520 continue
1521 else: # hosts not consistent
1522 main.log.error( "hosts from ONOS" +
1523 controllerStr +
1524 " is inconsistent with ONOS1" )
1525 main.log.warn( repr( hosts[ controller ] ) )
1526 consistentHostsResult = main.FALSE
1527
1528 else:
1529 main.log.error( "Error in getting ONOS hosts from ONOS" +
1530 controllerStr )
1531 consistentHostsResult = main.FALSE
1532 main.log.warn( "ONOS" + controllerStr +
1533 " hosts response: " +
1534 repr( hosts[ controller ] ) )
1535 utilities.assert_equals(
1536 expect=main.TRUE,
1537 actual=consistentHostsResult,
1538 onpass="Hosts view is consistent across all ONOS nodes",
1539 onfail="ONOS nodes have different views of hosts" )
1540
1541 main.step( "Each host has an IP address" )
1542 ipResult = main.TRUE
1543 for controller in range( 0, len( hosts ) ):
1544 controllerStr = str( controller + 1 )
1545 for host in hosts[ controller ]:
1546 if not host.get( 'ipAddresses', [ ] ):
1547 main.log.error( "DEBUG:Error with host ips on controller" +
1548 controllerStr + ": " + str( host ) )
1549 ipResult = main.FALSE
1550 utilities.assert_equals(
1551 expect=main.TRUE,
1552 actual=ipResult,
1553 onpass="The ips of the hosts aren't empty",
1554 onfail="The ip of at least one host is missing" )
1555
1556 # Strongly connected clusters of devices
1557 main.step( "Cluster view is consistent across ONOS nodes" )
1558 consistentClustersResult = main.TRUE
1559 for controller in range( len( clusters ) ):
1560 controllerStr = str( controller + 1 )
1561 if "Error" not in clusters[ controller ]:
1562 if clusters[ controller ] == clusters[ 0 ]:
1563 continue
1564 else: # clusters not consistent
1565 main.log.error( "clusters from ONOS" + controllerStr +
1566 " is inconsistent with ONOS1" )
1567 consistentClustersResult = main.FALSE
1568
1569 else:
1570 main.log.error( "Error in getting dataplane clusters " +
1571 "from ONOS" + controllerStr )
1572 consistentClustersResult = main.FALSE
1573 main.log.warn( "ONOS" + controllerStr +
1574 " clusters response: " +
1575 repr( clusters[ controller ] ) )
1576 utilities.assert_equals(
1577 expect=main.TRUE,
1578 actual=consistentClustersResult,
1579 onpass="Clusters view is consistent across all ONOS nodes",
1580 onfail="ONOS nodes have different views of clusters" )
1581 # there should always only be one cluster
1582 main.step( "Cluster view correct across ONOS nodes" )
1583 try:
1584 numClusters = len( json.loads( clusters[ 0 ] ) )
1585 except ( ValueError, TypeError ):
1586 main.log.exception( "Error parsing clusters[0]: " +
1587 repr( clusters[ 0 ] ) )
1588 clusterResults = main.FALSE
1589 if numClusters == 1:
1590 clusterResults = main.TRUE
1591 utilities.assert_equals(
1592 expect=1,
1593 actual=numClusters,
1594 onpass="ONOS shows 1 SCC",
1595 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1596
1597 main.step( "Comparing ONOS topology to MN" )
1598 devicesResults = main.TRUE
1599 linksResults = main.TRUE
1600 hostsResults = main.TRUE
1601 mnSwitches = main.Mininet1.getSwitches()
1602 mnLinks = main.Mininet1.getLinks()
1603 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001604 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001605 controllerStr = str( controller + 1 )
1606 if devices[ controller ] and ports[ controller ] and\
1607 "Error" not in devices[ controller ] and\
1608 "Error" not in ports[ controller ]:
1609
1610 currentDevicesResult = main.Mininet1.compareSwitches(
1611 mnSwitches,
1612 json.loads( devices[ controller ] ),
1613 json.loads( ports[ controller ] ) )
1614 else:
1615 currentDevicesResult = main.FALSE
1616 utilities.assert_equals( expect=main.TRUE,
1617 actual=currentDevicesResult,
1618 onpass="ONOS" + controllerStr +
1619 " Switches view is correct",
1620 onfail="ONOS" + controllerStr +
1621 " Switches view is incorrect" )
1622 if links[ controller ] and "Error" not in links[ controller ]:
1623 currentLinksResult = main.Mininet1.compareLinks(
1624 mnSwitches, mnLinks,
1625 json.loads( links[ controller ] ) )
1626 else:
1627 currentLinksResult = main.FALSE
1628 utilities.assert_equals( expect=main.TRUE,
1629 actual=currentLinksResult,
1630 onpass="ONOS" + controllerStr +
1631 " links view is correct",
1632 onfail="ONOS" + controllerStr +
1633 " links view is incorrect" )
1634
1635 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1636 currentHostsResult = main.Mininet1.compareHosts(
1637 mnHosts,
1638 hosts[ controller ] )
1639 else:
1640 currentHostsResult = main.FALSE
1641 utilities.assert_equals( expect=main.TRUE,
1642 actual=currentHostsResult,
1643 onpass="ONOS" + controllerStr +
1644 " hosts exist in Mininet",
1645 onfail="ONOS" + controllerStr +
1646 " hosts don't match Mininet" )
1647
1648 devicesResults = devicesResults and currentDevicesResult
1649 linksResults = linksResults and currentLinksResult
1650 hostsResults = hostsResults and currentHostsResult
1651
1652 main.step( "Device information is correct" )
1653 utilities.assert_equals(
1654 expect=main.TRUE,
1655 actual=devicesResults,
1656 onpass="Device information is correct",
1657 onfail="Device information is incorrect" )
1658
1659 main.step( "Links are correct" )
1660 utilities.assert_equals(
1661 expect=main.TRUE,
1662 actual=linksResults,
1663 onpass="Link are correct",
1664 onfail="Links are incorrect" )
1665
1666 main.step( "Hosts are correct" )
1667 utilities.assert_equals(
1668 expect=main.TRUE,
1669 actual=hostsResults,
1670 onpass="Hosts are correct",
1671 onfail="Hosts are incorrect" )
1672
1673 def CASE6( self, main ):
1674 """
1675 The Failure case. Since this is the Sanity test, we do nothing.
1676 """
1677 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001678 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001679 assert main, "main not defined"
1680 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001681 assert main.CLIs, "main.CLIs not defined"
1682 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001683 main.case( "Wait 60 seconds instead of inducing a failure" )
1684 time.sleep( 60 )
1685 utilities.assert_equals(
1686 expect=main.TRUE,
1687 actual=main.TRUE,
1688 onpass="Sleeping 60 seconds",
1689 onfail="Something is terribly wrong with my math" )
1690
1691 def CASE7( self, main ):
1692 """
1693 Check state after ONOS failure
1694 """
1695 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001696 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001697 assert main, "main not defined"
1698 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001699 assert main.CLIs, "main.CLIs not defined"
1700 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001701 main.case( "Running ONOS Constant State Tests" )
1702
1703 main.step( "Check that each switch has a master" )
1704 # Assert that each device has a master
1705 rolesNotNull = main.TRUE
1706 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001707 for i in range( main.numCtrls ):
1708 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001709 name="rolesNotNull-" + str( i ),
1710 args=[ ] )
1711 threads.append( t )
1712 t.start()
1713
1714 for t in threads:
1715 t.join()
1716 rolesNotNull = rolesNotNull and t.result
1717 utilities.assert_equals(
1718 expect=main.TRUE,
1719 actual=rolesNotNull,
1720 onpass="Each device has a master",
1721 onfail="Some devices don't have a master assigned" )
1722
1723 main.step( "Read device roles from ONOS" )
1724 ONOSMastership = []
1725 mastershipCheck = main.FALSE
1726 consistentMastership = True
1727 rolesResults = True
1728 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001729 for i in range( main.numCtrls ):
1730 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001731 name="roles-" + str( i ),
1732 args=[] )
1733 threads.append( t )
1734 t.start()
1735
1736 for t in threads:
1737 t.join()
1738 ONOSMastership.append( t.result )
1739
Jon Halle1a3b752015-07-22 13:02:46 -07001740 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001741 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1742 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1743 " roles" )
1744 main.log.warn(
1745 "ONOS" + str( i + 1 ) + " mastership response: " +
1746 repr( ONOSMastership[i] ) )
1747 rolesResults = False
1748 utilities.assert_equals(
1749 expect=True,
1750 actual=rolesResults,
1751 onpass="No error in reading roles output",
1752 onfail="Error in reading roles from ONOS" )
1753
1754 main.step( "Check for consistency in roles from each controller" )
1755 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1756 main.log.info(
1757 "Switch roles are consistent across all ONOS nodes" )
1758 else:
1759 consistentMastership = False
1760 utilities.assert_equals(
1761 expect=True,
1762 actual=consistentMastership,
1763 onpass="Switch roles are consistent across all ONOS nodes",
1764 onfail="ONOS nodes have different views of switch roles" )
1765
1766 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001767 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001768 main.log.warn(
1769 "ONOS" + str( i + 1 ) + " roles: ",
1770 json.dumps(
1771 json.loads( ONOSMastership[ i ] ),
1772 sort_keys=True,
1773 indent=4,
1774 separators=( ',', ': ' ) ) )
1775 elif rolesResults and not consistentMastership:
1776 mastershipCheck = main.TRUE
1777
1778 description2 = "Compare switch roles from before failure"
1779 main.step( description2 )
1780 try:
1781 currentJson = json.loads( ONOSMastership[0] )
1782 oldJson = json.loads( mastershipState )
1783 except ( ValueError, TypeError ):
1784 main.log.exception( "Something is wrong with parsing " +
1785 "ONOSMastership[0] or mastershipState" )
1786 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1787 main.log.error( "mastershipState" + repr( mastershipState ) )
1788 main.cleanup()
1789 main.exit()
1790 mastershipCheck = main.TRUE
1791 for i in range( 1, 29 ):
1792 switchDPID = str(
1793 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1794 current = [ switch[ 'master' ] for switch in currentJson
1795 if switchDPID in switch[ 'id' ] ]
1796 old = [ switch[ 'master' ] for switch in oldJson
1797 if switchDPID in switch[ 'id' ] ]
1798 if current == old:
1799 mastershipCheck = mastershipCheck and main.TRUE
1800 else:
1801 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1802 mastershipCheck = main.FALSE
1803 utilities.assert_equals(
1804 expect=main.TRUE,
1805 actual=mastershipCheck,
1806 onpass="Mastership of Switches was not changed",
1807 onfail="Mastership of some switches changed" )
1808 mastershipCheck = mastershipCheck and consistentMastership
1809
1810 main.step( "Get the intents and compare across all nodes" )
1811 ONOSIntents = []
1812 intentCheck = main.FALSE
1813 consistentIntents = True
1814 intentsResults = True
1815 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001816 for i in range( main.numCtrls ):
1817 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001818 name="intents-" + str( i ),
1819 args=[],
1820 kwargs={ 'jsonFormat': True } )
1821 threads.append( t )
1822 t.start()
1823
1824 for t in threads:
1825 t.join()
1826 ONOSIntents.append( t.result )
1827
Jon Halle1a3b752015-07-22 13:02:46 -07001828 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001829 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1830 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1831 " intents" )
1832 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1833 repr( ONOSIntents[ i ] ) )
1834 intentsResults = False
1835 utilities.assert_equals(
1836 expect=True,
1837 actual=intentsResults,
1838 onpass="No error in reading intents output",
1839 onfail="Error in reading intents from ONOS" )
1840
1841 main.step( "Check for consistency in Intents from each controller" )
1842 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1843 main.log.info( "Intents are consistent across all ONOS " +
1844 "nodes" )
1845 else:
1846 consistentIntents = False
1847
1848 # Try to make it easy to figure out what is happening
1849 #
1850 # Intent ONOS1 ONOS2 ...
1851 # 0x01 INSTALLED INSTALLING
1852 # ... ... ...
1853 # ... ... ...
1854 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001855 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001856 title += " " * 10 + "ONOS" + str( n + 1 )
1857 main.log.warn( title )
1858 # get all intent keys in the cluster
1859 keys = []
1860 for nodeStr in ONOSIntents:
1861 node = json.loads( nodeStr )
1862 for intent in node:
1863 keys.append( intent.get( 'id' ) )
1864 keys = set( keys )
1865 for key in keys:
1866 row = "%-13s" % key
1867 for nodeStr in ONOSIntents:
1868 node = json.loads( nodeStr )
1869 for intent in node:
1870 if intent.get( 'id' ) == key:
1871 row += "%-15s" % intent.get( 'state' )
1872 main.log.warn( row )
1873 # End table view
1874
1875 utilities.assert_equals(
1876 expect=True,
1877 actual=consistentIntents,
1878 onpass="Intents are consistent across all ONOS nodes",
1879 onfail="ONOS nodes have different views of intents" )
1880 intentStates = []
1881 for node in ONOSIntents: # Iter through ONOS nodes
1882 nodeStates = []
1883 # Iter through intents of a node
1884 try:
1885 for intent in json.loads( node ):
1886 nodeStates.append( intent[ 'state' ] )
1887 except ( ValueError, TypeError ):
1888 main.log.exception( "Error in parsing intents" )
1889 main.log.error( repr( node ) )
1890 intentStates.append( nodeStates )
1891 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1892 main.log.info( dict( out ) )
1893
1894 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001895 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001896 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1897 main.log.warn( json.dumps(
1898 json.loads( ONOSIntents[ i ] ),
1899 sort_keys=True,
1900 indent=4,
1901 separators=( ',', ': ' ) ) )
1902 elif intentsResults and consistentIntents:
1903 intentCheck = main.TRUE
1904
1905 # NOTE: Store has no durability, so intents are lost across system
1906 # restarts
1907 main.step( "Compare current intents with intents before the failure" )
1908 # NOTE: this requires case 5 to pass for intentState to be set.
1909 # maybe we should stop the test if that fails?
1910 sameIntents = main.FALSE
1911 if intentState and intentState == ONOSIntents[ 0 ]:
1912 sameIntents = main.TRUE
1913 main.log.info( "Intents are consistent with before failure" )
1914 # TODO: possibly the states have changed? we may need to figure out
1915 # what the acceptable states are
1916 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1917 sameIntents = main.TRUE
1918 try:
1919 before = json.loads( intentState )
1920 after = json.loads( ONOSIntents[ 0 ] )
1921 for intent in before:
1922 if intent not in after:
1923 sameIntents = main.FALSE
1924 main.log.debug( "Intent is not currently in ONOS " +
1925 "(at least in the same form):" )
1926 main.log.debug( json.dumps( intent ) )
1927 except ( ValueError, TypeError ):
1928 main.log.exception( "Exception printing intents" )
1929 main.log.debug( repr( ONOSIntents[0] ) )
1930 main.log.debug( repr( intentState ) )
1931 if sameIntents == main.FALSE:
1932 try:
1933 main.log.debug( "ONOS intents before: " )
1934 main.log.debug( json.dumps( json.loads( intentState ),
1935 sort_keys=True, indent=4,
1936 separators=( ',', ': ' ) ) )
1937 main.log.debug( "Current ONOS intents: " )
1938 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1939 sort_keys=True, indent=4,
1940 separators=( ',', ': ' ) ) )
1941 except ( ValueError, TypeError ):
1942 main.log.exception( "Exception printing intents" )
1943 main.log.debug( repr( ONOSIntents[0] ) )
1944 main.log.debug( repr( intentState ) )
1945 utilities.assert_equals(
1946 expect=main.TRUE,
1947 actual=sameIntents,
1948 onpass="Intents are consistent with before failure",
1949 onfail="The Intents changed during failure" )
1950 intentCheck = intentCheck and sameIntents
1951
1952 main.step( "Get the OF Table entries and compare to before " +
1953 "component failure" )
1954 FlowTables = main.TRUE
1955 flows2 = []
1956 for i in range( 28 ):
1957 main.log.info( "Checking flow table on s" + str( i + 1 ) )
1958 tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
1959 flows2.append( tmpFlows )
1960 tempResult = main.Mininet2.flowComp(
1961 flow1=flows[ i ],
1962 flow2=tmpFlows )
1963 FlowTables = FlowTables and tempResult
1964 if FlowTables == main.FALSE:
1965 main.log.info( "Differences in flow table for switch: s" +
1966 str( i + 1 ) )
1967 utilities.assert_equals(
1968 expect=main.TRUE,
1969 actual=FlowTables,
1970 onpass="No changes were found in the flow tables",
1971 onfail="Changes were found in the flow tables" )
1972
1973 main.Mininet2.pingLongKill()
1974 '''
1975 main.step( "Check the continuous pings to ensure that no packets " +
1976 "were dropped during component failure" )
1977 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1978 main.params[ 'TESTONIP' ] )
1979 LossInPings = main.FALSE
1980 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1981 for i in range( 8, 18 ):
1982 main.log.info(
1983 "Checking for a loss in pings along flow from s" +
1984 str( i ) )
1985 LossInPings = main.Mininet2.checkForLoss(
1986 "/tmp/ping.h" +
1987 str( i ) ) or LossInPings
1988 if LossInPings == main.TRUE:
1989 main.log.info( "Loss in ping detected" )
1990 elif LossInPings == main.ERROR:
1991 main.log.info( "There are multiple mininet process running" )
1992 elif LossInPings == main.FALSE:
1993 main.log.info( "No Loss in the pings" )
1994 main.log.info( "No loss of dataplane connectivity" )
1995 utilities.assert_equals(
1996 expect=main.FALSE,
1997 actual=LossInPings,
1998 onpass="No Loss of connectivity",
1999 onfail="Loss of dataplane connectivity detected" )
2000 '''
2001
2002 main.step( "Leadership Election is still functional" )
2003 # Test of LeadershipElection
2004 # NOTE: this only works for the sanity test. In case of failures,
2005 # leader will likely change
Jon Halle1a3b752015-07-22 13:02:46 -07002006 leader = main.nodes[ 0 ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002007 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002008 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002009 leaderN = cli.electionTestLeader()
2010 # verify leader is ONOS1
2011 if leaderN == leader:
2012 # all is well
2013 # NOTE: In failure scenario, this could be a new node, maybe
2014 # check != ONOS1
2015 pass
2016 elif leaderN == main.FALSE:
2017 # error in response
2018 main.log.error( "Something is wrong with " +
2019 "electionTestLeader function, check the" +
2020 " error logs" )
2021 leaderResult = main.FALSE
2022 elif leader != leaderN:
2023 leaderResult = main.FALSE
2024 main.log.error( cli.name + " sees " + str( leaderN ) +
2025 " as the leader of the election app. " +
2026 "Leader should be " + str( leader ) )
2027 utilities.assert_equals(
2028 expect=main.TRUE,
2029 actual=leaderResult,
2030 onpass="Leadership election passed",
2031 onfail="Something went wrong with Leadership election" )
2032
2033 def CASE8( self, main ):
2034 """
2035 Compare topo
2036 """
2037 import json
2038 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002039 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002040 assert main, "main not defined"
2041 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002042 assert main.CLIs, "main.CLIs not defined"
2043 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002044
2045 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002046 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002047 " and ONOS"
2048
2049 main.step( "Comparing ONOS topology to MN" )
2050 devicesResults = main.TRUE
2051 linksResults = main.TRUE
2052 hostsResults = main.TRUE
2053 hostAttachmentResults = True
2054 topoResult = main.FALSE
2055 elapsed = 0
2056 count = 0
2057 main.step( "Collecting topology information from ONOS" )
2058 startTime = time.time()
2059 # Give time for Gossip to work
2060 while topoResult == main.FALSE and elapsed < 60:
2061 count += 1
2062 cliStart = time.time()
2063 devices = []
2064 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002065 for i in range( main.numCtrls ):
2066 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002067 name="devices-" + str( i ),
2068 args=[ ] )
2069 threads.append( t )
2070 t.start()
2071
2072 for t in threads:
2073 t.join()
2074 devices.append( t.result )
2075 hosts = []
2076 ipResult = main.TRUE
2077 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002078 for i in range( main.numCtrls ):
2079 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07002080 name="hosts-" + str( i ),
2081 args=[ ] )
2082 threads.append( t )
2083 t.start()
2084
2085 for t in threads:
2086 t.join()
2087 try:
2088 hosts.append( json.loads( t.result ) )
2089 except ( ValueError, TypeError ):
2090 main.log.exception( "Error parsing hosts results" )
2091 main.log.error( repr( t.result ) )
2092 for controller in range( 0, len( hosts ) ):
2093 controllerStr = str( controller + 1 )
2094 for host in hosts[ controller ]:
2095 if host is None or host.get( 'ipAddresses', [] ) == []:
2096 main.log.error(
2097 "DEBUG:Error with host ipAddresses on controller" +
2098 controllerStr + ": " + str( host ) )
2099 ipResult = main.FALSE
2100 ports = []
2101 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002102 for i in range( main.numCtrls ):
2103 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002104 name="ports-" + str( i ),
2105 args=[ ] )
2106 threads.append( t )
2107 t.start()
2108
2109 for t in threads:
2110 t.join()
2111 ports.append( t.result )
2112 links = []
2113 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002114 for i in range( main.numCtrls ):
2115 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002116 name="links-" + str( i ),
2117 args=[ ] )
2118 threads.append( t )
2119 t.start()
2120
2121 for t in threads:
2122 t.join()
2123 links.append( t.result )
2124 clusters = []
2125 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002126 for i in range( main.numCtrls ):
2127 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002128 name="clusters-" + str( i ),
2129 args=[ ] )
2130 threads.append( t )
2131 t.start()
2132
2133 for t in threads:
2134 t.join()
2135 clusters.append( t.result )
2136
2137 elapsed = time.time() - startTime
2138 cliTime = time.time() - cliStart
2139 print "Elapsed time: " + str( elapsed )
2140 print "CLI time: " + str( cliTime )
2141
2142 mnSwitches = main.Mininet1.getSwitches()
2143 mnLinks = main.Mininet1.getLinks()
2144 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002145 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002146 controllerStr = str( controller + 1 )
2147 if devices[ controller ] and ports[ controller ] and\
2148 "Error" not in devices[ controller ] and\
2149 "Error" not in ports[ controller ]:
2150
2151 currentDevicesResult = main.Mininet1.compareSwitches(
2152 mnSwitches,
2153 json.loads( devices[ controller ] ),
2154 json.loads( ports[ controller ] ) )
2155 else:
2156 currentDevicesResult = main.FALSE
2157 utilities.assert_equals( expect=main.TRUE,
2158 actual=currentDevicesResult,
2159 onpass="ONOS" + controllerStr +
2160 " Switches view is correct",
2161 onfail="ONOS" + controllerStr +
2162 " Switches view is incorrect" )
2163
2164 if links[ controller ] and "Error" not in links[ controller ]:
2165 currentLinksResult = main.Mininet1.compareLinks(
2166 mnSwitches, mnLinks,
2167 json.loads( links[ controller ] ) )
2168 else:
2169 currentLinksResult = main.FALSE
2170 utilities.assert_equals( expect=main.TRUE,
2171 actual=currentLinksResult,
2172 onpass="ONOS" + controllerStr +
2173 " links view is correct",
2174 onfail="ONOS" + controllerStr +
2175 " links view is incorrect" )
2176
2177 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2178 currentHostsResult = main.Mininet1.compareHosts(
2179 mnHosts,
2180 hosts[ controller ] )
2181 else:
2182 currentHostsResult = main.FALSE
2183 utilities.assert_equals( expect=main.TRUE,
2184 actual=currentHostsResult,
2185 onpass="ONOS" + controllerStr +
2186 " hosts exist in Mininet",
2187 onfail="ONOS" + controllerStr +
2188 " hosts don't match Mininet" )
2189 # CHECKING HOST ATTACHMENT POINTS
2190 hostAttachment = True
2191 zeroHosts = False
2192 # FIXME: topo-HA/obelisk specific mappings:
2193 # key is mac and value is dpid
2194 mappings = {}
2195 for i in range( 1, 29 ): # hosts 1 through 28
2196 # set up correct variables:
2197 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2198 if i == 1:
2199 deviceId = "1000".zfill(16)
2200 elif i == 2:
2201 deviceId = "2000".zfill(16)
2202 elif i == 3:
2203 deviceId = "3000".zfill(16)
2204 elif i == 4:
2205 deviceId = "3004".zfill(16)
2206 elif i == 5:
2207 deviceId = "5000".zfill(16)
2208 elif i == 6:
2209 deviceId = "6000".zfill(16)
2210 elif i == 7:
2211 deviceId = "6007".zfill(16)
2212 elif i >= 8 and i <= 17:
2213 dpid = '3' + str( i ).zfill( 3 )
2214 deviceId = dpid.zfill(16)
2215 elif i >= 18 and i <= 27:
2216 dpid = '6' + str( i ).zfill( 3 )
2217 deviceId = dpid.zfill(16)
2218 elif i == 28:
2219 deviceId = "2800".zfill(16)
2220 mappings[ macId ] = deviceId
2221 if hosts[ controller ] or "Error" not in hosts[ controller ]:
2222 if hosts[ controller ] == []:
2223 main.log.warn( "There are no hosts discovered" )
2224 zeroHosts = True
2225 else:
2226 for host in hosts[ controller ]:
2227 mac = None
2228 location = None
2229 device = None
2230 port = None
2231 try:
2232 mac = host.get( 'mac' )
2233 assert mac, "mac field could not be found for this host object"
2234
2235 location = host.get( 'location' )
2236 assert location, "location field could not be found for this host object"
2237
2238 # Trim the protocol identifier off deviceId
2239 device = str( location.get( 'elementId' ) ).split(':')[1]
2240 assert device, "elementId field could not be found for this host location object"
2241
2242 port = location.get( 'port' )
2243 assert port, "port field could not be found for this host location object"
2244
2245 # Now check if this matches where they should be
2246 if mac and device and port:
2247 if str( port ) != "1":
2248 main.log.error( "The attachment port is incorrect for " +
2249 "host " + str( mac ) +
2250 ". Expected: 1 Actual: " + str( port) )
2251 hostAttachment = False
2252 if device != mappings[ str( mac ) ]:
2253 main.log.error( "The attachment device is incorrect for " +
2254 "host " + str( mac ) +
2255 ". Expected: " + mappings[ str( mac ) ] +
2256 " Actual: " + device )
2257 hostAttachment = False
2258 else:
2259 hostAttachment = False
2260 except AssertionError:
2261 main.log.exception( "Json object not as expected" )
2262 main.log.error( repr( host ) )
2263 hostAttachment = False
2264 else:
2265 main.log.error( "No hosts json output or \"Error\"" +
2266 " in output. hosts = " +
2267 repr( hosts[ controller ] ) )
2268 if zeroHosts is False:
2269 hostAttachment = True
2270
2271 # END CHECKING HOST ATTACHMENT POINTS
2272 devicesResults = devicesResults and currentDevicesResult
2273 linksResults = linksResults and currentLinksResult
2274 hostsResults = hostsResults and currentHostsResult
2275 hostAttachmentResults = hostAttachmentResults and\
2276 hostAttachment
2277 topoResult = ( devicesResults and linksResults
2278 and hostsResults and ipResult and
2279 hostAttachmentResults )
2280
2281 # Compare json objects for hosts and dataplane clusters
2282
2283 # hosts
2284 main.step( "Hosts view is consistent across all ONOS nodes" )
2285 consistentHostsResult = main.TRUE
2286 for controller in range( len( hosts ) ):
2287 controllerStr = str( controller + 1 )
2288 if "Error" not in hosts[ controller ]:
2289 if hosts[ controller ] == hosts[ 0 ]:
2290 continue
2291 else: # hosts not consistent
2292 main.log.error( "hosts from ONOS" + controllerStr +
2293 " is inconsistent with ONOS1" )
2294 main.log.warn( repr( hosts[ controller ] ) )
2295 consistentHostsResult = main.FALSE
2296
2297 else:
2298 main.log.error( "Error in getting ONOS hosts from ONOS" +
2299 controllerStr )
2300 consistentHostsResult = main.FALSE
2301 main.log.warn( "ONOS" + controllerStr +
2302 " hosts response: " +
2303 repr( hosts[ controller ] ) )
2304 utilities.assert_equals(
2305 expect=main.TRUE,
2306 actual=consistentHostsResult,
2307 onpass="Hosts view is consistent across all ONOS nodes",
2308 onfail="ONOS nodes have different views of hosts" )
2309
2310 main.step( "Hosts information is correct" )
2311 hostsResults = hostsResults and ipResult
2312 utilities.assert_equals(
2313 expect=main.TRUE,
2314 actual=hostsResults,
2315 onpass="Host information is correct",
2316 onfail="Host information is incorrect" )
2317
2318 main.step( "Host attachment points to the network" )
2319 utilities.assert_equals(
2320 expect=True,
2321 actual=hostAttachmentResults,
2322 onpass="Hosts are correctly attached to the network",
2323 onfail="ONOS did not correctly attach hosts to the network" )
2324
2325 # Strongly connected clusters of devices
2326 main.step( "Clusters view is consistent across all ONOS nodes" )
2327 consistentClustersResult = main.TRUE
2328 for controller in range( len( clusters ) ):
2329 controllerStr = str( controller + 1 )
2330 if "Error" not in clusters[ controller ]:
2331 if clusters[ controller ] == clusters[ 0 ]:
2332 continue
2333 else: # clusters not consistent
2334 main.log.error( "clusters from ONOS" +
2335 controllerStr +
2336 " is inconsistent with ONOS1" )
2337 consistentClustersResult = main.FALSE
2338
2339 else:
2340 main.log.error( "Error in getting dataplane clusters " +
2341 "from ONOS" + controllerStr )
2342 consistentClustersResult = main.FALSE
2343 main.log.warn( "ONOS" + controllerStr +
2344 " clusters response: " +
2345 repr( clusters[ controller ] ) )
2346 utilities.assert_equals(
2347 expect=main.TRUE,
2348 actual=consistentClustersResult,
2349 onpass="Clusters view is consistent across all ONOS nodes",
2350 onfail="ONOS nodes have different views of clusters" )
2351
2352 main.step( "There is only one SCC" )
2353 # there should always only be one cluster
2354 try:
2355 numClusters = len( json.loads( clusters[ 0 ] ) )
2356 except ( ValueError, TypeError ):
2357 main.log.exception( "Error parsing clusters[0]: " +
2358 repr( clusters[0] ) )
2359 clusterResults = main.FALSE
2360 if numClusters == 1:
2361 clusterResults = main.TRUE
2362 utilities.assert_equals(
2363 expect=1,
2364 actual=numClusters,
2365 onpass="ONOS shows 1 SCC",
2366 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2367
2368 topoResult = ( devicesResults and linksResults
2369 and hostsResults and consistentHostsResult
2370 and consistentClustersResult and clusterResults
2371 and ipResult and hostAttachmentResults )
2372
2373 topoResult = topoResult and int( count <= 2 )
2374 note = "note it takes about " + str( int( cliTime ) ) + \
2375 " seconds for the test to make all the cli calls to fetch " +\
2376 "the topology from each ONOS instance"
2377 main.log.info(
2378 "Very crass estimate for topology discovery/convergence( " +
2379 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2380 str( count ) + " tries" )
2381
2382 main.step( "Device information is correct" )
2383 utilities.assert_equals(
2384 expect=main.TRUE,
2385 actual=devicesResults,
2386 onpass="Device information is correct",
2387 onfail="Device information is incorrect" )
2388
2389 main.step( "Links are correct" )
2390 utilities.assert_equals(
2391 expect=main.TRUE,
2392 actual=linksResults,
2393 onpass="Link are correct",
2394 onfail="Links are incorrect" )
2395
2396 main.step( "Hosts are correct" )
2397 utilities.assert_equals(
2398 expect=main.TRUE,
2399 actual=hostsResults,
2400 onpass="Hosts are correct",
2401 onfail="Hosts are incorrect" )
2402
2403 # FIXME: move this to an ONOS state case
2404 main.step( "Checking ONOS nodes" )
2405 nodesOutput = []
2406 nodeResults = main.TRUE
2407 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002408 for i in range( main.numCtrls ):
2409 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002410 name="nodes-" + str( i ),
2411 args=[ ] )
2412 threads.append( t )
2413 t.start()
2414
2415 for t in threads:
2416 t.join()
2417 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002418 ips = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002419 for i in nodesOutput:
2420 try:
2421 current = json.loads( i )
2422 for node in current:
2423 currentResult = main.FALSE
2424 if node['ip'] in ips: # node in nodes() output is in cell
2425 if node['state'] == 'ACTIVE':
2426 currentResult = main.TRUE
2427 else:
2428 main.log.error( "Error in ONOS node availability" )
2429 main.log.error(
2430 json.dumps( current,
2431 sort_keys=True,
2432 indent=4,
2433 separators=( ',', ': ' ) ) )
2434 break
2435 nodeResults = nodeResults and currentResult
2436 except ( ValueError, TypeError ):
2437 main.log.error( "Error parsing nodes output" )
2438 main.log.warn( repr( i ) )
2439 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2440 onpass="Nodes check successful",
2441 onfail="Nodes check NOT successful" )
2442
2443 def CASE9( self, main ):
2444 """
2445 Link s3-s28 down
2446 """
2447 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002448 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002449 assert main, "main not defined"
2450 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002451 assert main.CLIs, "main.CLIs not defined"
2452 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002453 # NOTE: You should probably run a topology check after this
2454
2455 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2456
2457 description = "Turn off a link to ensure that Link Discovery " +\
2458 "is working properly"
2459 main.case( description )
2460
2461 main.step( "Kill Link between s3 and s28" )
2462 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2463 main.log.info( "Waiting " + str( linkSleep ) +
2464 " seconds for link down to be discovered" )
2465 time.sleep( linkSleep )
2466 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2467 onpass="Link down successful",
2468 onfail="Failed to bring link down" )
2469 # TODO do some sort of check here
2470
2471 def CASE10( self, main ):
2472 """
2473 Link s3-s28 up
2474 """
2475 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002476 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002477 assert main, "main not defined"
2478 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002479 assert main.CLIs, "main.CLIs not defined"
2480 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002481 # NOTE: You should probably run a topology check after this
2482
2483 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2484
2485 description = "Restore a link to ensure that Link Discovery is " + \
2486 "working properly"
2487 main.case( description )
2488
2489 main.step( "Bring link between s3 and s28 back up" )
2490 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2491 main.log.info( "Waiting " + str( linkSleep ) +
2492 " seconds for link up to be discovered" )
2493 time.sleep( linkSleep )
2494 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2495 onpass="Link up successful",
2496 onfail="Failed to bring link up" )
2497 # TODO do some sort of check here
2498
2499 def CASE11( self, main ):
2500 """
2501 Switch Down
2502 """
2503 # NOTE: You should probably run a topology check after this
2504 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002505 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002506 assert main, "main not defined"
2507 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002508 assert main.CLIs, "main.CLIs not defined"
2509 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002510
2511 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2512
2513 description = "Killing a switch to ensure it is discovered correctly"
2514 main.case( description )
2515 switch = main.params[ 'kill' ][ 'switch' ]
2516 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2517
2518 # TODO: Make this switch parameterizable
2519 main.step( "Kill " + switch )
2520 main.log.info( "Deleting " + switch )
2521 main.Mininet1.delSwitch( switch )
2522 main.log.info( "Waiting " + str( switchSleep ) +
2523 " seconds for switch down to be discovered" )
2524 time.sleep( switchSleep )
2525 device = main.ONOScli1.getDevice( dpid=switchDPID )
2526 # Peek at the deleted switch
2527 main.log.warn( str( device ) )
2528 result = main.FALSE
2529 if device and device[ 'available' ] is False:
2530 result = main.TRUE
2531 utilities.assert_equals( expect=main.TRUE, actual=result,
2532 onpass="Kill switch successful",
2533 onfail="Failed to kill switch?" )
2534
2535 def CASE12( self, main ):
2536 """
2537 Switch Up
2538 """
2539 # NOTE: You should probably run a topology check after this
2540 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002541 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002542 assert main, "main not defined"
2543 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002544 assert main.CLIs, "main.CLIs not defined"
2545 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002546 assert ONOS1Port, "ONOS1Port not defined"
2547 assert ONOS2Port, "ONOS2Port not defined"
2548 assert ONOS3Port, "ONOS3Port not defined"
2549 assert ONOS4Port, "ONOS4Port not defined"
2550 assert ONOS5Port, "ONOS5Port not defined"
2551 assert ONOS6Port, "ONOS6Port not defined"
2552 assert ONOS7Port, "ONOS7Port not defined"
2553
2554 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2555 switch = main.params[ 'kill' ][ 'switch' ]
2556 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2557 links = main.params[ 'kill' ][ 'links' ].split()
2558 description = "Adding a switch to ensure it is discovered correctly"
2559 main.case( description )
2560
2561 main.step( "Add back " + switch )
2562 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2563 for peer in links:
2564 main.Mininet1.addLink( switch, peer )
2565 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002566 for i in range( main.numCtrls ):
2567 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002568 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2569 main.log.info( "Waiting " + str( switchSleep ) +
2570 " seconds for switch up to be discovered" )
2571 time.sleep( switchSleep )
2572 device = main.ONOScli1.getDevice( dpid=switchDPID )
2573 # Peek at the deleted switch
2574 main.log.warn( str( device ) )
2575 result = main.FALSE
2576 if device and device[ 'available' ]:
2577 result = main.TRUE
2578 utilities.assert_equals( expect=main.TRUE, actual=result,
2579 onpass="add switch successful",
2580 onfail="Failed to add switch?" )
2581
2582 def CASE13( self, main ):
2583 """
2584 Clean up
2585 """
2586 import os
2587 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002588 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002589 assert main, "main not defined"
2590 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002591 assert main.CLIs, "main.CLIs not defined"
2592 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002593
2594 # printing colors to terminal
2595 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2596 'blue': '\033[94m', 'green': '\033[92m',
2597 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2598 main.case( "Test Cleanup" )
2599 main.step( "Killing tcpdumps" )
2600 main.Mininet2.stopTcpdump()
2601
2602 testname = main.TEST
2603 if main.params[ 'BACKUP' ] == "True":
2604 main.step( "Copying MN pcap and ONOS log files to test station" )
2605 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2606 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
2607 # NOTE: MN Pcap file is being saved to ~/packet_captures
2608 # scp this file as MN and TestON aren't necessarily the same vm
2609 # FIXME: scp
2610 # mn files
2611 # TODO: Load these from params
2612 # NOTE: must end in /
2613 logFolder = "/opt/onos/log/"
2614 logFiles = [ "karaf.log", "karaf.log.1" ]
2615 # NOTE: must end in /
2616 dstDir = "~/packet_captures/"
2617 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002618 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002619 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2620 ":" + logFolder + f + " " +
2621 teststationUser + "@" +
2622 teststationIP + ":" +
2623 dstDir + str( testname ) +
2624 "-" + node.name + "-" + f )
2625 main.ONOSbench.handle.expect( "\$" )
2626
2627 # std*.log's
2628 # NOTE: must end in /
2629 logFolder = "/opt/onos/var/"
2630 logFiles = [ "stderr.log", "stdout.log" ]
2631 # NOTE: must end in /
2632 dstDir = "~/packet_captures/"
2633 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002634 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002635 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2636 ":" + logFolder + f + " " +
2637 teststationUser + "@" +
2638 teststationIP + ":" +
2639 dstDir + str( testname ) +
2640 "-" + node.name + "-" + f )
2641 main.ONOSbench.handle.expect( "\$" )
2642 # sleep so scp can finish
2643 time.sleep( 10 )
2644 main.step( "Packing and rotating pcap archives" )
2645 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
2646
2647 main.step( "Stopping Mininet" )
2648 mnResult = main.Mininet1.stopNet()
2649 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2650 onpass="Mininet stopped",
2651 onfail="MN cleanup NOT successful" )
2652
2653 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002654 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002655 print colors[ 'purple' ] + "Checking logs for errors on " + \
2656 node.name + ":" + colors[ 'end' ]
2657 print main.ONOSbench.checkLogs( node.ip_address )
2658
2659 try:
2660 timerLog = open( main.logdir + "/Timers.csv", 'w')
2661 # Overwrite with empty line and close
2662 labels = "Gossip Intents"
2663 data = str( gossipTime )
2664 timerLog.write( labels + "\n" + data )
2665 timerLog.close()
2666 except NameError, e:
2667 main.log.exception(e)
2668
2669 def CASE14( self, main ):
2670 """
2671 start election app on all onos nodes
2672 """
Jon Halle1a3b752015-07-22 13:02:46 -07002673 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002674 assert main, "main not defined"
2675 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002676 assert main.CLIs, "main.CLIs not defined"
2677 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002678
2679 main.case("Start Leadership Election app")
2680 main.step( "Install leadership election app" )
2681 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2682 utilities.assert_equals(
2683 expect=main.TRUE,
2684 actual=appResult,
2685 onpass="Election app installed",
2686 onfail="Something went wrong with installing Leadership election" )
2687
2688 main.step( "Run for election on each node" )
2689 leaderResult = main.TRUE
2690 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002691 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002692 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002693 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002694 leader = cli.electionTestLeader()
2695 if leader is None or leader == main.FALSE:
2696 main.log.error( cli.name + ": Leader for the election app " +
2697 "should be an ONOS node, instead got '" +
2698 str( leader ) + "'" )
2699 leaderResult = main.FALSE
2700 leaders.append( leader )
2701 utilities.assert_equals(
2702 expect=main.TRUE,
2703 actual=leaderResult,
2704 onpass="Successfully ran for leadership",
2705 onfail="Failed to run for leadership" )
2706
2707 main.step( "Check that each node shows the same leader" )
2708 sameLeader = main.TRUE
2709 if len( set( leaders ) ) != 1:
2710 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002711 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002712 str( leaders ) )
2713 utilities.assert_equals(
2714 expect=main.TRUE,
2715 actual=sameLeader,
2716 onpass="Leadership is consistent for the election topic",
2717 onfail="Nodes have different leaders" )
2718
2719 def CASE15( self, main ):
2720 """
2721 Check that Leadership Election is still functional
2722 """
2723 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002724 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002725 assert main, "main not defined"
2726 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002727 assert main.CLIs, "main.CLIs not defined"
2728 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002729
2730 leaderResult = main.TRUE
2731 description = "Check that Leadership Election is still functional"
2732 main.case( description )
2733
2734 main.step( "Check that each node shows the same leader" )
2735 sameLeader = main.TRUE
2736 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002737 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002738 leader = cli.electionTestLeader()
2739 leaders.append( leader )
2740 if len( set( leaders ) ) != 1:
2741 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002742 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002743 str( leaders ) )
2744 utilities.assert_equals(
2745 expect=main.TRUE,
2746 actual=sameLeader,
2747 onpass="Leadership is consistent for the election topic",
2748 onfail="Nodes have different leaders" )
2749
2750 main.step( "Find current leader and withdraw" )
2751 leader = main.ONOScli1.electionTestLeader()
2752 # do some sanity checking on leader before using it
2753 withdrawResult = main.FALSE
2754 if leader is None or leader == main.FALSE:
2755 main.log.error(
2756 "Leader for the election app should be an ONOS node," +
2757 "instead got '" + str( leader ) + "'" )
2758 leaderResult = main.FALSE
2759 oldLeader = None
Jon Halle1a3b752015-07-22 13:02:46 -07002760 for i in range( len( main.CLIs ) ):
2761 if leader == main.nodes[ i ].ip_address:
2762 oldLeader = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002763 break
2764 else: # FOR/ELSE statement
2765 main.log.error( "Leader election, could not find current leader" )
2766 if oldLeader:
2767 withdrawResult = oldLeader.electionTestWithdraw()
2768 utilities.assert_equals(
2769 expect=main.TRUE,
2770 actual=withdrawResult,
2771 onpass="Node was withdrawn from election",
2772 onfail="Node was not withdrawn from election" )
2773
2774 main.step( "Make sure new leader is elected" )
2775 # FIXME: use threads
2776 leaderList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002777 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002778 leaderN = cli.electionTestLeader()
2779 leaderList.append( leaderN )
2780 if leaderN == leader:
2781 main.log.error( cli.name + " still sees " + str( leader ) +
2782 " as leader after they withdrew" )
2783 leaderResult = main.FALSE
2784 elif leaderN == main.FALSE:
2785 # error in response
2786 # TODO: add check for "Command not found:" in the driver, this
2787 # means the app isn't loaded
2788 main.log.error( "Something is wrong with " +
2789 "electionTestLeader function, " +
2790 "check the error logs" )
2791 leaderResult = main.FALSE
2792 elif leaderN is None:
2793 # node may not have recieved the event yet
2794 time.sleep(7)
2795 leaderN = cli.electionTestLeader()
2796 leaderList.pop()
2797 leaderList.append( leaderN )
2798 consistentLeader = main.FALSE
2799 if len( set( leaderList ) ) == 1:
2800 main.log.info( "Each Election-app sees '" +
2801 str( leaderList[ 0 ] ) +
2802 "' as the leader" )
2803 consistentLeader = main.TRUE
2804 else:
2805 main.log.error(
2806 "Inconsistent responses for leader of Election-app:" )
2807 for n in range( len( leaderList ) ):
2808 main.log.error( "ONOS" + str( n + 1 ) + " response: " +
2809 str( leaderList[ n ] ) )
2810 leaderResult = leaderResult and consistentLeader
2811 utilities.assert_equals(
2812 expect=main.TRUE,
2813 actual=leaderResult,
2814 onpass="Leadership election passed",
2815 onfail="Something went wrong with Leadership election" )
2816
2817 main.step( "Run for election on old leader( just so everyone " +
2818 "is in the hat )" )
2819 if oldLeader:
2820 runResult = oldLeader.electionTestRun()
2821 else:
2822 runResult = main.FALSE
2823 utilities.assert_equals(
2824 expect=main.TRUE,
2825 actual=runResult,
2826 onpass="App re-ran for election",
2827 onfail="App failed to run for election" )
2828
2829 main.step( "Leader did not change when old leader re-ran" )
2830 afterRun = main.ONOScli1.electionTestLeader()
2831 # verify leader didn't just change
2832 if afterRun == leaderList[ 0 ]:
2833 afterResult = main.TRUE
2834 else:
2835 afterResult = main.FALSE
2836
2837 utilities.assert_equals(
2838 expect=main.TRUE,
2839 actual=afterResult,
2840 onpass="Old leader successfully re-ran for election",
2841 onfail="Something went wrong with Leadership election after " +
2842 "the old leader re-ran for election" )
2843
2844 def CASE16( self, main ):
2845 """
2846 Install Distributed Primitives app
2847 """
2848 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002849 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002850 assert main, "main not defined"
2851 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002852 assert main.CLIs, "main.CLIs not defined"
2853 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002854
2855 # Variables for the distributed primitives tests
2856 global pCounterName
2857 global iCounterName
2858 global pCounterValue
2859 global iCounterValue
2860 global onosSet
2861 global onosSetName
2862 pCounterName = "TestON-Partitions"
2863 iCounterName = "TestON-inMemory"
2864 pCounterValue = 0
2865 iCounterValue = 0
2866 onosSet = set([])
2867 onosSetName = "TestON-set"
2868
2869 description = "Install Primitives app"
2870 main.case( description )
2871 main.step( "Install Primitives app" )
2872 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002873 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002874 utilities.assert_equals( expect=main.TRUE,
2875 actual=appResults,
2876 onpass="Primitives app activated",
2877 onfail="Primitives app not activated" )
2878 time.sleep( 5 ) # To allow all nodes to activate
2879
2880 def CASE17( self, main ):
2881 """
2882 Check for basic functionality with distributed primitives
2883 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002884 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002885 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002886 assert main, "main not defined"
2887 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002888 assert main.CLIs, "main.CLIs not defined"
2889 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002890 assert pCounterName, "pCounterName not defined"
2891 assert iCounterName, "iCounterName not defined"
2892 assert onosSetName, "onosSetName not defined"
2893 # NOTE: assert fails if value is 0/None/Empty/False
2894 try:
2895 pCounterValue
2896 except NameError:
2897 main.log.error( "pCounterValue not defined, setting to 0" )
2898 pCounterValue = 0
2899 try:
2900 iCounterValue
2901 except NameError:
2902 main.log.error( "iCounterValue not defined, setting to 0" )
2903 iCounterValue = 0
2904 try:
2905 onosSet
2906 except NameError:
2907 main.log.error( "onosSet not defined, setting to empty Set" )
2908 onosSet = set([])
2909 # Variables for the distributed primitives tests. These are local only
2910 addValue = "a"
2911 addAllValue = "a b c d e f"
2912 retainValue = "c d e f"
2913
2914 description = "Check for basic functionality with distributed " +\
2915 "primitives"
2916 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07002917 main.caseExplanation = "Test the methods of the distributed " +\
2918 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07002919 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07002920 # Partitioned counters
2921 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002922 pCounters = []
2923 threads = []
2924 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07002925 for i in range( main.numCtrls ):
2926 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2927 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07002928 args=[ pCounterName ] )
2929 pCounterValue += 1
2930 addedPValues.append( pCounterValue )
2931 threads.append( t )
2932 t.start()
2933
2934 for t in threads:
2935 t.join()
2936 pCounters.append( t.result )
2937 # Check that counter incremented numController times
2938 pCounterResults = True
2939 for i in addedPValues:
2940 tmpResult = i in pCounters
2941 pCounterResults = pCounterResults and tmpResult
2942 if not tmpResult:
2943 main.log.error( str( i ) + " is not in partitioned "
2944 "counter incremented results" )
2945 utilities.assert_equals( expect=True,
2946 actual=pCounterResults,
2947 onpass="Default counter incremented",
2948 onfail="Error incrementing default" +
2949 " counter" )
2950
Jon Halle1a3b752015-07-22 13:02:46 -07002951 main.step( "Get then Increment a default counter on each node" )
2952 pCounters = []
2953 threads = []
2954 addedPValues = []
2955 for i in range( main.numCtrls ):
2956 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2957 name="counterGetAndAdd-" + str( i ),
2958 args=[ pCounterName ] )
2959 addedPValues.append( pCounterValue )
2960 pCounterValue += 1
2961 threads.append( t )
2962 t.start()
2963
2964 for t in threads:
2965 t.join()
2966 pCounters.append( t.result )
2967 # Check that counter incremented numController times
2968 pCounterResults = True
2969 for i in addedPValues:
2970 tmpResult = i in pCounters
2971 pCounterResults = pCounterResults and tmpResult
2972 if not tmpResult:
2973 main.log.error( str( i ) + " is not in partitioned "
2974 "counter incremented results" )
2975 utilities.assert_equals( expect=True,
2976 actual=pCounterResults,
2977 onpass="Default counter incremented",
2978 onfail="Error incrementing default" +
2979 " counter" )
2980
2981 main.step( "Counters we added have the correct values" )
2982 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
2983 utilities.assert_equals( expect=main.TRUE,
2984 actual=incrementCheck,
2985 onpass="Added counters are correct",
2986 onfail="Added counters are incorrect" )
2987
2988 main.step( "Add -8 to then get a default counter on each node" )
2989 pCounters = []
2990 threads = []
2991 addedPValues = []
2992 for i in range( main.numCtrls ):
2993 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2994 name="counterIncrement-" + str( i ),
2995 args=[ pCounterName ],
2996 kwargs={ "delta": -8 } )
2997 pCounterValue += -8
2998 addedPValues.append( pCounterValue )
2999 threads.append( t )
3000 t.start()
3001
3002 for t in threads:
3003 t.join()
3004 pCounters.append( t.result )
3005 # Check that counter incremented numController times
3006 pCounterResults = True
3007 for i in addedPValues:
3008 tmpResult = i in pCounters
3009 pCounterResults = pCounterResults and tmpResult
3010 if not tmpResult:
3011 main.log.error( str( i ) + " is not in partitioned "
3012 "counter incremented results" )
3013 utilities.assert_equals( expect=True,
3014 actual=pCounterResults,
3015 onpass="Default counter incremented",
3016 onfail="Error incrementing default" +
3017 " counter" )
3018
3019 main.step( "Add 5 to then get a default counter on each node" )
3020 pCounters = []
3021 threads = []
3022 addedPValues = []
3023 for i in range( main.numCtrls ):
3024 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3025 name="counterIncrement-" + str( i ),
3026 args=[ pCounterName ],
3027 kwargs={ "delta": 5 } )
3028 pCounterValue += 5
3029 addedPValues.append( pCounterValue )
3030 threads.append( t )
3031 t.start()
3032
3033 for t in threads:
3034 t.join()
3035 pCounters.append( t.result )
3036 # Check that counter incremented numController times
3037 pCounterResults = True
3038 for i in addedPValues:
3039 tmpResult = i in pCounters
3040 pCounterResults = pCounterResults and tmpResult
3041 if not tmpResult:
3042 main.log.error( str( i ) + " is not in partitioned "
3043 "counter incremented results" )
3044 utilities.assert_equals( expect=True,
3045 actual=pCounterResults,
3046 onpass="Default counter incremented",
3047 onfail="Error incrementing default" +
3048 " counter" )
3049
3050 main.step( "Get then add 5 to a default counter on each node" )
3051 pCounters = []
3052 threads = []
3053 addedPValues = []
3054 for i in range( main.numCtrls ):
3055 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3056 name="counterIncrement-" + str( i ),
3057 args=[ pCounterName ],
3058 kwargs={ "delta": 5 } )
3059 addedPValues.append( pCounterValue )
3060 pCounterValue += 5
3061 threads.append( t )
3062 t.start()
3063
3064 for t in threads:
3065 t.join()
3066 pCounters.append( t.result )
3067 # Check that counter incremented numController times
3068 pCounterResults = True
3069 for i in addedPValues:
3070 tmpResult = i in pCounters
3071 pCounterResults = pCounterResults and tmpResult
3072 if not tmpResult:
3073 main.log.error( str( i ) + " is not in partitioned "
3074 "counter incremented results" )
3075 utilities.assert_equals( expect=True,
3076 actual=pCounterResults,
3077 onpass="Default counter incremented",
3078 onfail="Error incrementing default" +
3079 " counter" )
3080
3081 main.step( "Counters we added have the correct values" )
3082 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3083 utilities.assert_equals( expect=main.TRUE,
3084 actual=incrementCheck,
3085 onpass="Added counters are correct",
3086 onfail="Added counters are incorrect" )
3087
3088 # In-Memory counters
3089 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003090 iCounters = []
3091 addedIValues = []
3092 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003093 for i in range( main.numCtrls ):
3094 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003095 name="icounterIncrement-" + str( i ),
3096 args=[ iCounterName ],
3097 kwargs={ "inMemory": True } )
3098 iCounterValue += 1
3099 addedIValues.append( iCounterValue )
3100 threads.append( t )
3101 t.start()
3102
3103 for t in threads:
3104 t.join()
3105 iCounters.append( t.result )
3106 # Check that counter incremented numController times
3107 iCounterResults = True
3108 for i in addedIValues:
3109 tmpResult = i in iCounters
3110 iCounterResults = iCounterResults and tmpResult
3111 if not tmpResult:
3112 main.log.error( str( i ) + " is not in the in-memory "
3113 "counter incremented results" )
3114 utilities.assert_equals( expect=True,
3115 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003116 onpass="In-memory counter incremented",
3117 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003118 " counter" )
3119
Jon Halle1a3b752015-07-22 13:02:46 -07003120 main.step( "Get then Increment a in-memory counter on each node" )
3121 iCounters = []
3122 threads = []
3123 addedIValues = []
3124 for i in range( main.numCtrls ):
3125 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3126 name="counterGetAndAdd-" + str( i ),
3127 args=[ iCounterName ],
3128 kwargs={ "inMemory": True } )
3129 addedIValues.append( iCounterValue )
3130 iCounterValue += 1
3131 threads.append( t )
3132 t.start()
3133
3134 for t in threads:
3135 t.join()
3136 iCounters.append( t.result )
3137 # Check that counter incremented numController times
3138 iCounterResults = True
3139 for i in addedIValues:
3140 tmpResult = i in iCounters
3141 iCounterResults = iCounterResults and tmpResult
3142 if not tmpResult:
3143 main.log.error( str( i ) + " is not in in-memory "
3144 "counter incremented results" )
3145 utilities.assert_equals( expect=True,
3146 actual=iCounterResults,
3147 onpass="In-memory counter incremented",
3148 onfail="Error incrementing in-memory" +
3149 " counter" )
3150
3151 main.step( "Counters we added have the correct values" )
3152 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3153 utilities.assert_equals( expect=main.TRUE,
3154 actual=incrementCheck,
3155 onpass="Added counters are correct",
3156 onfail="Added counters are incorrect" )
3157
3158 main.step( "Add -8 to then get a in-memory counter on each node" )
3159 iCounters = []
3160 threads = []
3161 addedIValues = []
3162 for i in range( main.numCtrls ):
3163 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3164 name="counterIncrement-" + str( i ),
3165 args=[ iCounterName ],
3166 kwargs={ "delta": -8, "inMemory": True } )
3167 iCounterValue += -8
3168 addedIValues.append( iCounterValue )
3169 threads.append( t )
3170 t.start()
3171
3172 for t in threads:
3173 t.join()
3174 iCounters.append( t.result )
3175 # Check that counter incremented numController times
3176 iCounterResults = True
3177 for i in addedIValues:
3178 tmpResult = i in iCounters
3179 iCounterResults = iCounterResults and tmpResult
3180 if not tmpResult:
3181 main.log.error( str( i ) + " is not in in-memory "
3182 "counter incremented results" )
3183 utilities.assert_equals( expect=True,
3184 actual=pCounterResults,
3185 onpass="In-memory counter incremented",
3186 onfail="Error incrementing in-memory" +
3187 " counter" )
3188
3189 main.step( "Add 5 to then get a in-memory counter on each node" )
3190 iCounters = []
3191 threads = []
3192 addedIValues = []
3193 for i in range( main.numCtrls ):
3194 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3195 name="counterIncrement-" + str( i ),
3196 args=[ iCounterName ],
3197 kwargs={ "delta": 5, "inMemory": True } )
3198 iCounterValue += 5
3199 addedIValues.append( iCounterValue )
3200 threads.append( t )
3201 t.start()
3202
3203 for t in threads:
3204 t.join()
3205 iCounters.append( t.result )
3206 # Check that counter incremented numController times
3207 iCounterResults = True
3208 for i in addedIValues:
3209 tmpResult = i in iCounters
3210 iCounterResults = iCounterResults and tmpResult
3211 if not tmpResult:
3212 main.log.error( str( i ) + " is not in in-memory "
3213 "counter incremented results" )
3214 utilities.assert_equals( expect=True,
3215 actual=pCounterResults,
3216 onpass="In-memory counter incremented",
3217 onfail="Error incrementing in-memory" +
3218 " counter" )
3219
3220 main.step( "Get then add 5 to a in-memory counter on each node" )
3221 iCounters = []
3222 threads = []
3223 addedIValues = []
3224 for i in range( main.numCtrls ):
3225 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3226 name="counterIncrement-" + str( i ),
3227 args=[ iCounterName ],
3228 kwargs={ "delta": 5, "inMemory": True } )
3229 addedIValues.append( iCounterValue )
3230 iCounterValue += 5
3231 threads.append( t )
3232 t.start()
3233
3234 for t in threads:
3235 t.join()
3236 iCounters.append( t.result )
3237 # Check that counter incremented numController times
3238 iCounterResults = True
3239 for i in addedIValues:
3240 tmpResult = i in iCounters
3241 iCounterResults = iCounterResults and tmpResult
3242 if not tmpResult:
3243 main.log.error( str( i ) + " is not in in-memory "
3244 "counter incremented results" )
3245 utilities.assert_equals( expect=True,
3246 actual=iCounterResults,
3247 onpass="In-memory counter incremented",
3248 onfail="Error incrementing in-memory" +
3249 " counter" )
3250
3251 main.step( "Counters we added have the correct values" )
3252 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3253 utilities.assert_equals( expect=main.TRUE,
3254 actual=incrementCheck,
3255 onpass="Added counters are correct",
3256 onfail="Added counters are incorrect" )
3257
Jon Hall5cf14d52015-07-16 12:15:19 -07003258 main.step( "Check counters are consistant across nodes" )
3259 onosCounters = []
3260 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003261 for i in range( main.numCtrls ):
3262 t = main.Thread( target=main.CLIs[i].counters,
Jon Hall5cf14d52015-07-16 12:15:19 -07003263 name="counters-" + str( i ) )
3264 threads.append( t )
3265 t.start()
3266 for t in threads:
3267 t.join()
3268 onosCounters.append( t.result )
3269 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ]
3270 if all( tmp ):
3271 main.log.info( "Counters are consistent across all nodes" )
3272 consistentCounterResults = main.TRUE
3273 else:
3274 main.log.error( "Counters are not consistent across all nodes" )
3275 consistentCounterResults = main.FALSE
3276 utilities.assert_equals( expect=main.TRUE,
3277 actual=consistentCounterResults,
3278 onpass="ONOS counters are consistent " +
3279 "across nodes",
3280 onfail="ONOS Counters are inconsistent " +
3281 "across nodes" )
3282
3283 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003284 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3285 incrementCheck = incrementCheck and \
3286 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003287 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003288 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003289 onpass="Added counters are correct",
3290 onfail="Added counters are incorrect" )
Jon Halle1a3b752015-07-22 13:02:46 -07003291
Jon Hall5cf14d52015-07-16 12:15:19 -07003292 # DISTRIBUTED SETS
3293 main.step( "Distributed Set get" )
3294 size = len( onosSet )
3295 getResponses = []
3296 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003297 for i in range( main.numCtrls ):
3298 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003299 name="setTestGet-" + str( i ),
3300 args=[ onosSetName ] )
3301 threads.append( t )
3302 t.start()
3303 for t in threads:
3304 t.join()
3305 getResponses.append( t.result )
3306
3307 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003308 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003309 if isinstance( getResponses[ i ], list):
3310 current = set( getResponses[ i ] )
3311 if len( current ) == len( getResponses[ i ] ):
3312 # no repeats
3313 if onosSet != current:
3314 main.log.error( "ONOS" + str( i + 1 ) +
3315 " has incorrect view" +
3316 " of set " + onosSetName + ":\n" +
3317 str( getResponses[ i ] ) )
3318 main.log.debug( "Expected: " + str( onosSet ) )
3319 main.log.debug( "Actual: " + str( current ) )
3320 getResults = main.FALSE
3321 else:
3322 # error, set is not a set
3323 main.log.error( "ONOS" + str( i + 1 ) +
3324 " has repeat elements in" +
3325 " set " + onosSetName + ":\n" +
3326 str( getResponses[ i ] ) )
3327 getResults = main.FALSE
3328 elif getResponses[ i ] == main.ERROR:
3329 getResults = main.FALSE
3330 utilities.assert_equals( expect=main.TRUE,
3331 actual=getResults,
3332 onpass="Set elements are correct",
3333 onfail="Set elements are incorrect" )
3334
3335 main.step( "Distributed Set size" )
3336 sizeResponses = []
3337 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003338 for i in range( main.numCtrls ):
3339 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003340 name="setTestSize-" + str( i ),
3341 args=[ onosSetName ] )
3342 threads.append( t )
3343 t.start()
3344 for t in threads:
3345 t.join()
3346 sizeResponses.append( t.result )
3347
3348 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003349 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003350 if size != sizeResponses[ i ]:
3351 sizeResults = main.FALSE
3352 main.log.error( "ONOS" + str( i + 1 ) +
3353 " expected a size of " + str( size ) +
3354 " for set " + onosSetName +
3355 " but got " + str( sizeResponses[ i ] ) )
3356 utilities.assert_equals( expect=main.TRUE,
3357 actual=sizeResults,
3358 onpass="Set sizes are correct",
3359 onfail="Set sizes are incorrect" )
3360
3361 main.step( "Distributed Set add()" )
3362 onosSet.add( addValue )
3363 addResponses = []
3364 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003365 for i in range( main.numCtrls ):
3366 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003367 name="setTestAdd-" + str( i ),
3368 args=[ onosSetName, addValue ] )
3369 threads.append( t )
3370 t.start()
3371 for t in threads:
3372 t.join()
3373 addResponses.append( t.result )
3374
3375 # main.TRUE = successfully changed the set
3376 # main.FALSE = action resulted in no change in set
3377 # main.ERROR - Some error in executing the function
3378 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003379 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003380 if addResponses[ i ] == main.TRUE:
3381 # All is well
3382 pass
3383 elif addResponses[ i ] == main.FALSE:
3384 # Already in set, probably fine
3385 pass
3386 elif addResponses[ i ] == main.ERROR:
3387 # Error in execution
3388 addResults = main.FALSE
3389 else:
3390 # unexpected result
3391 addResults = main.FALSE
3392 if addResults != main.TRUE:
3393 main.log.error( "Error executing set add" )
3394
3395 # Check if set is still correct
3396 size = len( onosSet )
3397 getResponses = []
3398 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003399 for i in range( main.numCtrls ):
3400 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003401 name="setTestGet-" + str( i ),
3402 args=[ onosSetName ] )
3403 threads.append( t )
3404 t.start()
3405 for t in threads:
3406 t.join()
3407 getResponses.append( t.result )
3408 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003409 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003410 if isinstance( getResponses[ i ], list):
3411 current = set( getResponses[ i ] )
3412 if len( current ) == len( getResponses[ i ] ):
3413 # no repeats
3414 if onosSet != current:
3415 main.log.error( "ONOS" + str( i + 1 ) +
3416 " has incorrect view" +
3417 " of set " + onosSetName + ":\n" +
3418 str( getResponses[ i ] ) )
3419 main.log.debug( "Expected: " + str( onosSet ) )
3420 main.log.debug( "Actual: " + str( current ) )
3421 getResults = main.FALSE
3422 else:
3423 # error, set is not a set
3424 main.log.error( "ONOS" + str( i + 1 ) +
3425 " has repeat elements in" +
3426 " set " + onosSetName + ":\n" +
3427 str( getResponses[ i ] ) )
3428 getResults = main.FALSE
3429 elif getResponses[ i ] == main.ERROR:
3430 getResults = main.FALSE
3431 sizeResponses = []
3432 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003433 for i in range( main.numCtrls ):
3434 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003435 name="setTestSize-" + str( i ),
3436 args=[ onosSetName ] )
3437 threads.append( t )
3438 t.start()
3439 for t in threads:
3440 t.join()
3441 sizeResponses.append( t.result )
3442 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003443 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003444 if size != sizeResponses[ i ]:
3445 sizeResults = main.FALSE
3446 main.log.error( "ONOS" + str( i + 1 ) +
3447 " expected a size of " + str( size ) +
3448 " for set " + onosSetName +
3449 " but got " + str( sizeResponses[ i ] ) )
3450 addResults = addResults and getResults and sizeResults
3451 utilities.assert_equals( expect=main.TRUE,
3452 actual=addResults,
3453 onpass="Set add correct",
3454 onfail="Set add was incorrect" )
3455
3456 main.step( "Distributed Set addAll()" )
3457 onosSet.update( addAllValue.split() )
3458 addResponses = []
3459 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003460 for i in range( main.numCtrls ):
3461 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003462 name="setTestAddAll-" + str( i ),
3463 args=[ onosSetName, addAllValue ] )
3464 threads.append( t )
3465 t.start()
3466 for t in threads:
3467 t.join()
3468 addResponses.append( t.result )
3469
3470 # main.TRUE = successfully changed the set
3471 # main.FALSE = action resulted in no change in set
3472 # main.ERROR - Some error in executing the function
3473 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003474 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003475 if addResponses[ i ] == main.TRUE:
3476 # All is well
3477 pass
3478 elif addResponses[ i ] == main.FALSE:
3479 # Already in set, probably fine
3480 pass
3481 elif addResponses[ i ] == main.ERROR:
3482 # Error in execution
3483 addAllResults = main.FALSE
3484 else:
3485 # unexpected result
3486 addAllResults = main.FALSE
3487 if addAllResults != main.TRUE:
3488 main.log.error( "Error executing set addAll" )
3489
3490 # Check if set is still correct
3491 size = len( onosSet )
3492 getResponses = []
3493 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003494 for i in range( main.numCtrls ):
3495 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003496 name="setTestGet-" + str( i ),
3497 args=[ onosSetName ] )
3498 threads.append( t )
3499 t.start()
3500 for t in threads:
3501 t.join()
3502 getResponses.append( t.result )
3503 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003504 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003505 if isinstance( getResponses[ i ], list):
3506 current = set( getResponses[ i ] )
3507 if len( current ) == len( getResponses[ i ] ):
3508 # no repeats
3509 if onosSet != current:
3510 main.log.error( "ONOS" + str( i + 1 ) +
3511 " has incorrect view" +
3512 " of set " + onosSetName + ":\n" +
3513 str( getResponses[ i ] ) )
3514 main.log.debug( "Expected: " + str( onosSet ) )
3515 main.log.debug( "Actual: " + str( current ) )
3516 getResults = main.FALSE
3517 else:
3518 # error, set is not a set
3519 main.log.error( "ONOS" + str( i + 1 ) +
3520 " has repeat elements in" +
3521 " set " + onosSetName + ":\n" +
3522 str( getResponses[ i ] ) )
3523 getResults = main.FALSE
3524 elif getResponses[ i ] == main.ERROR:
3525 getResults = main.FALSE
3526 sizeResponses = []
3527 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003528 for i in range( main.numCtrls ):
3529 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003530 name="setTestSize-" + str( i ),
3531 args=[ onosSetName ] )
3532 threads.append( t )
3533 t.start()
3534 for t in threads:
3535 t.join()
3536 sizeResponses.append( t.result )
3537 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003538 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003539 if size != sizeResponses[ i ]:
3540 sizeResults = main.FALSE
3541 main.log.error( "ONOS" + str( i + 1 ) +
3542 " expected a size of " + str( size ) +
3543 " for set " + onosSetName +
3544 " but got " + str( sizeResponses[ i ] ) )
3545 addAllResults = addAllResults and getResults and sizeResults
3546 utilities.assert_equals( expect=main.TRUE,
3547 actual=addAllResults,
3548 onpass="Set addAll correct",
3549 onfail="Set addAll was incorrect" )
3550
3551 main.step( "Distributed Set contains()" )
3552 containsResponses = []
3553 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003554 for i in range( main.numCtrls ):
3555 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003556 name="setContains-" + str( i ),
3557 args=[ onosSetName ],
3558 kwargs={ "values": addValue } )
3559 threads.append( t )
3560 t.start()
3561 for t in threads:
3562 t.join()
3563 # NOTE: This is the tuple
3564 containsResponses.append( t.result )
3565
3566 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003567 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003568 if containsResponses[ i ] == main.ERROR:
3569 containsResults = main.FALSE
3570 else:
3571 containsResults = containsResults and\
3572 containsResponses[ i ][ 1 ]
3573 utilities.assert_equals( expect=main.TRUE,
3574 actual=containsResults,
3575 onpass="Set contains is functional",
3576 onfail="Set contains failed" )
3577
3578 main.step( "Distributed Set containsAll()" )
3579 containsAllResponses = []
3580 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003581 for i in range( main.numCtrls ):
3582 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003583 name="setContainsAll-" + str( i ),
3584 args=[ onosSetName ],
3585 kwargs={ "values": addAllValue } )
3586 threads.append( t )
3587 t.start()
3588 for t in threads:
3589 t.join()
3590 # NOTE: This is the tuple
3591 containsAllResponses.append( t.result )
3592
3593 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003594 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003595 if containsResponses[ i ] == main.ERROR:
3596 containsResults = main.FALSE
3597 else:
3598 containsResults = containsResults and\
3599 containsResponses[ i ][ 1 ]
3600 utilities.assert_equals( expect=main.TRUE,
3601 actual=containsAllResults,
3602 onpass="Set containsAll is functional",
3603 onfail="Set containsAll failed" )
3604
3605 main.step( "Distributed Set remove()" )
3606 onosSet.remove( addValue )
3607 removeResponses = []
3608 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003609 for i in range( main.numCtrls ):
3610 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003611 name="setTestRemove-" + str( i ),
3612 args=[ onosSetName, addValue ] )
3613 threads.append( t )
3614 t.start()
3615 for t in threads:
3616 t.join()
3617 removeResponses.append( t.result )
3618
3619 # main.TRUE = successfully changed the set
3620 # main.FALSE = action resulted in no change in set
3621 # main.ERROR - Some error in executing the function
3622 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003623 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003624 if removeResponses[ i ] == main.TRUE:
3625 # All is well
3626 pass
3627 elif removeResponses[ i ] == main.FALSE:
3628 # not in set, probably fine
3629 pass
3630 elif removeResponses[ i ] == main.ERROR:
3631 # Error in execution
3632 removeResults = main.FALSE
3633 else:
3634 # unexpected result
3635 removeResults = main.FALSE
3636 if removeResults != main.TRUE:
3637 main.log.error( "Error executing set remove" )
3638
3639 # Check if set is still correct
3640 size = len( onosSet )
3641 getResponses = []
3642 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003643 for i in range( main.numCtrls ):
3644 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003645 name="setTestGet-" + str( i ),
3646 args=[ onosSetName ] )
3647 threads.append( t )
3648 t.start()
3649 for t in threads:
3650 t.join()
3651 getResponses.append( t.result )
3652 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003653 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003654 if isinstance( getResponses[ i ], list):
3655 current = set( getResponses[ i ] )
3656 if len( current ) == len( getResponses[ i ] ):
3657 # no repeats
3658 if onosSet != current:
3659 main.log.error( "ONOS" + str( i + 1 ) +
3660 " has incorrect view" +
3661 " of set " + onosSetName + ":\n" +
3662 str( getResponses[ i ] ) )
3663 main.log.debug( "Expected: " + str( onosSet ) )
3664 main.log.debug( "Actual: " + str( current ) )
3665 getResults = main.FALSE
3666 else:
3667 # error, set is not a set
3668 main.log.error( "ONOS" + str( i + 1 ) +
3669 " has repeat elements in" +
3670 " set " + onosSetName + ":\n" +
3671 str( getResponses[ i ] ) )
3672 getResults = main.FALSE
3673 elif getResponses[ i ] == main.ERROR:
3674 getResults = main.FALSE
3675 sizeResponses = []
3676 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003677 for i in range( main.numCtrls ):
3678 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003679 name="setTestSize-" + str( i ),
3680 args=[ onosSetName ] )
3681 threads.append( t )
3682 t.start()
3683 for t in threads:
3684 t.join()
3685 sizeResponses.append( t.result )
3686 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003687 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003688 if size != sizeResponses[ i ]:
3689 sizeResults = main.FALSE
3690 main.log.error( "ONOS" + str( i + 1 ) +
3691 " expected a size of " + str( size ) +
3692 " for set " + onosSetName +
3693 " but got " + str( sizeResponses[ i ] ) )
3694 removeResults = removeResults and getResults and sizeResults
3695 utilities.assert_equals( expect=main.TRUE,
3696 actual=removeResults,
3697 onpass="Set remove correct",
3698 onfail="Set remove was incorrect" )
3699
3700 main.step( "Distributed Set removeAll()" )
3701 onosSet.difference_update( addAllValue.split() )
3702 removeAllResponses = []
3703 threads = []
3704 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003705 for i in range( main.numCtrls ):
3706 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003707 name="setTestRemoveAll-" + str( i ),
3708 args=[ onosSetName, addAllValue ] )
3709 threads.append( t )
3710 t.start()
3711 for t in threads:
3712 t.join()
3713 removeAllResponses.append( t.result )
3714 except Exception, e:
3715 main.log.exception(e)
3716
3717 # main.TRUE = successfully changed the set
3718 # main.FALSE = action resulted in no change in set
3719 # main.ERROR - Some error in executing the function
3720 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003721 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003722 if removeAllResponses[ i ] == main.TRUE:
3723 # All is well
3724 pass
3725 elif removeAllResponses[ i ] == main.FALSE:
3726 # not in set, probably fine
3727 pass
3728 elif removeAllResponses[ i ] == main.ERROR:
3729 # Error in execution
3730 removeAllResults = main.FALSE
3731 else:
3732 # unexpected result
3733 removeAllResults = main.FALSE
3734 if removeAllResults != main.TRUE:
3735 main.log.error( "Error executing set removeAll" )
3736
3737 # Check if set is still correct
3738 size = len( onosSet )
3739 getResponses = []
3740 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003741 for i in range( main.numCtrls ):
3742 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003743 name="setTestGet-" + str( i ),
3744 args=[ onosSetName ] )
3745 threads.append( t )
3746 t.start()
3747 for t in threads:
3748 t.join()
3749 getResponses.append( t.result )
3750 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003751 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003752 if isinstance( getResponses[ i ], list):
3753 current = set( getResponses[ i ] )
3754 if len( current ) == len( getResponses[ i ] ):
3755 # no repeats
3756 if onosSet != current:
3757 main.log.error( "ONOS" + str( i + 1 ) +
3758 " has incorrect view" +
3759 " of set " + onosSetName + ":\n" +
3760 str( getResponses[ i ] ) )
3761 main.log.debug( "Expected: " + str( onosSet ) )
3762 main.log.debug( "Actual: " + str( current ) )
3763 getResults = main.FALSE
3764 else:
3765 # error, set is not a set
3766 main.log.error( "ONOS" + str( i + 1 ) +
3767 " has repeat elements in" +
3768 " set " + onosSetName + ":\n" +
3769 str( getResponses[ i ] ) )
3770 getResults = main.FALSE
3771 elif getResponses[ i ] == main.ERROR:
3772 getResults = main.FALSE
3773 sizeResponses = []
3774 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003775 for i in range( main.numCtrls ):
3776 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003777 name="setTestSize-" + str( i ),
3778 args=[ onosSetName ] )
3779 threads.append( t )
3780 t.start()
3781 for t in threads:
3782 t.join()
3783 sizeResponses.append( t.result )
3784 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003785 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003786 if size != sizeResponses[ i ]:
3787 sizeResults = main.FALSE
3788 main.log.error( "ONOS" + str( i + 1 ) +
3789 " expected a size of " + str( size ) +
3790 " for set " + onosSetName +
3791 " but got " + str( sizeResponses[ i ] ) )
3792 removeAllResults = removeAllResults and getResults and sizeResults
3793 utilities.assert_equals( expect=main.TRUE,
3794 actual=removeAllResults,
3795 onpass="Set removeAll correct",
3796 onfail="Set removeAll was incorrect" )
3797
3798 main.step( "Distributed Set addAll()" )
3799 onosSet.update( addAllValue.split() )
3800 addResponses = []
3801 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003802 for i in range( main.numCtrls ):
3803 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003804 name="setTestAddAll-" + str( i ),
3805 args=[ onosSetName, addAllValue ] )
3806 threads.append( t )
3807 t.start()
3808 for t in threads:
3809 t.join()
3810 addResponses.append( t.result )
3811
3812 # main.TRUE = successfully changed the set
3813 # main.FALSE = action resulted in no change in set
3814 # main.ERROR - Some error in executing the function
3815 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003816 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003817 if addResponses[ i ] == main.TRUE:
3818 # All is well
3819 pass
3820 elif addResponses[ i ] == main.FALSE:
3821 # Already in set, probably fine
3822 pass
3823 elif addResponses[ i ] == main.ERROR:
3824 # Error in execution
3825 addAllResults = main.FALSE
3826 else:
3827 # unexpected result
3828 addAllResults = main.FALSE
3829 if addAllResults != main.TRUE:
3830 main.log.error( "Error executing set addAll" )
3831
3832 # Check if set is still correct
3833 size = len( onosSet )
3834 getResponses = []
3835 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003836 for i in range( main.numCtrls ):
3837 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003838 name="setTestGet-" + str( i ),
3839 args=[ onosSetName ] )
3840 threads.append( t )
3841 t.start()
3842 for t in threads:
3843 t.join()
3844 getResponses.append( t.result )
3845 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003846 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003847 if isinstance( getResponses[ i ], list):
3848 current = set( getResponses[ i ] )
3849 if len( current ) == len( getResponses[ i ] ):
3850 # no repeats
3851 if onosSet != current:
3852 main.log.error( "ONOS" + str( i + 1 ) +
3853 " has incorrect view" +
3854 " of set " + onosSetName + ":\n" +
3855 str( getResponses[ i ] ) )
3856 main.log.debug( "Expected: " + str( onosSet ) )
3857 main.log.debug( "Actual: " + str( current ) )
3858 getResults = main.FALSE
3859 else:
3860 # error, set is not a set
3861 main.log.error( "ONOS" + str( i + 1 ) +
3862 " has repeat elements in" +
3863 " set " + onosSetName + ":\n" +
3864 str( getResponses[ i ] ) )
3865 getResults = main.FALSE
3866 elif getResponses[ i ] == main.ERROR:
3867 getResults = main.FALSE
3868 sizeResponses = []
3869 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003870 for i in range( main.numCtrls ):
3871 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003872 name="setTestSize-" + str( i ),
3873 args=[ onosSetName ] )
3874 threads.append( t )
3875 t.start()
3876 for t in threads:
3877 t.join()
3878 sizeResponses.append( t.result )
3879 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003880 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003881 if size != sizeResponses[ i ]:
3882 sizeResults = main.FALSE
3883 main.log.error( "ONOS" + str( i + 1 ) +
3884 " expected a size of " + str( size ) +
3885 " for set " + onosSetName +
3886 " but got " + str( sizeResponses[ i ] ) )
3887 addAllResults = addAllResults and getResults and sizeResults
3888 utilities.assert_equals( expect=main.TRUE,
3889 actual=addAllResults,
3890 onpass="Set addAll correct",
3891 onfail="Set addAll was incorrect" )
3892
3893 main.step( "Distributed Set clear()" )
3894 onosSet.clear()
3895 clearResponses = []
3896 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003897 for i in range( main.numCtrls ):
3898 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003899 name="setTestClear-" + str( i ),
3900 args=[ onosSetName, " "], # Values doesn't matter
3901 kwargs={ "clear": True } )
3902 threads.append( t )
3903 t.start()
3904 for t in threads:
3905 t.join()
3906 clearResponses.append( t.result )
3907
3908 # main.TRUE = successfully changed the set
3909 # main.FALSE = action resulted in no change in set
3910 # main.ERROR - Some error in executing the function
3911 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003912 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003913 if clearResponses[ i ] == main.TRUE:
3914 # All is well
3915 pass
3916 elif clearResponses[ i ] == main.FALSE:
3917 # Nothing set, probably fine
3918 pass
3919 elif clearResponses[ i ] == main.ERROR:
3920 # Error in execution
3921 clearResults = main.FALSE
3922 else:
3923 # unexpected result
3924 clearResults = main.FALSE
3925 if clearResults != main.TRUE:
3926 main.log.error( "Error executing set clear" )
3927
3928 # Check if set is still correct
3929 size = len( onosSet )
3930 getResponses = []
3931 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003932 for i in range( main.numCtrls ):
3933 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003934 name="setTestGet-" + str( i ),
3935 args=[ onosSetName ] )
3936 threads.append( t )
3937 t.start()
3938 for t in threads:
3939 t.join()
3940 getResponses.append( t.result )
3941 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003942 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003943 if isinstance( getResponses[ i ], list):
3944 current = set( getResponses[ i ] )
3945 if len( current ) == len( getResponses[ i ] ):
3946 # no repeats
3947 if onosSet != current:
3948 main.log.error( "ONOS" + str( i + 1 ) +
3949 " has incorrect view" +
3950 " of set " + onosSetName + ":\n" +
3951 str( getResponses[ i ] ) )
3952 main.log.debug( "Expected: " + str( onosSet ) )
3953 main.log.debug( "Actual: " + str( current ) )
3954 getResults = main.FALSE
3955 else:
3956 # error, set is not a set
3957 main.log.error( "ONOS" + str( i + 1 ) +
3958 " has repeat elements in" +
3959 " set " + onosSetName + ":\n" +
3960 str( getResponses[ i ] ) )
3961 getResults = main.FALSE
3962 elif getResponses[ i ] == main.ERROR:
3963 getResults = main.FALSE
3964 sizeResponses = []
3965 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003966 for i in range( main.numCtrls ):
3967 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003968 name="setTestSize-" + str( i ),
3969 args=[ onosSetName ] )
3970 threads.append( t )
3971 t.start()
3972 for t in threads:
3973 t.join()
3974 sizeResponses.append( t.result )
3975 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003976 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003977 if size != sizeResponses[ i ]:
3978 sizeResults = main.FALSE
3979 main.log.error( "ONOS" + str( i + 1 ) +
3980 " expected a size of " + str( size ) +
3981 " for set " + onosSetName +
3982 " but got " + str( sizeResponses[ i ] ) )
3983 clearResults = clearResults and getResults and sizeResults
3984 utilities.assert_equals( expect=main.TRUE,
3985 actual=clearResults,
3986 onpass="Set clear correct",
3987 onfail="Set clear was incorrect" )
3988
3989 main.step( "Distributed Set addAll()" )
3990 onosSet.update( addAllValue.split() )
3991 addResponses = []
3992 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003993 for i in range( main.numCtrls ):
3994 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003995 name="setTestAddAll-" + str( i ),
3996 args=[ onosSetName, addAllValue ] )
3997 threads.append( t )
3998 t.start()
3999 for t in threads:
4000 t.join()
4001 addResponses.append( t.result )
4002
4003 # main.TRUE = successfully changed the set
4004 # main.FALSE = action resulted in no change in set
4005 # main.ERROR - Some error in executing the function
4006 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004007 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004008 if addResponses[ i ] == main.TRUE:
4009 # All is well
4010 pass
4011 elif addResponses[ i ] == main.FALSE:
4012 # Already in set, probably fine
4013 pass
4014 elif addResponses[ i ] == main.ERROR:
4015 # Error in execution
4016 addAllResults = main.FALSE
4017 else:
4018 # unexpected result
4019 addAllResults = main.FALSE
4020 if addAllResults != main.TRUE:
4021 main.log.error( "Error executing set addAll" )
4022
4023 # Check if set is still correct
4024 size = len( onosSet )
4025 getResponses = []
4026 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004027 for i in range( main.numCtrls ):
4028 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004029 name="setTestGet-" + str( i ),
4030 args=[ onosSetName ] )
4031 threads.append( t )
4032 t.start()
4033 for t in threads:
4034 t.join()
4035 getResponses.append( t.result )
4036 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004037 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004038 if isinstance( getResponses[ i ], list):
4039 current = set( getResponses[ i ] )
4040 if len( current ) == len( getResponses[ i ] ):
4041 # no repeats
4042 if onosSet != current:
4043 main.log.error( "ONOS" + str( i + 1 ) +
4044 " has incorrect view" +
4045 " of set " + onosSetName + ":\n" +
4046 str( getResponses[ i ] ) )
4047 main.log.debug( "Expected: " + str( onosSet ) )
4048 main.log.debug( "Actual: " + str( current ) )
4049 getResults = main.FALSE
4050 else:
4051 # error, set is not a set
4052 main.log.error( "ONOS" + str( i + 1 ) +
4053 " has repeat elements in" +
4054 " set " + onosSetName + ":\n" +
4055 str( getResponses[ i ] ) )
4056 getResults = main.FALSE
4057 elif getResponses[ i ] == main.ERROR:
4058 getResults = main.FALSE
4059 sizeResponses = []
4060 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004061 for i in range( main.numCtrls ):
4062 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004063 name="setTestSize-" + str( i ),
4064 args=[ onosSetName ] )
4065 threads.append( t )
4066 t.start()
4067 for t in threads:
4068 t.join()
4069 sizeResponses.append( t.result )
4070 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004071 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004072 if size != sizeResponses[ i ]:
4073 sizeResults = main.FALSE
4074 main.log.error( "ONOS" + str( i + 1 ) +
4075 " expected a size of " + str( size ) +
4076 " for set " + onosSetName +
4077 " but got " + str( sizeResponses[ i ] ) )
4078 addAllResults = addAllResults and getResults and sizeResults
4079 utilities.assert_equals( expect=main.TRUE,
4080 actual=addAllResults,
4081 onpass="Set addAll correct",
4082 onfail="Set addAll was incorrect" )
4083
4084 main.step( "Distributed Set retain()" )
4085 onosSet.intersection_update( retainValue.split() )
4086 retainResponses = []
4087 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004088 for i in range( main.numCtrls ):
4089 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004090 name="setTestRetain-" + str( i ),
4091 args=[ onosSetName, retainValue ],
4092 kwargs={ "retain": True } )
4093 threads.append( t )
4094 t.start()
4095 for t in threads:
4096 t.join()
4097 retainResponses.append( t.result )
4098
4099 # main.TRUE = successfully changed the set
4100 # main.FALSE = action resulted in no change in set
4101 # main.ERROR - Some error in executing the function
4102 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004103 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004104 if retainResponses[ i ] == main.TRUE:
4105 # All is well
4106 pass
4107 elif retainResponses[ i ] == main.FALSE:
4108 # Already in set, probably fine
4109 pass
4110 elif retainResponses[ i ] == main.ERROR:
4111 # Error in execution
4112 retainResults = main.FALSE
4113 else:
4114 # unexpected result
4115 retainResults = main.FALSE
4116 if retainResults != main.TRUE:
4117 main.log.error( "Error executing set retain" )
4118
4119 # Check if set is still correct
4120 size = len( onosSet )
4121 getResponses = []
4122 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004123 for i in range( main.numCtrls ):
4124 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004125 name="setTestGet-" + str( i ),
4126 args=[ onosSetName ] )
4127 threads.append( t )
4128 t.start()
4129 for t in threads:
4130 t.join()
4131 getResponses.append( t.result )
4132 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004133 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004134 if isinstance( getResponses[ i ], list):
4135 current = set( getResponses[ i ] )
4136 if len( current ) == len( getResponses[ i ] ):
4137 # no repeats
4138 if onosSet != current:
4139 main.log.error( "ONOS" + str( i + 1 ) +
4140 " has incorrect view" +
4141 " of set " + onosSetName + ":\n" +
4142 str( getResponses[ i ] ) )
4143 main.log.debug( "Expected: " + str( onosSet ) )
4144 main.log.debug( "Actual: " + str( current ) )
4145 getResults = main.FALSE
4146 else:
4147 # error, set is not a set
4148 main.log.error( "ONOS" + str( i + 1 ) +
4149 " has repeat elements in" +
4150 " set " + onosSetName + ":\n" +
4151 str( getResponses[ i ] ) )
4152 getResults = main.FALSE
4153 elif getResponses[ i ] == main.ERROR:
4154 getResults = main.FALSE
4155 sizeResponses = []
4156 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004157 for i in range( main.numCtrls ):
4158 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004159 name="setTestSize-" + str( i ),
4160 args=[ onosSetName ] )
4161 threads.append( t )
4162 t.start()
4163 for t in threads:
4164 t.join()
4165 sizeResponses.append( t.result )
4166 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004167 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004168 if size != sizeResponses[ i ]:
4169 sizeResults = main.FALSE
4170 main.log.error( "ONOS" + str( i + 1 ) +
4171 " expected a size of " +
4172 str( size ) + " for set " + onosSetName +
4173 " but got " + str( sizeResponses[ i ] ) )
4174 retainResults = retainResults and getResults and sizeResults
4175 utilities.assert_equals( expect=main.TRUE,
4176 actual=retainResults,
4177 onpass="Set retain correct",
4178 onfail="Set retain was incorrect" )
4179