blob: ed37e33a765bc694320826894da22792c18f922c [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
28class HAsanity:
29
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080052 import time
Jon Hall5cf14d52015-07-16 12:15:19 -070053 main.log.info( "ONOS HA Sanity test - initialization" )
54 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070055 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "installing ONOS, starting Mininet and ONOS" +\
57 "cli sessions."
58 # TODO: save all the timers and output them for plotting
59
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070071 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070072 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070073 global ONOS1Port
74 global ONOS2Port
75 global ONOS3Port
76 global ONOS4Port
77 global ONOS5Port
78 global ONOS6Port
79 global ONOS7Port
80
81 # FIXME: just get controller port from params?
82 # TODO: do we really need all these?
83 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
84 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
85 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
86 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
87 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
88 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
89 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
90
Jon Halle1a3b752015-07-22 13:02:46 -070091 try:
92 fileName = "Counters"
93 path = main.params[ 'imports' ][ 'path' ]
94 main.Counters = imp.load_source( fileName,
95 path + fileName + ".py" )
96 except Exception as e:
97 main.log.exception( e )
98 main.cleanup()
99 main.exit()
100
101 main.CLIs = []
102 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700103 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700104 for i in range( 1, main.numCtrls + 1 ):
105 try:
106 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
107 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
108 ipList.append( main.nodes[ -1 ].ip_address )
109 except AttributeError:
110 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700111
112 main.step( "Create cell file" )
113 cellAppString = main.params[ 'ENV' ][ 'appString' ]
114 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
115 main.Mininet1.ip_address,
116 cellAppString, ipList )
117 main.step( "Applying cell variable to environment" )
118 cellResult = main.ONOSbench.setCell( cellName )
119 verifyResult = main.ONOSbench.verifyCell()
120
121 # FIXME:this is short term fix
122 main.log.info( "Removing raft logs" )
123 main.ONOSbench.onosRemoveRaftLogs()
124
125 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700126 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700127 main.ONOSbench.onosUninstall( node.ip_address )
128
129 # Make sure ONOS is DEAD
130 main.log.info( "Killing any ONOS processes" )
131 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700132 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700133 killed = main.ONOSbench.onosKill( node.ip_address )
134 killResults = killResults and killed
135
136 cleanInstallResult = main.TRUE
137 gitPullResult = main.TRUE
138
139 main.step( "Starting Mininet" )
140 # scp topo file to mininet
141 # TODO: move to params?
142 topoName = "obelisk.py"
143 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700144 main.ONOSbench.scp( main.Mininet1,
145 filePath + topoName,
146 main.Mininet1.home,
147 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700148 mnResult = main.Mininet1.startNet( )
149 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
150 onpass="Mininet Started",
151 onfail="Error starting Mininet" )
152
153 main.step( "Git checkout and pull " + gitBranch )
154 if PULLCODE:
155 main.ONOSbench.gitCheckout( gitBranch )
156 gitPullResult = main.ONOSbench.gitPull()
157 # values of 1 or 3 are good
158 utilities.assert_lesser( expect=0, actual=gitPullResult,
159 onpass="Git pull successful",
160 onfail="Git pull failed" )
161 main.ONOSbench.getVersion( report=True )
162
163 main.step( "Using mvn clean install" )
164 cleanInstallResult = main.TRUE
165 if PULLCODE and gitPullResult == main.TRUE:
166 cleanInstallResult = main.ONOSbench.cleanInstall()
167 else:
168 main.log.warn( "Did not pull new code so skipping mvn " +
169 "clean install" )
170 utilities.assert_equals( expect=main.TRUE,
171 actual=cleanInstallResult,
172 onpass="MCI successful",
173 onfail="MCI failed" )
174 # GRAPHS
175 # NOTE: important params here:
176 # job = name of Jenkins job
177 # Plot Name = Plot-HA, only can be used if multiple plots
178 # index = The number of the graph under plot name
179 job = "HAsanity"
180 plotName = "Plot-HA"
Jon Hallff566d52016-01-15 14:45:36 -0800181 index = "1"
Jon Hall5cf14d52015-07-16 12:15:19 -0700182 graphs = '<ac:structured-macro ac:name="html">\n'
183 graphs += '<ac:plain-text-body><![CDATA[\n'
184 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800185 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700186 '&width=500&height=300"' +\
187 'noborder="0" width="500" height="300" scrolling="yes" ' +\
188 'seamless="seamless"></iframe>\n'
189 graphs += ']]></ac:plain-text-body>\n'
190 graphs += '</ac:structured-macro>\n'
191 main.log.wiki(graphs)
192
193 main.step( "Creating ONOS package" )
194 packageResult = main.ONOSbench.onosPackage()
195 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
196 onpass="ONOS package successful",
197 onfail="ONOS package failed" )
198
199 main.step( "Installing ONOS package" )
200 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700201 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700202 tmpResult = main.ONOSbench.onosInstall( options="-f",
203 node=node.ip_address )
204 onosInstallResult = onosInstallResult and tmpResult
205 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
206 onpass="ONOS install successful",
207 onfail="ONOS install failed" )
208
209 main.step( "Checking if ONOS is up yet" )
210 for i in range( 2 ):
211 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700212 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700213 started = main.ONOSbench.isup( node.ip_address )
214 if not started:
215 main.log.error( node.name + " didn't start!" )
216 main.ONOSbench.onosStop( node.ip_address )
217 main.ONOSbench.onosStart( node.ip_address )
218 onosIsupResult = onosIsupResult and started
219 if onosIsupResult == main.TRUE:
220 break
221 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
222 onpass="ONOS startup successful",
223 onfail="ONOS startup failed" )
224
225 main.log.step( "Starting ONOS CLI sessions" )
226 cliResults = main.TRUE
227 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700228 for i in range( main.numCtrls ):
229 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700231 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700232 threads.append( t )
233 t.start()
234
235 for t in threads:
236 t.join()
237 cliResults = cliResults and t.result
238 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
239 onpass="ONOS cli startup successful",
240 onfail="ONOS cli startup failed" )
241
242 if main.params[ 'tcpdump' ].lower() == "true":
243 main.step( "Start Packet Capture MN" )
244 main.Mininet2.startTcpdump(
245 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
246 + "-MN.pcap",
247 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
248 port=main.params[ 'MNtcpdump' ][ 'port' ] )
249
250 main.step( "App Ids check" )
Jon Hallf3d16e72015-12-16 17:45:08 -0800251 time.sleep(60)
Jon Hall5cf14d52015-07-16 12:15:19 -0700252 appCheck = main.TRUE
253 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700254 for i in range( main.numCtrls ):
255 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700256 name="appToIDCheck-" + str( i ),
257 args=[] )
258 threads.append( t )
259 t.start()
260
261 for t in threads:
262 t.join()
263 appCheck = appCheck and t.result
264 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700265 main.log.warn( main.CLIs[0].apps() )
266 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700267 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
268 onpass="App Ids seem to be correct",
269 onfail="Something is wrong with app Ids" )
270
271 if cliResults == main.FALSE:
272 main.log.error( "Failed to start ONOS, stopping test" )
273 main.cleanup()
274 main.exit()
275
276 def CASE2( self, main ):
277 """
278 Assign devices to controllers
279 """
280 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700281 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700282 assert main, "main not defined"
283 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700284 assert main.CLIs, "main.CLIs not defined"
285 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700286 assert ONOS1Port, "ONOS1Port not defined"
287 assert ONOS2Port, "ONOS2Port not defined"
288 assert ONOS3Port, "ONOS3Port not defined"
289 assert ONOS4Port, "ONOS4Port not defined"
290 assert ONOS5Port, "ONOS5Port not defined"
291 assert ONOS6Port, "ONOS6Port not defined"
292 assert ONOS7Port, "ONOS7Port not defined"
293
294 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700295 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700296 "and check that an ONOS node becomes the " +\
297 "master of the device."
298 main.step( "Assign switches to controllers" )
299
300 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700301 for i in range( main.numCtrls ):
302 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700303 swList = []
304 for i in range( 1, 29 ):
305 swList.append( "s" + str( i ) )
306 main.Mininet1.assignSwController( sw=swList, ip=ipList )
307
308 mastershipCheck = main.TRUE
309 for i in range( 1, 29 ):
310 response = main.Mininet1.getSwController( "s" + str( i ) )
311 try:
312 main.log.info( str( response ) )
313 except Exception:
314 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700315 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700316 if re.search( "tcp:" + node.ip_address, response ):
317 mastershipCheck = mastershipCheck and main.TRUE
318 else:
319 main.log.error( "Error, node " + node.ip_address + " is " +
320 "not in the list of controllers s" +
321 str( i ) + " is connecting to." )
322 mastershipCheck = main.FALSE
323 utilities.assert_equals(
324 expect=main.TRUE,
325 actual=mastershipCheck,
326 onpass="Switch mastership assigned correctly",
327 onfail="Switches not assigned correctly to controllers" )
328
329 def CASE21( self, main ):
330 """
331 Assign mastership to controllers
332 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700333 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700334 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700335 assert main, "main not defined"
336 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700337 assert main.CLIs, "main.CLIs not defined"
338 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700339 assert ONOS1Port, "ONOS1Port not defined"
340 assert ONOS2Port, "ONOS2Port not defined"
341 assert ONOS3Port, "ONOS3Port not defined"
342 assert ONOS4Port, "ONOS4Port not defined"
343 assert ONOS5Port, "ONOS5Port not defined"
344 assert ONOS6Port, "ONOS6Port not defined"
345 assert ONOS7Port, "ONOS7Port not defined"
346
347 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700348 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700349 "device. Then manually assign" +\
350 " mastership to specific ONOS nodes using" +\
351 " 'device-role'"
352 main.step( "Assign mastership of switches to specific controllers" )
353 # Manually assign mastership to the controller we want
354 roleCall = main.TRUE
355
356 ipList = [ ]
357 deviceList = []
358 try:
359 # Assign mastership to specific controllers. This assignment was
360 # determined for a 7 node cluser, but will work with any sized
361 # cluster
362 for i in range( 1, 29 ): # switches 1 through 28
363 # set up correct variables:
364 if i == 1:
365 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700366 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700367 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
368 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700369 c = 1 % main.numCtrls
370 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700371 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
372 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700373 c = 1 % main.numCtrls
374 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700375 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
376 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700377 c = 3 % main.numCtrls
378 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700379 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
380 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700381 c = 2 % main.numCtrls
382 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700383 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
384 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700385 c = 2 % main.numCtrls
386 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700387 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
388 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700389 c = 5 % main.numCtrls
390 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700391 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
392 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700393 c = 4 % main.numCtrls
394 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700395 dpid = '3' + str( i ).zfill( 3 )
396 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
397 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700398 c = 6 % main.numCtrls
399 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700400 dpid = '6' + str( i ).zfill( 3 )
401 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
402 elif i == 28:
403 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700404 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700405 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
406 else:
407 main.log.error( "You didn't write an else statement for " +
408 "switch s" + str( i ) )
409 roleCall = main.FALSE
410 # Assign switch
411 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
412 # TODO: make this controller dynamic
413 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
414 ip )
415 ipList.append( ip )
416 deviceList.append( deviceId )
417 except ( AttributeError, AssertionError ):
418 main.log.exception( "Something is wrong with ONOS device view" )
419 main.log.info( main.ONOScli1.devices() )
420 utilities.assert_equals(
421 expect=main.TRUE,
422 actual=roleCall,
423 onpass="Re-assigned switch mastership to designated controller",
424 onfail="Something wrong with deviceRole calls" )
425
426 main.step( "Check mastership was correctly assigned" )
427 roleCheck = main.TRUE
428 # NOTE: This is due to the fact that device mastership change is not
429 # atomic and is actually a multi step process
430 time.sleep( 5 )
431 for i in range( len( ipList ) ):
432 ip = ipList[i]
433 deviceId = deviceList[i]
434 # Check assignment
435 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
436 if ip in master:
437 roleCheck = roleCheck and main.TRUE
438 else:
439 roleCheck = roleCheck and main.FALSE
440 main.log.error( "Error, controller " + ip + " is not" +
441 " master " + "of device " +
442 str( deviceId ) + ". Master is " +
443 repr( master ) + "." )
444 utilities.assert_equals(
445 expect=main.TRUE,
446 actual=roleCheck,
447 onpass="Switches were successfully reassigned to designated " +
448 "controller",
449 onfail="Switches were not successfully reassigned" )
450
451 def CASE3( self, main ):
452 """
453 Assign intents
454 """
455 import time
456 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700457 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700458 assert main, "main not defined"
459 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700460 assert main.CLIs, "main.CLIs not defined"
461 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700462 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700463 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700464 "assign predetermined host-to-host intents." +\
465 " After installation, check that the intent" +\
466 " is distributed to all nodes and the state" +\
467 " is INSTALLED"
468
469 # install onos-app-fwd
470 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700471 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700472 utilities.assert_equals( expect=main.TRUE, actual=installResults,
473 onpass="Install fwd successful",
474 onfail="Install fwd failed" )
475
476 main.step( "Check app ids" )
477 appCheck = main.TRUE
478 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700479 for i in range( main.numCtrls ):
480 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700481 name="appToIDCheck-" + str( i ),
482 args=[] )
483 threads.append( t )
484 t.start()
485
486 for t in threads:
487 t.join()
488 appCheck = appCheck and t.result
489 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700490 main.log.warn( main.CLIs[0].apps() )
491 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700492 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
493 onpass="App Ids seem to be correct",
494 onfail="Something is wrong with app Ids" )
495
496 main.step( "Discovering Hosts( Via pingall for now )" )
497 # FIXME: Once we have a host discovery mechanism, use that instead
498 # REACTIVE FWD test
499 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700500 passMsg = "Reactive Pingall test passed"
501 time1 = time.time()
502 pingResult = main.Mininet1.pingall()
503 time2 = time.time()
504 if not pingResult:
505 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700506 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700507 passMsg += " on the second try"
508 utilities.assert_equals(
509 expect=main.TRUE,
510 actual=pingResult,
511 onpass= passMsg,
512 onfail="Reactive Pingall failed, " +
513 "one or more ping pairs failed" )
514 main.log.info( "Time for pingall: %2f seconds" %
515 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700516 # timeout for fwd flows
517 time.sleep( 11 )
518 # uninstall onos-app-fwd
519 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700520 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700521 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
522 onpass="Uninstall fwd successful",
523 onfail="Uninstall fwd failed" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700524
525 main.step( "Check app ids" )
526 threads = []
527 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700528 for i in range( main.numCtrls ):
529 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700530 name="appToIDCheck-" + str( i ),
531 args=[] )
532 threads.append( t )
533 t.start()
534
535 for t in threads:
536 t.join()
537 appCheck2 = appCheck2 and t.result
538 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700539 main.log.warn( main.CLIs[0].apps() )
540 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700541 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
542 onpass="App Ids seem to be correct",
543 onfail="Something is wrong with app Ids" )
544
545 main.step( "Add host intents via cli" )
546 intentIds = []
547 # TODO: move the host numbers to params
548 # Maybe look at all the paths we ping?
549 intentAddResult = True
550 hostResult = main.TRUE
551 for i in range( 8, 18 ):
552 main.log.info( "Adding host intent between h" + str( i ) +
553 " and h" + str( i + 10 ) )
554 host1 = "00:00:00:00:00:" + \
555 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
556 host2 = "00:00:00:00:00:" + \
557 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
558 # NOTE: getHost can return None
559 host1Dict = main.ONOScli1.getHost( host1 )
560 host2Dict = main.ONOScli1.getHost( host2 )
561 host1Id = None
562 host2Id = None
563 if host1Dict and host2Dict:
564 host1Id = host1Dict.get( 'id', None )
565 host2Id = host2Dict.get( 'id', None )
566 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700567 nodeNum = ( i % main.numCtrls )
568 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700569 if tmpId:
570 main.log.info( "Added intent with id: " + tmpId )
571 intentIds.append( tmpId )
572 else:
573 main.log.error( "addHostIntent returned: " +
574 repr( tmpId ) )
575 else:
576 main.log.error( "Error, getHost() failed for h" + str( i ) +
577 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700578 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700579 main.log.warn( "Hosts output: " )
580 try:
581 main.log.warn( json.dumps( json.loads( hosts ),
582 sort_keys=True,
583 indent=4,
584 separators=( ',', ': ' ) ) )
585 except ( ValueError, TypeError ):
586 main.log.warn( repr( hosts ) )
587 hostResult = main.FALSE
588 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
589 onpass="Found a host id for each host",
590 onfail="Error looking up host ids" )
591
592 intentStart = time.time()
593 onosIds = main.ONOScli1.getAllIntentsId()
594 main.log.info( "Submitted intents: " + str( intentIds ) )
595 main.log.info( "Intents in ONOS: " + str( onosIds ) )
596 for intent in intentIds:
597 if intent in onosIds:
598 pass # intent submitted is in onos
599 else:
600 intentAddResult = False
601 if intentAddResult:
602 intentStop = time.time()
603 else:
604 intentStop = None
605 # Print the intent states
606 intents = main.ONOScli1.intents()
607 intentStates = []
608 installedCheck = True
609 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
610 count = 0
611 try:
612 for intent in json.loads( intents ):
613 state = intent.get( 'state', None )
614 if "INSTALLED" not in state:
615 installedCheck = False
616 intentId = intent.get( 'id', None )
617 intentStates.append( ( intentId, state ) )
618 except ( ValueError, TypeError ):
619 main.log.exception( "Error parsing intents" )
620 # add submitted intents not in the store
621 tmplist = [ i for i, s in intentStates ]
622 missingIntents = False
623 for i in intentIds:
624 if i not in tmplist:
625 intentStates.append( ( i, " - " ) )
626 missingIntents = True
627 intentStates.sort()
628 for i, s in intentStates:
629 count += 1
630 main.log.info( "%-6s%-15s%-15s" %
631 ( str( count ), str( i ), str( s ) ) )
632 leaders = main.ONOScli1.leaders()
633 try:
634 missing = False
635 if leaders:
636 parsedLeaders = json.loads( leaders )
637 main.log.warn( json.dumps( parsedLeaders,
638 sort_keys=True,
639 indent=4,
640 separators=( ',', ': ' ) ) )
641 # check for all intent partitions
642 topics = []
643 for i in range( 14 ):
644 topics.append( "intent-partition-" + str( i ) )
645 main.log.debug( topics )
646 ONOStopics = [ j['topic'] for j in parsedLeaders ]
647 for topic in topics:
648 if topic not in ONOStopics:
649 main.log.error( "Error: " + topic +
650 " not in leaders" )
651 missing = True
652 else:
653 main.log.error( "leaders() returned None" )
654 except ( ValueError, TypeError ):
655 main.log.exception( "Error parsing leaders" )
656 main.log.error( repr( leaders ) )
657 # Check all nodes
658 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700659 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700660 response = node.leaders( jsonFormat=False)
661 main.log.warn( str( node.name ) + " leaders output: \n" +
662 str( response ) )
663
664 partitions = main.ONOScli1.partitions()
665 try:
666 if partitions :
667 parsedPartitions = json.loads( partitions )
668 main.log.warn( json.dumps( parsedPartitions,
669 sort_keys=True,
670 indent=4,
671 separators=( ',', ': ' ) ) )
672 # TODO check for a leader in all paritions
673 # TODO check for consistency among nodes
674 else:
675 main.log.error( "partitions() returned None" )
676 except ( ValueError, TypeError ):
677 main.log.exception( "Error parsing partitions" )
678 main.log.error( repr( partitions ) )
679 pendingMap = main.ONOScli1.pendingMap()
680 try:
681 if pendingMap :
682 parsedPending = json.loads( pendingMap )
683 main.log.warn( json.dumps( parsedPending,
684 sort_keys=True,
685 indent=4,
686 separators=( ',', ': ' ) ) )
687 # TODO check something here?
688 else:
689 main.log.error( "pendingMap() returned None" )
690 except ( ValueError, TypeError ):
691 main.log.exception( "Error parsing pending map" )
692 main.log.error( repr( pendingMap ) )
693
694 intentAddResult = bool( intentAddResult and not missingIntents and
695 installedCheck )
696 if not intentAddResult:
697 main.log.error( "Error in pushing host intents to ONOS" )
698
699 main.step( "Intent Anti-Entropy dispersion" )
700 for i in range(100):
701 correct = True
702 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700703 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700704 onosIds = []
705 ids = cli.getAllIntentsId()
706 onosIds.append( ids )
707 main.log.debug( "Intents in " + cli.name + ": " +
708 str( sorted( onosIds ) ) )
709 if sorted( ids ) != sorted( intentIds ):
710 main.log.warn( "Set of intent IDs doesn't match" )
711 correct = False
712 break
713 else:
714 intents = json.loads( cli.intents() )
715 for intent in intents:
716 if intent[ 'state' ] != "INSTALLED":
717 main.log.warn( "Intent " + intent[ 'id' ] +
718 " is " + intent[ 'state' ] )
719 correct = False
720 break
721 if correct:
722 break
723 else:
724 time.sleep(1)
725 if not intentStop:
726 intentStop = time.time()
727 global gossipTime
728 gossipTime = intentStop - intentStart
729 main.log.info( "It took about " + str( gossipTime ) +
730 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700731 gossipPeriod = int( main.params['timers']['gossip'] )
732 maxGossipTime = gossipPeriod * len( main.nodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700733 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700734 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700735 onpass="ECM anti-entropy for intents worked within " +
736 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700737 onfail="Intent ECM anti-entropy took too long. " +
738 "Expected time:{}, Actual time:{}".format( maxGossipTime,
739 gossipTime ) )
740 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700741 intentAddResult = True
742
743 if not intentAddResult or "key" in pendingMap:
744 import time
745 installedCheck = True
746 main.log.info( "Sleeping 60 seconds to see if intents are found" )
747 time.sleep( 60 )
748 onosIds = main.ONOScli1.getAllIntentsId()
749 main.log.info( "Submitted intents: " + str( intentIds ) )
750 main.log.info( "Intents in ONOS: " + str( onosIds ) )
751 # Print the intent states
752 intents = main.ONOScli1.intents()
753 intentStates = []
754 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
755 count = 0
756 try:
757 for intent in json.loads( intents ):
758 # Iter through intents of a node
759 state = intent.get( 'state', None )
760 if "INSTALLED" not in state:
761 installedCheck = False
762 intentId = intent.get( 'id', None )
763 intentStates.append( ( intentId, state ) )
764 except ( ValueError, TypeError ):
765 main.log.exception( "Error parsing intents" )
766 # add submitted intents not in the store
767 tmplist = [ i for i, s in intentStates ]
768 for i in intentIds:
769 if i not in tmplist:
770 intentStates.append( ( i, " - " ) )
771 intentStates.sort()
772 for i, s in intentStates:
773 count += 1
774 main.log.info( "%-6s%-15s%-15s" %
775 ( str( count ), str( i ), str( s ) ) )
776 leaders = main.ONOScli1.leaders()
777 try:
778 missing = False
779 if leaders:
780 parsedLeaders = json.loads( leaders )
781 main.log.warn( json.dumps( parsedLeaders,
782 sort_keys=True,
783 indent=4,
784 separators=( ',', ': ' ) ) )
785 # check for all intent partitions
786 # check for election
787 topics = []
788 for i in range( 14 ):
789 topics.append( "intent-partition-" + str( i ) )
790 # FIXME: this should only be after we start the app
791 topics.append( "org.onosproject.election" )
792 main.log.debug( topics )
793 ONOStopics = [ j['topic'] for j in parsedLeaders ]
794 for topic in topics:
795 if topic not in ONOStopics:
796 main.log.error( "Error: " + topic +
797 " not in leaders" )
798 missing = True
799 else:
800 main.log.error( "leaders() returned None" )
801 except ( ValueError, TypeError ):
802 main.log.exception( "Error parsing leaders" )
803 main.log.error( repr( leaders ) )
804 # Check all nodes
805 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700806 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700807 response = node.leaders( jsonFormat=False)
808 main.log.warn( str( node.name ) + " leaders output: \n" +
809 str( response ) )
810
811 partitions = main.ONOScli1.partitions()
812 try:
813 if partitions :
814 parsedPartitions = json.loads( partitions )
815 main.log.warn( json.dumps( parsedPartitions,
816 sort_keys=True,
817 indent=4,
818 separators=( ',', ': ' ) ) )
819 # TODO check for a leader in all paritions
820 # TODO check for consistency among nodes
821 else:
822 main.log.error( "partitions() returned None" )
823 except ( ValueError, TypeError ):
824 main.log.exception( "Error parsing partitions" )
825 main.log.error( repr( partitions ) )
826 pendingMap = main.ONOScli1.pendingMap()
827 try:
828 if pendingMap :
829 parsedPending = json.loads( pendingMap )
830 main.log.warn( json.dumps( parsedPending,
831 sort_keys=True,
832 indent=4,
833 separators=( ',', ': ' ) ) )
834 # TODO check something here?
835 else:
836 main.log.error( "pendingMap() returned None" )
837 except ( ValueError, TypeError ):
838 main.log.exception( "Error parsing pending map" )
839 main.log.error( repr( pendingMap ) )
840
841 def CASE4( self, main ):
842 """
843 Ping across added host intents
844 """
845 import json
846 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700847 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700848 assert main, "main not defined"
849 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700850 assert main.CLIs, "main.CLIs not defined"
851 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700852 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700853 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700854 "functionality and check the state of " +\
855 "the intent"
856 main.step( "Ping across added host intents" )
857 PingResult = main.TRUE
858 for i in range( 8, 18 ):
859 ping = main.Mininet1.pingHost( src="h" + str( i ),
860 target="h" + str( i + 10 ) )
861 PingResult = PingResult and ping
862 if ping == main.FALSE:
863 main.log.warn( "Ping failed between h" + str( i ) +
864 " and h" + str( i + 10 ) )
865 elif ping == main.TRUE:
866 main.log.info( "Ping test passed!" )
867 # Don't set PingResult or you'd override failures
868 if PingResult == main.FALSE:
869 main.log.error(
870 "Intents have not been installed correctly, pings failed." )
871 # TODO: pretty print
872 main.log.warn( "ONOS1 intents: " )
873 try:
874 tmpIntents = main.ONOScli1.intents()
875 main.log.warn( json.dumps( json.loads( tmpIntents ),
876 sort_keys=True,
877 indent=4,
878 separators=( ',', ': ' ) ) )
879 except ( ValueError, TypeError ):
880 main.log.warn( repr( tmpIntents ) )
881 utilities.assert_equals(
882 expect=main.TRUE,
883 actual=PingResult,
884 onpass="Intents have been installed correctly and pings work",
885 onfail="Intents have not been installed correctly, pings failed." )
886
887 main.step( "Check Intent state" )
888 installedCheck = False
889 loopCount = 0
890 while not installedCheck and loopCount < 40:
891 installedCheck = True
892 # Print the intent states
893 intents = main.ONOScli1.intents()
894 intentStates = []
895 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
896 count = 0
897 # Iter through intents of a node
898 try:
899 for intent in json.loads( intents ):
900 state = intent.get( 'state', None )
901 if "INSTALLED" not in state:
902 installedCheck = False
903 intentId = intent.get( 'id', None )
904 intentStates.append( ( intentId, state ) )
905 except ( ValueError, TypeError ):
906 main.log.exception( "Error parsing intents." )
907 # Print states
908 intentStates.sort()
909 for i, s in intentStates:
910 count += 1
911 main.log.info( "%-6s%-15s%-15s" %
912 ( str( count ), str( i ), str( s ) ) )
913 if not installedCheck:
914 time.sleep( 1 )
915 loopCount += 1
916 utilities.assert_equals( expect=True, actual=installedCheck,
917 onpass="Intents are all INSTALLED",
918 onfail="Intents are not all in " +
919 "INSTALLED state" )
920
921 main.step( "Check leadership of topics" )
922 leaders = main.ONOScli1.leaders()
923 topicCheck = main.TRUE
924 try:
925 if leaders:
926 parsedLeaders = json.loads( leaders )
927 main.log.warn( json.dumps( parsedLeaders,
928 sort_keys=True,
929 indent=4,
930 separators=( ',', ': ' ) ) )
931 # check for all intent partitions
932 # check for election
933 # TODO: Look at Devices as topics now that it uses this system
934 topics = []
935 for i in range( 14 ):
936 topics.append( "intent-partition-" + str( i ) )
937 # FIXME: this should only be after we start the app
938 # FIXME: topics.append( "org.onosproject.election" )
939 # Print leaders output
940 main.log.debug( topics )
941 ONOStopics = [ j['topic'] for j in parsedLeaders ]
942 for topic in topics:
943 if topic not in ONOStopics:
944 main.log.error( "Error: " + topic +
945 " not in leaders" )
946 topicCheck = main.FALSE
947 else:
948 main.log.error( "leaders() returned None" )
949 topicCheck = main.FALSE
950 except ( ValueError, TypeError ):
951 topicCheck = main.FALSE
952 main.log.exception( "Error parsing leaders" )
953 main.log.error( repr( leaders ) )
954 # TODO: Check for a leader of these topics
955 # Check all nodes
956 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700957 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700958 response = node.leaders( jsonFormat=False)
959 main.log.warn( str( node.name ) + " leaders output: \n" +
960 str( response ) )
961
962 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
963 onpass="intent Partitions is in leaders",
964 onfail="Some topics were lost " )
965 # Print partitions
966 partitions = main.ONOScli1.partitions()
967 try:
968 if partitions :
969 parsedPartitions = json.loads( partitions )
970 main.log.warn( json.dumps( parsedPartitions,
971 sort_keys=True,
972 indent=4,
973 separators=( ',', ': ' ) ) )
974 # TODO check for a leader in all paritions
975 # TODO check for consistency among nodes
976 else:
977 main.log.error( "partitions() returned None" )
978 except ( ValueError, TypeError ):
979 main.log.exception( "Error parsing partitions" )
980 main.log.error( repr( partitions ) )
981 # Print Pending Map
982 pendingMap = main.ONOScli1.pendingMap()
983 try:
984 if pendingMap :
985 parsedPending = json.loads( pendingMap )
986 main.log.warn( json.dumps( parsedPending,
987 sort_keys=True,
988 indent=4,
989 separators=( ',', ': ' ) ) )
990 # TODO check something here?
991 else:
992 main.log.error( "pendingMap() returned None" )
993 except ( ValueError, TypeError ):
994 main.log.exception( "Error parsing pending map" )
995 main.log.error( repr( pendingMap ) )
996
997 if not installedCheck:
998 main.log.info( "Waiting 60 seconds to see if the state of " +
999 "intents change" )
1000 time.sleep( 60 )
1001 # Print the intent states
1002 intents = main.ONOScli1.intents()
1003 intentStates = []
1004 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1005 count = 0
1006 # Iter through intents of a node
1007 try:
1008 for intent in json.loads( intents ):
1009 state = intent.get( 'state', None )
1010 if "INSTALLED" not in state:
1011 installedCheck = False
1012 intentId = intent.get( 'id', None )
1013 intentStates.append( ( intentId, state ) )
1014 except ( ValueError, TypeError ):
1015 main.log.exception( "Error parsing intents." )
1016 intentStates.sort()
1017 for i, s in intentStates:
1018 count += 1
1019 main.log.info( "%-6s%-15s%-15s" %
1020 ( str( count ), str( i ), str( s ) ) )
1021 leaders = main.ONOScli1.leaders()
1022 try:
1023 missing = False
1024 if leaders:
1025 parsedLeaders = json.loads( leaders )
1026 main.log.warn( json.dumps( parsedLeaders,
1027 sort_keys=True,
1028 indent=4,
1029 separators=( ',', ': ' ) ) )
1030 # check for all intent partitions
1031 # check for election
1032 topics = []
1033 for i in range( 14 ):
1034 topics.append( "intent-partition-" + str( i ) )
1035 # FIXME: this should only be after we start the app
1036 topics.append( "org.onosproject.election" )
1037 main.log.debug( topics )
1038 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1039 for topic in topics:
1040 if topic not in ONOStopics:
1041 main.log.error( "Error: " + topic +
1042 " not in leaders" )
1043 missing = True
1044 else:
1045 main.log.error( "leaders() returned None" )
1046 except ( ValueError, TypeError ):
1047 main.log.exception( "Error parsing leaders" )
1048 main.log.error( repr( leaders ) )
1049 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001050 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001051 response = node.leaders( jsonFormat=False)
1052 main.log.warn( str( node.name ) + " leaders output: \n" +
1053 str( response ) )
1054
1055 partitions = main.ONOScli1.partitions()
1056 try:
1057 if partitions :
1058 parsedPartitions = json.loads( partitions )
1059 main.log.warn( json.dumps( parsedPartitions,
1060 sort_keys=True,
1061 indent=4,
1062 separators=( ',', ': ' ) ) )
1063 # TODO check for a leader in all paritions
1064 # TODO check for consistency among nodes
1065 else:
1066 main.log.error( "partitions() returned None" )
1067 except ( ValueError, TypeError ):
1068 main.log.exception( "Error parsing partitions" )
1069 main.log.error( repr( partitions ) )
1070 pendingMap = main.ONOScli1.pendingMap()
1071 try:
1072 if pendingMap :
1073 parsedPending = json.loads( pendingMap )
1074 main.log.warn( json.dumps( parsedPending,
1075 sort_keys=True,
1076 indent=4,
1077 separators=( ',', ': ' ) ) )
1078 # TODO check something here?
1079 else:
1080 main.log.error( "pendingMap() returned None" )
1081 except ( ValueError, TypeError ):
1082 main.log.exception( "Error parsing pending map" )
1083 main.log.error( repr( pendingMap ) )
1084 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001085 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001086 main.step( "Wait a minute then ping again" )
1087 # the wait is above
1088 PingResult = main.TRUE
1089 for i in range( 8, 18 ):
1090 ping = main.Mininet1.pingHost( src="h" + str( i ),
1091 target="h" + str( i + 10 ) )
1092 PingResult = PingResult and ping
1093 if ping == main.FALSE:
1094 main.log.warn( "Ping failed between h" + str( i ) +
1095 " and h" + str( i + 10 ) )
1096 elif ping == main.TRUE:
1097 main.log.info( "Ping test passed!" )
1098 # Don't set PingResult or you'd override failures
1099 if PingResult == main.FALSE:
1100 main.log.error(
1101 "Intents have not been installed correctly, pings failed." )
1102 # TODO: pretty print
1103 main.log.warn( "ONOS1 intents: " )
1104 try:
1105 tmpIntents = main.ONOScli1.intents()
1106 main.log.warn( json.dumps( json.loads( tmpIntents ),
1107 sort_keys=True,
1108 indent=4,
1109 separators=( ',', ': ' ) ) )
1110 except ( ValueError, TypeError ):
1111 main.log.warn( repr( tmpIntents ) )
1112 utilities.assert_equals(
1113 expect=main.TRUE,
1114 actual=PingResult,
1115 onpass="Intents have been installed correctly and pings work",
1116 onfail="Intents have not been installed correctly, pings failed." )
1117
1118 def CASE5( self, main ):
1119 """
1120 Reading state of ONOS
1121 """
1122 import json
1123 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001124 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001125 assert main, "main not defined"
1126 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001127 assert main.CLIs, "main.CLIs not defined"
1128 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001129
1130 main.case( "Setting up and gathering data for current state" )
1131 # The general idea for this test case is to pull the state of
1132 # ( intents,flows, topology,... ) from each ONOS node
1133 # We can then compare them with each other and also with past states
1134
1135 main.step( "Check that each switch has a master" )
1136 global mastershipState
1137 mastershipState = '[]'
1138
1139 # Assert that each device has a master
1140 rolesNotNull = main.TRUE
1141 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001142 for i in range( main.numCtrls ):
1143 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001144 name="rolesNotNull-" + str( i ),
1145 args=[] )
1146 threads.append( t )
1147 t.start()
1148
1149 for t in threads:
1150 t.join()
1151 rolesNotNull = rolesNotNull and t.result
1152 utilities.assert_equals(
1153 expect=main.TRUE,
1154 actual=rolesNotNull,
1155 onpass="Each device has a master",
1156 onfail="Some devices don't have a master assigned" )
1157
1158 main.step( "Get the Mastership of each switch from each controller" )
1159 ONOSMastership = []
1160 mastershipCheck = main.FALSE
1161 consistentMastership = True
1162 rolesResults = True
1163 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001164 for i in range( main.numCtrls ):
1165 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001166 name="roles-" + str( i ),
1167 args=[] )
1168 threads.append( t )
1169 t.start()
1170
1171 for t in threads:
1172 t.join()
1173 ONOSMastership.append( t.result )
1174
Jon Halle1a3b752015-07-22 13:02:46 -07001175 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001176 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1177 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1178 " roles" )
1179 main.log.warn(
1180 "ONOS" + str( i + 1 ) + " mastership response: " +
1181 repr( ONOSMastership[i] ) )
1182 rolesResults = False
1183 utilities.assert_equals(
1184 expect=True,
1185 actual=rolesResults,
1186 onpass="No error in reading roles output",
1187 onfail="Error in reading roles from ONOS" )
1188
1189 main.step( "Check for consistency in roles from each controller" )
1190 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1191 main.log.info(
1192 "Switch roles are consistent across all ONOS nodes" )
1193 else:
1194 consistentMastership = False
1195 utilities.assert_equals(
1196 expect=True,
1197 actual=consistentMastership,
1198 onpass="Switch roles are consistent across all ONOS nodes",
1199 onfail="ONOS nodes have different views of switch roles" )
1200
1201 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001202 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001203 try:
1204 main.log.warn(
1205 "ONOS" + str( i + 1 ) + " roles: ",
1206 json.dumps(
1207 json.loads( ONOSMastership[ i ] ),
1208 sort_keys=True,
1209 indent=4,
1210 separators=( ',', ': ' ) ) )
1211 except ( ValueError, TypeError ):
1212 main.log.warn( repr( ONOSMastership[ i ] ) )
1213 elif rolesResults and consistentMastership:
1214 mastershipCheck = main.TRUE
1215 mastershipState = ONOSMastership[ 0 ]
1216
1217 main.step( "Get the intents from each controller" )
1218 global intentState
1219 intentState = []
1220 ONOSIntents = []
1221 intentCheck = main.FALSE
1222 consistentIntents = True
1223 intentsResults = True
1224 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001225 for i in range( main.numCtrls ):
1226 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001227 name="intents-" + str( i ),
1228 args=[],
1229 kwargs={ 'jsonFormat': True } )
1230 threads.append( t )
1231 t.start()
1232
1233 for t in threads:
1234 t.join()
1235 ONOSIntents.append( t.result )
1236
Jon Halle1a3b752015-07-22 13:02:46 -07001237 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001238 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1239 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1240 " intents" )
1241 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1242 repr( ONOSIntents[ i ] ) )
1243 intentsResults = False
1244 utilities.assert_equals(
1245 expect=True,
1246 actual=intentsResults,
1247 onpass="No error in reading intents output",
1248 onfail="Error in reading intents from ONOS" )
1249
1250 main.step( "Check for consistency in Intents from each controller" )
1251 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1252 main.log.info( "Intents are consistent across all ONOS " +
1253 "nodes" )
1254 else:
1255 consistentIntents = False
1256 main.log.error( "Intents not consistent" )
1257 utilities.assert_equals(
1258 expect=True,
1259 actual=consistentIntents,
1260 onpass="Intents are consistent across all ONOS nodes",
1261 onfail="ONOS nodes have different views of intents" )
1262
1263 if intentsResults:
1264 # Try to make it easy to figure out what is happening
1265 #
1266 # Intent ONOS1 ONOS2 ...
1267 # 0x01 INSTALLED INSTALLING
1268 # ... ... ...
1269 # ... ... ...
1270 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001271 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001272 title += " " * 10 + "ONOS" + str( n + 1 )
1273 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001274 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001275 keys = []
1276 try:
1277 # Get the set of all intent keys
1278 for nodeStr in ONOSIntents:
1279 node = json.loads( nodeStr )
1280 for intent in node:
1281 keys.append( intent.get( 'id' ) )
1282 keys = set( keys )
1283 # For each intent key, print the state on each node
1284 for key in keys:
1285 row = "%-13s" % key
1286 for nodeStr in ONOSIntents:
1287 node = json.loads( nodeStr )
1288 for intent in node:
1289 if intent.get( 'id', "Error" ) == key:
1290 row += "%-15s" % intent.get( 'state' )
1291 main.log.warn( row )
1292 # End of intent state table
1293 except ValueError as e:
1294 main.log.exception( e )
1295 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1296
1297 if intentsResults and not consistentIntents:
1298 # print the json objects
1299 n = len(ONOSIntents)
1300 main.log.debug( "ONOS" + str( n ) + " intents: " )
1301 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1302 sort_keys=True,
1303 indent=4,
1304 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001305 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001306 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1307 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1308 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1309 sort_keys=True,
1310 indent=4,
1311 separators=( ',', ': ' ) ) )
1312 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001313 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001314 str( n ) + " intents" )
1315 elif intentsResults and consistentIntents:
1316 intentCheck = main.TRUE
1317 intentState = ONOSIntents[ 0 ]
1318
1319 main.step( "Get the flows from each controller" )
1320 global flowState
1321 flowState = []
1322 ONOSFlows = []
1323 ONOSFlowsJson = []
1324 flowCheck = main.FALSE
1325 consistentFlows = True
1326 flowsResults = True
1327 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001328 for i in range( main.numCtrls ):
1329 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001330 name="flows-" + str( i ),
1331 args=[],
1332 kwargs={ 'jsonFormat': True } )
1333 threads.append( t )
1334 t.start()
1335
1336 # NOTE: Flows command can take some time to run
1337 time.sleep(30)
1338 for t in threads:
1339 t.join()
1340 result = t.result
1341 ONOSFlows.append( result )
1342
Jon Halle1a3b752015-07-22 13:02:46 -07001343 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001344 num = str( i + 1 )
1345 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1346 main.log.error( "Error in getting ONOS" + num + " flows" )
1347 main.log.warn( "ONOS" + num + " flows response: " +
1348 repr( ONOSFlows[ i ] ) )
1349 flowsResults = False
1350 ONOSFlowsJson.append( None )
1351 else:
1352 try:
1353 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1354 except ( ValueError, TypeError ):
1355 # FIXME: change this to log.error?
1356 main.log.exception( "Error in parsing ONOS" + num +
1357 " response as json." )
1358 main.log.error( repr( ONOSFlows[ i ] ) )
1359 ONOSFlowsJson.append( None )
1360 flowsResults = False
1361 utilities.assert_equals(
1362 expect=True,
1363 actual=flowsResults,
1364 onpass="No error in reading flows output",
1365 onfail="Error in reading flows from ONOS" )
1366
1367 main.step( "Check for consistency in Flows from each controller" )
1368 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1369 if all( tmp ):
1370 main.log.info( "Flow count is consistent across all ONOS nodes" )
1371 else:
1372 consistentFlows = False
1373 utilities.assert_equals(
1374 expect=True,
1375 actual=consistentFlows,
1376 onpass="The flow count is consistent across all ONOS nodes",
1377 onfail="ONOS nodes have different flow counts" )
1378
1379 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001380 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001381 try:
1382 main.log.warn(
1383 "ONOS" + str( i + 1 ) + " flows: " +
1384 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1385 indent=4, separators=( ',', ': ' ) ) )
1386 except ( ValueError, TypeError ):
1387 main.log.warn(
1388 "ONOS" + str( i + 1 ) + " flows: " +
1389 repr( ONOSFlows[ i ] ) )
1390 elif flowsResults and consistentFlows:
1391 flowCheck = main.TRUE
1392 flowState = ONOSFlows[ 0 ]
1393
1394 main.step( "Get the OF Table entries" )
1395 global flows
1396 flows = []
1397 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001398 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001399 if flowCheck == main.FALSE:
1400 for table in flows:
1401 main.log.warn( table )
GlennRC68467eb2015-11-16 18:01:01 -08001402
Jon Hall5cf14d52015-07-16 12:15:19 -07001403 # TODO: Compare switch flow tables with ONOS flow tables
1404
1405 main.step( "Start continuous pings" )
1406 main.Mininet2.pingLong(
1407 src=main.params[ 'PING' ][ 'source1' ],
1408 target=main.params[ 'PING' ][ 'target1' ],
1409 pingTime=500 )
1410 main.Mininet2.pingLong(
1411 src=main.params[ 'PING' ][ 'source2' ],
1412 target=main.params[ 'PING' ][ 'target2' ],
1413 pingTime=500 )
1414 main.Mininet2.pingLong(
1415 src=main.params[ 'PING' ][ 'source3' ],
1416 target=main.params[ 'PING' ][ 'target3' ],
1417 pingTime=500 )
1418 main.Mininet2.pingLong(
1419 src=main.params[ 'PING' ][ 'source4' ],
1420 target=main.params[ 'PING' ][ 'target4' ],
1421 pingTime=500 )
1422 main.Mininet2.pingLong(
1423 src=main.params[ 'PING' ][ 'source5' ],
1424 target=main.params[ 'PING' ][ 'target5' ],
1425 pingTime=500 )
1426 main.Mininet2.pingLong(
1427 src=main.params[ 'PING' ][ 'source6' ],
1428 target=main.params[ 'PING' ][ 'target6' ],
1429 pingTime=500 )
1430 main.Mininet2.pingLong(
1431 src=main.params[ 'PING' ][ 'source7' ],
1432 target=main.params[ 'PING' ][ 'target7' ],
1433 pingTime=500 )
1434 main.Mininet2.pingLong(
1435 src=main.params[ 'PING' ][ 'source8' ],
1436 target=main.params[ 'PING' ][ 'target8' ],
1437 pingTime=500 )
1438 main.Mininet2.pingLong(
1439 src=main.params[ 'PING' ][ 'source9' ],
1440 target=main.params[ 'PING' ][ 'target9' ],
1441 pingTime=500 )
1442 main.Mininet2.pingLong(
1443 src=main.params[ 'PING' ][ 'source10' ],
1444 target=main.params[ 'PING' ][ 'target10' ],
1445 pingTime=500 )
1446
1447 main.step( "Collecting topology information from ONOS" )
1448 devices = []
1449 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001450 for i in range( main.numCtrls ):
1451 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001452 name="devices-" + str( i ),
1453 args=[ ] )
1454 threads.append( t )
1455 t.start()
1456
1457 for t in threads:
1458 t.join()
1459 devices.append( t.result )
1460 hosts = []
1461 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001462 for i in range( main.numCtrls ):
1463 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001464 name="hosts-" + str( i ),
1465 args=[ ] )
1466 threads.append( t )
1467 t.start()
1468
1469 for t in threads:
1470 t.join()
1471 try:
1472 hosts.append( json.loads( t.result ) )
1473 except ( ValueError, TypeError ):
1474 # FIXME: better handling of this, print which node
1475 # Maybe use thread name?
1476 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001477 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001478 hosts.append( None )
1479
1480 ports = []
1481 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001482 for i in range( main.numCtrls ):
1483 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001484 name="ports-" + str( i ),
1485 args=[ ] )
1486 threads.append( t )
1487 t.start()
1488
1489 for t in threads:
1490 t.join()
1491 ports.append( t.result )
1492 links = []
1493 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001494 for i in range( main.numCtrls ):
1495 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001496 name="links-" + str( i ),
1497 args=[ ] )
1498 threads.append( t )
1499 t.start()
1500
1501 for t in threads:
1502 t.join()
1503 links.append( t.result )
1504 clusters = []
1505 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001506 for i in range( main.numCtrls ):
1507 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001508 name="clusters-" + str( i ),
1509 args=[ ] )
1510 threads.append( t )
1511 t.start()
1512
1513 for t in threads:
1514 t.join()
1515 clusters.append( t.result )
1516 # Compare json objects for hosts and dataplane clusters
1517
1518 # hosts
1519 main.step( "Host view is consistent across ONOS nodes" )
1520 consistentHostsResult = main.TRUE
1521 for controller in range( len( hosts ) ):
1522 controllerStr = str( controller + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001523 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001524 if hosts[ controller ] == hosts[ 0 ]:
1525 continue
1526 else: # hosts not consistent
1527 main.log.error( "hosts from ONOS" +
1528 controllerStr +
1529 " is inconsistent with ONOS1" )
1530 main.log.warn( repr( hosts[ controller ] ) )
1531 consistentHostsResult = main.FALSE
1532
1533 else:
1534 main.log.error( "Error in getting ONOS hosts from ONOS" +
1535 controllerStr )
1536 consistentHostsResult = main.FALSE
1537 main.log.warn( "ONOS" + controllerStr +
1538 " hosts response: " +
1539 repr( hosts[ controller ] ) )
1540 utilities.assert_equals(
1541 expect=main.TRUE,
1542 actual=consistentHostsResult,
1543 onpass="Hosts view is consistent across all ONOS nodes",
1544 onfail="ONOS nodes have different views of hosts" )
1545
1546 main.step( "Each host has an IP address" )
1547 ipResult = main.TRUE
1548 for controller in range( 0, len( hosts ) ):
1549 controllerStr = str( controller + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001550 if hosts[ controller ]:
1551 for host in hosts[ controller ]:
1552 if not host.get( 'ipAddresses', [ ] ):
1553 main.log.error( "Error with host ips on controller" +
1554 controllerStr + ": " + str( host ) )
1555 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001556 utilities.assert_equals(
1557 expect=main.TRUE,
1558 actual=ipResult,
1559 onpass="The ips of the hosts aren't empty",
1560 onfail="The ip of at least one host is missing" )
1561
1562 # Strongly connected clusters of devices
1563 main.step( "Cluster view is consistent across ONOS nodes" )
1564 consistentClustersResult = main.TRUE
1565 for controller in range( len( clusters ) ):
1566 controllerStr = str( controller + 1 )
1567 if "Error" not in clusters[ controller ]:
1568 if clusters[ controller ] == clusters[ 0 ]:
1569 continue
1570 else: # clusters not consistent
1571 main.log.error( "clusters from ONOS" + controllerStr +
1572 " is inconsistent with ONOS1" )
1573 consistentClustersResult = main.FALSE
1574
1575 else:
1576 main.log.error( "Error in getting dataplane clusters " +
1577 "from ONOS" + controllerStr )
1578 consistentClustersResult = main.FALSE
1579 main.log.warn( "ONOS" + controllerStr +
1580 " clusters response: " +
1581 repr( clusters[ controller ] ) )
1582 utilities.assert_equals(
1583 expect=main.TRUE,
1584 actual=consistentClustersResult,
1585 onpass="Clusters view is consistent across all ONOS nodes",
1586 onfail="ONOS nodes have different views of clusters" )
1587 # there should always only be one cluster
1588 main.step( "Cluster view correct across ONOS nodes" )
1589 try:
1590 numClusters = len( json.loads( clusters[ 0 ] ) )
1591 except ( ValueError, TypeError ):
1592 main.log.exception( "Error parsing clusters[0]: " +
1593 repr( clusters[ 0 ] ) )
1594 clusterResults = main.FALSE
1595 if numClusters == 1:
1596 clusterResults = main.TRUE
1597 utilities.assert_equals(
1598 expect=1,
1599 actual=numClusters,
1600 onpass="ONOS shows 1 SCC",
1601 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1602
1603 main.step( "Comparing ONOS topology to MN" )
1604 devicesResults = main.TRUE
1605 linksResults = main.TRUE
1606 hostsResults = main.TRUE
1607 mnSwitches = main.Mininet1.getSwitches()
1608 mnLinks = main.Mininet1.getLinks()
1609 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001610 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001611 controllerStr = str( controller + 1 )
1612 if devices[ controller ] and ports[ controller ] and\
1613 "Error" not in devices[ controller ] and\
1614 "Error" not in ports[ controller ]:
1615
1616 currentDevicesResult = main.Mininet1.compareSwitches(
1617 mnSwitches,
1618 json.loads( devices[ controller ] ),
1619 json.loads( ports[ controller ] ) )
1620 else:
1621 currentDevicesResult = main.FALSE
1622 utilities.assert_equals( expect=main.TRUE,
1623 actual=currentDevicesResult,
1624 onpass="ONOS" + controllerStr +
1625 " Switches view is correct",
1626 onfail="ONOS" + controllerStr +
1627 " Switches view is incorrect" )
1628 if links[ controller ] and "Error" not in links[ controller ]:
1629 currentLinksResult = main.Mininet1.compareLinks(
1630 mnSwitches, mnLinks,
1631 json.loads( links[ controller ] ) )
1632 else:
1633 currentLinksResult = main.FALSE
1634 utilities.assert_equals( expect=main.TRUE,
1635 actual=currentLinksResult,
1636 onpass="ONOS" + controllerStr +
1637 " links view is correct",
1638 onfail="ONOS" + controllerStr +
1639 " links view is incorrect" )
1640
Jon Hall657cdf62015-12-17 14:40:51 -08001641 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001642 currentHostsResult = main.Mininet1.compareHosts(
1643 mnHosts,
1644 hosts[ controller ] )
1645 else:
1646 currentHostsResult = main.FALSE
1647 utilities.assert_equals( expect=main.TRUE,
1648 actual=currentHostsResult,
1649 onpass="ONOS" + controllerStr +
1650 " hosts exist in Mininet",
1651 onfail="ONOS" + controllerStr +
1652 " hosts don't match Mininet" )
1653
1654 devicesResults = devicesResults and currentDevicesResult
1655 linksResults = linksResults and currentLinksResult
1656 hostsResults = hostsResults and currentHostsResult
1657
1658 main.step( "Device information is correct" )
1659 utilities.assert_equals(
1660 expect=main.TRUE,
1661 actual=devicesResults,
1662 onpass="Device information is correct",
1663 onfail="Device information is incorrect" )
1664
1665 main.step( "Links are correct" )
1666 utilities.assert_equals(
1667 expect=main.TRUE,
1668 actual=linksResults,
1669 onpass="Link are correct",
1670 onfail="Links are incorrect" )
1671
1672 main.step( "Hosts are correct" )
1673 utilities.assert_equals(
1674 expect=main.TRUE,
1675 actual=hostsResults,
1676 onpass="Hosts are correct",
1677 onfail="Hosts are incorrect" )
1678
1679 def CASE6( self, main ):
1680 """
1681 The Failure case. Since this is the Sanity test, we do nothing.
1682 """
1683 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001684 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001685 assert main, "main not defined"
1686 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001687 assert main.CLIs, "main.CLIs not defined"
1688 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001689 main.case( "Wait 60 seconds instead of inducing a failure" )
1690 time.sleep( 60 )
1691 utilities.assert_equals(
1692 expect=main.TRUE,
1693 actual=main.TRUE,
1694 onpass="Sleeping 60 seconds",
1695 onfail="Something is terribly wrong with my math" )
1696
1697 def CASE7( self, main ):
1698 """
1699 Check state after ONOS failure
1700 """
1701 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001702 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001703 assert main, "main not defined"
1704 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001705 assert main.CLIs, "main.CLIs not defined"
1706 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001707 main.case( "Running ONOS Constant State Tests" )
1708
1709 main.step( "Check that each switch has a master" )
1710 # Assert that each device has a master
1711 rolesNotNull = main.TRUE
1712 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001713 for i in range( main.numCtrls ):
1714 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001715 name="rolesNotNull-" + str( i ),
1716 args=[ ] )
1717 threads.append( t )
1718 t.start()
1719
1720 for t in threads:
1721 t.join()
1722 rolesNotNull = rolesNotNull and t.result
1723 utilities.assert_equals(
1724 expect=main.TRUE,
1725 actual=rolesNotNull,
1726 onpass="Each device has a master",
1727 onfail="Some devices don't have a master assigned" )
1728
1729 main.step( "Read device roles from ONOS" )
1730 ONOSMastership = []
1731 mastershipCheck = main.FALSE
1732 consistentMastership = True
1733 rolesResults = True
1734 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001735 for i in range( main.numCtrls ):
1736 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001737 name="roles-" + str( i ),
1738 args=[] )
1739 threads.append( t )
1740 t.start()
1741
1742 for t in threads:
1743 t.join()
1744 ONOSMastership.append( t.result )
1745
Jon Halle1a3b752015-07-22 13:02:46 -07001746 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001747 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1748 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1749 " roles" )
1750 main.log.warn(
1751 "ONOS" + str( i + 1 ) + " mastership response: " +
1752 repr( ONOSMastership[i] ) )
1753 rolesResults = False
1754 utilities.assert_equals(
1755 expect=True,
1756 actual=rolesResults,
1757 onpass="No error in reading roles output",
1758 onfail="Error in reading roles from ONOS" )
1759
1760 main.step( "Check for consistency in roles from each controller" )
1761 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1762 main.log.info(
1763 "Switch roles are consistent across all ONOS nodes" )
1764 else:
1765 consistentMastership = False
1766 utilities.assert_equals(
1767 expect=True,
1768 actual=consistentMastership,
1769 onpass="Switch roles are consistent across all ONOS nodes",
1770 onfail="ONOS nodes have different views of switch roles" )
1771
1772 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001773 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001774 main.log.warn(
1775 "ONOS" + str( i + 1 ) + " roles: ",
1776 json.dumps(
1777 json.loads( ONOSMastership[ i ] ),
1778 sort_keys=True,
1779 indent=4,
1780 separators=( ',', ': ' ) ) )
1781 elif rolesResults and not consistentMastership:
1782 mastershipCheck = main.TRUE
1783
1784 description2 = "Compare switch roles from before failure"
1785 main.step( description2 )
1786 try:
1787 currentJson = json.loads( ONOSMastership[0] )
1788 oldJson = json.loads( mastershipState )
1789 except ( ValueError, TypeError ):
1790 main.log.exception( "Something is wrong with parsing " +
1791 "ONOSMastership[0] or mastershipState" )
1792 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1793 main.log.error( "mastershipState" + repr( mastershipState ) )
1794 main.cleanup()
1795 main.exit()
1796 mastershipCheck = main.TRUE
1797 for i in range( 1, 29 ):
1798 switchDPID = str(
1799 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1800 current = [ switch[ 'master' ] for switch in currentJson
1801 if switchDPID in switch[ 'id' ] ]
1802 old = [ switch[ 'master' ] for switch in oldJson
1803 if switchDPID in switch[ 'id' ] ]
1804 if current == old:
1805 mastershipCheck = mastershipCheck and main.TRUE
1806 else:
1807 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1808 mastershipCheck = main.FALSE
1809 utilities.assert_equals(
1810 expect=main.TRUE,
1811 actual=mastershipCheck,
1812 onpass="Mastership of Switches was not changed",
1813 onfail="Mastership of some switches changed" )
1814 mastershipCheck = mastershipCheck and consistentMastership
1815
1816 main.step( "Get the intents and compare across all nodes" )
1817 ONOSIntents = []
1818 intentCheck = main.FALSE
1819 consistentIntents = True
1820 intentsResults = True
1821 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001822 for i in range( main.numCtrls ):
1823 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001824 name="intents-" + str( i ),
1825 args=[],
1826 kwargs={ 'jsonFormat': True } )
1827 threads.append( t )
1828 t.start()
1829
1830 for t in threads:
1831 t.join()
1832 ONOSIntents.append( t.result )
1833
Jon Halle1a3b752015-07-22 13:02:46 -07001834 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001835 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1836 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1837 " intents" )
1838 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1839 repr( ONOSIntents[ i ] ) )
1840 intentsResults = False
1841 utilities.assert_equals(
1842 expect=True,
1843 actual=intentsResults,
1844 onpass="No error in reading intents output",
1845 onfail="Error in reading intents from ONOS" )
1846
1847 main.step( "Check for consistency in Intents from each controller" )
1848 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1849 main.log.info( "Intents are consistent across all ONOS " +
1850 "nodes" )
1851 else:
1852 consistentIntents = False
1853
1854 # Try to make it easy to figure out what is happening
1855 #
1856 # Intent ONOS1 ONOS2 ...
1857 # 0x01 INSTALLED INSTALLING
1858 # ... ... ...
1859 # ... ... ...
1860 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001861 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001862 title += " " * 10 + "ONOS" + str( n + 1 )
1863 main.log.warn( title )
1864 # get all intent keys in the cluster
1865 keys = []
1866 for nodeStr in ONOSIntents:
1867 node = json.loads( nodeStr )
1868 for intent in node:
1869 keys.append( intent.get( 'id' ) )
1870 keys = set( keys )
1871 for key in keys:
1872 row = "%-13s" % key
1873 for nodeStr in ONOSIntents:
1874 node = json.loads( nodeStr )
1875 for intent in node:
1876 if intent.get( 'id' ) == key:
1877 row += "%-15s" % intent.get( 'state' )
1878 main.log.warn( row )
1879 # End table view
1880
1881 utilities.assert_equals(
1882 expect=True,
1883 actual=consistentIntents,
1884 onpass="Intents are consistent across all ONOS nodes",
1885 onfail="ONOS nodes have different views of intents" )
1886 intentStates = []
1887 for node in ONOSIntents: # Iter through ONOS nodes
1888 nodeStates = []
1889 # Iter through intents of a node
1890 try:
1891 for intent in json.loads( node ):
1892 nodeStates.append( intent[ 'state' ] )
1893 except ( ValueError, TypeError ):
1894 main.log.exception( "Error in parsing intents" )
1895 main.log.error( repr( node ) )
1896 intentStates.append( nodeStates )
1897 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1898 main.log.info( dict( out ) )
1899
1900 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001901 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001902 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1903 main.log.warn( json.dumps(
1904 json.loads( ONOSIntents[ i ] ),
1905 sort_keys=True,
1906 indent=4,
1907 separators=( ',', ': ' ) ) )
1908 elif intentsResults and consistentIntents:
1909 intentCheck = main.TRUE
1910
1911 # NOTE: Store has no durability, so intents are lost across system
1912 # restarts
1913 main.step( "Compare current intents with intents before the failure" )
1914 # NOTE: this requires case 5 to pass for intentState to be set.
1915 # maybe we should stop the test if that fails?
1916 sameIntents = main.FALSE
1917 if intentState and intentState == ONOSIntents[ 0 ]:
1918 sameIntents = main.TRUE
1919 main.log.info( "Intents are consistent with before failure" )
1920 # TODO: possibly the states have changed? we may need to figure out
1921 # what the acceptable states are
1922 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1923 sameIntents = main.TRUE
1924 try:
1925 before = json.loads( intentState )
1926 after = json.loads( ONOSIntents[ 0 ] )
1927 for intent in before:
1928 if intent not in after:
1929 sameIntents = main.FALSE
1930 main.log.debug( "Intent is not currently in ONOS " +
1931 "(at least in the same form):" )
1932 main.log.debug( json.dumps( intent ) )
1933 except ( ValueError, TypeError ):
1934 main.log.exception( "Exception printing intents" )
1935 main.log.debug( repr( ONOSIntents[0] ) )
1936 main.log.debug( repr( intentState ) )
1937 if sameIntents == main.FALSE:
1938 try:
1939 main.log.debug( "ONOS intents before: " )
1940 main.log.debug( json.dumps( json.loads( intentState ),
1941 sort_keys=True, indent=4,
1942 separators=( ',', ': ' ) ) )
1943 main.log.debug( "Current ONOS intents: " )
1944 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1945 sort_keys=True, indent=4,
1946 separators=( ',', ': ' ) ) )
1947 except ( ValueError, TypeError ):
1948 main.log.exception( "Exception printing intents" )
1949 main.log.debug( repr( ONOSIntents[0] ) )
1950 main.log.debug( repr( intentState ) )
1951 utilities.assert_equals(
1952 expect=main.TRUE,
1953 actual=sameIntents,
1954 onpass="Intents are consistent with before failure",
1955 onfail="The Intents changed during failure" )
1956 intentCheck = intentCheck and sameIntents
1957
1958 main.step( "Get the OF Table entries and compare to before " +
1959 "component failure" )
1960 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07001961 for i in range( 28 ):
1962 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08001963 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
1964 FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
Jon Hall5cf14d52015-07-16 12:15:19 -07001965 if FlowTables == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08001966 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
1967
Jon Hall5cf14d52015-07-16 12:15:19 -07001968 utilities.assert_equals(
1969 expect=main.TRUE,
1970 actual=FlowTables,
1971 onpass="No changes were found in the flow tables",
1972 onfail="Changes were found in the flow tables" )
1973
1974 main.Mininet2.pingLongKill()
1975 '''
1976 main.step( "Check the continuous pings to ensure that no packets " +
1977 "were dropped during component failure" )
1978 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1979 main.params[ 'TESTONIP' ] )
1980 LossInPings = main.FALSE
1981 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1982 for i in range( 8, 18 ):
1983 main.log.info(
1984 "Checking for a loss in pings along flow from s" +
1985 str( i ) )
1986 LossInPings = main.Mininet2.checkForLoss(
1987 "/tmp/ping.h" +
1988 str( i ) ) or LossInPings
1989 if LossInPings == main.TRUE:
1990 main.log.info( "Loss in ping detected" )
1991 elif LossInPings == main.ERROR:
1992 main.log.info( "There are multiple mininet process running" )
1993 elif LossInPings == main.FALSE:
1994 main.log.info( "No Loss in the pings" )
1995 main.log.info( "No loss of dataplane connectivity" )
1996 utilities.assert_equals(
1997 expect=main.FALSE,
1998 actual=LossInPings,
1999 onpass="No Loss of connectivity",
2000 onfail="Loss of dataplane connectivity detected" )
2001 '''
2002
2003 main.step( "Leadership Election is still functional" )
2004 # Test of LeadershipElection
2005 # NOTE: this only works for the sanity test. In case of failures,
2006 # leader will likely change
Jon Halle1a3b752015-07-22 13:02:46 -07002007 leader = main.nodes[ 0 ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002008 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002009 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002010 leaderN = cli.electionTestLeader()
2011 # verify leader is ONOS1
2012 if leaderN == leader:
2013 # all is well
2014 # NOTE: In failure scenario, this could be a new node, maybe
2015 # check != ONOS1
2016 pass
2017 elif leaderN == main.FALSE:
2018 # error in response
2019 main.log.error( "Something is wrong with " +
2020 "electionTestLeader function, check the" +
2021 " error logs" )
2022 leaderResult = main.FALSE
2023 elif leader != leaderN:
2024 leaderResult = main.FALSE
2025 main.log.error( cli.name + " sees " + str( leaderN ) +
2026 " as the leader of the election app. " +
2027 "Leader should be " + str( leader ) )
2028 utilities.assert_equals(
2029 expect=main.TRUE,
2030 actual=leaderResult,
2031 onpass="Leadership election passed",
2032 onfail="Something went wrong with Leadership election" )
2033
2034 def CASE8( self, main ):
2035 """
2036 Compare topo
2037 """
2038 import json
2039 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002040 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002041 assert main, "main not defined"
2042 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002043 assert main.CLIs, "main.CLIs not defined"
2044 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002045
2046 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002047 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002048 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002049 topoResult = main.FALSE
2050 elapsed = 0
2051 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002052 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002053 startTime = time.time()
2054 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002055 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002056 devicesResults = main.TRUE
2057 linksResults = main.TRUE
2058 hostsResults = main.TRUE
2059 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002060 count += 1
2061 cliStart = time.time()
2062 devices = []
2063 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002064 for i in range( main.numCtrls ):
2065 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002066 name="devices-" + str( i ),
2067 args=[ ] )
2068 threads.append( t )
2069 t.start()
2070
2071 for t in threads:
2072 t.join()
2073 devices.append( t.result )
2074 hosts = []
2075 ipResult = main.TRUE
2076 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002077 for i in range( main.numCtrls ):
Jon Halld8f6de82015-12-17 17:04:34 -08002078 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002079 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002080 args=[ main.CLIs[i].hosts, [ None ] ],
2081 kwargs= { 'sleep': 5, 'attempts': 5,
2082 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002083 threads.append( t )
2084 t.start()
2085
2086 for t in threads:
2087 t.join()
2088 try:
2089 hosts.append( json.loads( t.result ) )
2090 except ( ValueError, TypeError ):
2091 main.log.exception( "Error parsing hosts results" )
2092 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002093 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002094 for controller in range( 0, len( hosts ) ):
2095 controllerStr = str( controller + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002096 if hosts[ controller ]:
2097 for host in hosts[ controller ]:
2098 if host is None or host.get( 'ipAddresses', [] ) == []:
2099 main.log.error(
2100 "Error with host ipAddresses on controller" +
2101 controllerStr + ": " + str( host ) )
2102 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002103 ports = []
2104 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002105 for i in range( main.numCtrls ):
2106 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002107 name="ports-" + str( i ),
2108 args=[ ] )
2109 threads.append( t )
2110 t.start()
2111
2112 for t in threads:
2113 t.join()
2114 ports.append( t.result )
2115 links = []
2116 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002117 for i in range( main.numCtrls ):
2118 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002119 name="links-" + str( i ),
2120 args=[ ] )
2121 threads.append( t )
2122 t.start()
2123
2124 for t in threads:
2125 t.join()
2126 links.append( t.result )
2127 clusters = []
2128 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002129 for i in range( main.numCtrls ):
2130 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002131 name="clusters-" + str( i ),
2132 args=[ ] )
2133 threads.append( t )
2134 t.start()
2135
2136 for t in threads:
2137 t.join()
2138 clusters.append( t.result )
2139
2140 elapsed = time.time() - startTime
2141 cliTime = time.time() - cliStart
2142 print "Elapsed time: " + str( elapsed )
2143 print "CLI time: " + str( cliTime )
2144
2145 mnSwitches = main.Mininet1.getSwitches()
2146 mnLinks = main.Mininet1.getLinks()
2147 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002148 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002149 controllerStr = str( controller + 1 )
2150 if devices[ controller ] and ports[ controller ] and\
2151 "Error" not in devices[ controller ] and\
2152 "Error" not in ports[ controller ]:
2153
2154 currentDevicesResult = main.Mininet1.compareSwitches(
2155 mnSwitches,
2156 json.loads( devices[ controller ] ),
2157 json.loads( ports[ controller ] ) )
2158 else:
2159 currentDevicesResult = main.FALSE
2160 utilities.assert_equals( expect=main.TRUE,
2161 actual=currentDevicesResult,
2162 onpass="ONOS" + controllerStr +
2163 " Switches view is correct",
2164 onfail="ONOS" + controllerStr +
2165 " Switches view is incorrect" )
2166
2167 if links[ controller ] and "Error" not in links[ controller ]:
2168 currentLinksResult = main.Mininet1.compareLinks(
2169 mnSwitches, mnLinks,
2170 json.loads( links[ controller ] ) )
2171 else:
2172 currentLinksResult = main.FALSE
2173 utilities.assert_equals( expect=main.TRUE,
2174 actual=currentLinksResult,
2175 onpass="ONOS" + controllerStr +
2176 " links view is correct",
2177 onfail="ONOS" + controllerStr +
2178 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002179 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002180 currentHostsResult = main.Mininet1.compareHosts(
2181 mnHosts,
2182 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002183 elif hosts[ controller ] == []:
2184 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002185 else:
2186 currentHostsResult = main.FALSE
2187 utilities.assert_equals( expect=main.TRUE,
2188 actual=currentHostsResult,
2189 onpass="ONOS" + controllerStr +
2190 " hosts exist in Mininet",
2191 onfail="ONOS" + controllerStr +
2192 " hosts don't match Mininet" )
2193 # CHECKING HOST ATTACHMENT POINTS
2194 hostAttachment = True
2195 zeroHosts = False
2196 # FIXME: topo-HA/obelisk specific mappings:
2197 # key is mac and value is dpid
2198 mappings = {}
2199 for i in range( 1, 29 ): # hosts 1 through 28
2200 # set up correct variables:
2201 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2202 if i == 1:
2203 deviceId = "1000".zfill(16)
2204 elif i == 2:
2205 deviceId = "2000".zfill(16)
2206 elif i == 3:
2207 deviceId = "3000".zfill(16)
2208 elif i == 4:
2209 deviceId = "3004".zfill(16)
2210 elif i == 5:
2211 deviceId = "5000".zfill(16)
2212 elif i == 6:
2213 deviceId = "6000".zfill(16)
2214 elif i == 7:
2215 deviceId = "6007".zfill(16)
2216 elif i >= 8 and i <= 17:
2217 dpid = '3' + str( i ).zfill( 3 )
2218 deviceId = dpid.zfill(16)
2219 elif i >= 18 and i <= 27:
2220 dpid = '6' + str( i ).zfill( 3 )
2221 deviceId = dpid.zfill(16)
2222 elif i == 28:
2223 deviceId = "2800".zfill(16)
2224 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002225 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002226 if hosts[ controller ] == []:
2227 main.log.warn( "There are no hosts discovered" )
2228 zeroHosts = True
2229 else:
2230 for host in hosts[ controller ]:
2231 mac = None
2232 location = None
2233 device = None
2234 port = None
2235 try:
2236 mac = host.get( 'mac' )
2237 assert mac, "mac field could not be found for this host object"
2238
2239 location = host.get( 'location' )
2240 assert location, "location field could not be found for this host object"
2241
2242 # Trim the protocol identifier off deviceId
2243 device = str( location.get( 'elementId' ) ).split(':')[1]
2244 assert device, "elementId field could not be found for this host location object"
2245
2246 port = location.get( 'port' )
2247 assert port, "port field could not be found for this host location object"
2248
2249 # Now check if this matches where they should be
2250 if mac and device and port:
2251 if str( port ) != "1":
2252 main.log.error( "The attachment port is incorrect for " +
2253 "host " + str( mac ) +
2254 ". Expected: 1 Actual: " + str( port) )
2255 hostAttachment = False
2256 if device != mappings[ str( mac ) ]:
2257 main.log.error( "The attachment device is incorrect for " +
2258 "host " + str( mac ) +
2259 ". Expected: " + mappings[ str( mac ) ] +
2260 " Actual: " + device )
2261 hostAttachment = False
2262 else:
2263 hostAttachment = False
2264 except AssertionError:
2265 main.log.exception( "Json object not as expected" )
2266 main.log.error( repr( host ) )
2267 hostAttachment = False
2268 else:
2269 main.log.error( "No hosts json output or \"Error\"" +
2270 " in output. hosts = " +
2271 repr( hosts[ controller ] ) )
2272 if zeroHosts is False:
2273 hostAttachment = True
2274
2275 # END CHECKING HOST ATTACHMENT POINTS
2276 devicesResults = devicesResults and currentDevicesResult
2277 linksResults = linksResults and currentLinksResult
2278 hostsResults = hostsResults and currentHostsResult
2279 hostAttachmentResults = hostAttachmentResults and\
2280 hostAttachment
2281 topoResult = ( devicesResults and linksResults
2282 and hostsResults and ipResult and
2283 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002284 utilities.assert_equals( expect=True,
2285 actual=topoResult,
2286 onpass="ONOS topology matches Mininet",
2287 onfail="ONOS topology don't match Mininet" )
2288 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002289
2290 # Compare json objects for hosts and dataplane clusters
2291
2292 # hosts
2293 main.step( "Hosts view is consistent across all ONOS nodes" )
2294 consistentHostsResult = main.TRUE
2295 for controller in range( len( hosts ) ):
2296 controllerStr = str( controller + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002297 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002298 if hosts[ controller ] == hosts[ 0 ]:
2299 continue
2300 else: # hosts not consistent
2301 main.log.error( "hosts from ONOS" + controllerStr +
2302 " is inconsistent with ONOS1" )
2303 main.log.warn( repr( hosts[ controller ] ) )
2304 consistentHostsResult = main.FALSE
2305
2306 else:
2307 main.log.error( "Error in getting ONOS hosts from ONOS" +
2308 controllerStr )
2309 consistentHostsResult = main.FALSE
2310 main.log.warn( "ONOS" + controllerStr +
2311 " hosts response: " +
2312 repr( hosts[ controller ] ) )
2313 utilities.assert_equals(
2314 expect=main.TRUE,
2315 actual=consistentHostsResult,
2316 onpass="Hosts view is consistent across all ONOS nodes",
2317 onfail="ONOS nodes have different views of hosts" )
2318
2319 main.step( "Hosts information is correct" )
2320 hostsResults = hostsResults and ipResult
2321 utilities.assert_equals(
2322 expect=main.TRUE,
2323 actual=hostsResults,
2324 onpass="Host information is correct",
2325 onfail="Host information is incorrect" )
2326
2327 main.step( "Host attachment points to the network" )
2328 utilities.assert_equals(
2329 expect=True,
2330 actual=hostAttachmentResults,
2331 onpass="Hosts are correctly attached to the network",
2332 onfail="ONOS did not correctly attach hosts to the network" )
2333
2334 # Strongly connected clusters of devices
2335 main.step( "Clusters view is consistent across all ONOS nodes" )
2336 consistentClustersResult = main.TRUE
2337 for controller in range( len( clusters ) ):
2338 controllerStr = str( controller + 1 )
2339 if "Error" not in clusters[ controller ]:
2340 if clusters[ controller ] == clusters[ 0 ]:
2341 continue
2342 else: # clusters not consistent
2343 main.log.error( "clusters from ONOS" +
2344 controllerStr +
2345 " is inconsistent with ONOS1" )
2346 consistentClustersResult = main.FALSE
2347
2348 else:
2349 main.log.error( "Error in getting dataplane clusters " +
2350 "from ONOS" + controllerStr )
2351 consistentClustersResult = main.FALSE
2352 main.log.warn( "ONOS" + controllerStr +
2353 " clusters response: " +
2354 repr( clusters[ controller ] ) )
2355 utilities.assert_equals(
2356 expect=main.TRUE,
2357 actual=consistentClustersResult,
2358 onpass="Clusters view is consistent across all ONOS nodes",
2359 onfail="ONOS nodes have different views of clusters" )
2360
2361 main.step( "There is only one SCC" )
2362 # there should always only be one cluster
2363 try:
2364 numClusters = len( json.loads( clusters[ 0 ] ) )
2365 except ( ValueError, TypeError ):
2366 main.log.exception( "Error parsing clusters[0]: " +
2367 repr( clusters[0] ) )
2368 clusterResults = main.FALSE
2369 if numClusters == 1:
2370 clusterResults = main.TRUE
2371 utilities.assert_equals(
2372 expect=1,
2373 actual=numClusters,
2374 onpass="ONOS shows 1 SCC",
2375 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2376
2377 topoResult = ( devicesResults and linksResults
2378 and hostsResults and consistentHostsResult
2379 and consistentClustersResult and clusterResults
2380 and ipResult and hostAttachmentResults )
2381
2382 topoResult = topoResult and int( count <= 2 )
2383 note = "note it takes about " + str( int( cliTime ) ) + \
2384 " seconds for the test to make all the cli calls to fetch " +\
2385 "the topology from each ONOS instance"
2386 main.log.info(
2387 "Very crass estimate for topology discovery/convergence( " +
2388 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2389 str( count ) + " tries" )
2390
2391 main.step( "Device information is correct" )
2392 utilities.assert_equals(
2393 expect=main.TRUE,
2394 actual=devicesResults,
2395 onpass="Device information is correct",
2396 onfail="Device information is incorrect" )
2397
2398 main.step( "Links are correct" )
2399 utilities.assert_equals(
2400 expect=main.TRUE,
2401 actual=linksResults,
2402 onpass="Link are correct",
2403 onfail="Links are incorrect" )
2404
2405 main.step( "Hosts are correct" )
2406 utilities.assert_equals(
2407 expect=main.TRUE,
2408 actual=hostsResults,
2409 onpass="Hosts are correct",
2410 onfail="Hosts are incorrect" )
2411
2412 # FIXME: move this to an ONOS state case
2413 main.step( "Checking ONOS nodes" )
2414 nodesOutput = []
2415 nodeResults = main.TRUE
2416 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002417 for i in range( main.numCtrls ):
2418 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002419 name="nodes-" + str( i ),
2420 args=[ ] )
2421 threads.append( t )
2422 t.start()
2423
2424 for t in threads:
2425 t.join()
2426 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002427 ips = [ node.ip_address for node in main.nodes ]
Jon Halle9b1fa32015-12-08 15:32:21 -08002428 ips.sort()
Jon Hall5cf14d52015-07-16 12:15:19 -07002429 for i in nodesOutput:
2430 try:
2431 current = json.loads( i )
Jon Halle9b1fa32015-12-08 15:32:21 -08002432 activeIps = []
2433 currentResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002434 for node in current:
Jon Halle9b1fa32015-12-08 15:32:21 -08002435 if node['state'] == 'ACTIVE':
2436 activeIps.append( node['ip'] )
2437 activeIps.sort()
2438 if ips == activeIps:
2439 currentResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002440 except ( ValueError, TypeError ):
2441 main.log.error( "Error parsing nodes output" )
2442 main.log.warn( repr( i ) )
Jon Halle9b1fa32015-12-08 15:32:21 -08002443 currentResult = main.FALSE
2444 nodeResults = nodeResults and currentResult
Jon Hall5cf14d52015-07-16 12:15:19 -07002445 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2446 onpass="Nodes check successful",
2447 onfail="Nodes check NOT successful" )
2448
2449 def CASE9( self, main ):
2450 """
2451 Link s3-s28 down
2452 """
2453 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002454 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002455 assert main, "main not defined"
2456 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002457 assert main.CLIs, "main.CLIs not defined"
2458 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002459 # NOTE: You should probably run a topology check after this
2460
2461 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2462
2463 description = "Turn off a link to ensure that Link Discovery " +\
2464 "is working properly"
2465 main.case( description )
2466
2467 main.step( "Kill Link between s3 and s28" )
2468 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2469 main.log.info( "Waiting " + str( linkSleep ) +
2470 " seconds for link down to be discovered" )
2471 time.sleep( linkSleep )
2472 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2473 onpass="Link down successful",
2474 onfail="Failed to bring link down" )
2475 # TODO do some sort of check here
2476
2477 def CASE10( self, main ):
2478 """
2479 Link s3-s28 up
2480 """
2481 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002482 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002483 assert main, "main not defined"
2484 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002485 assert main.CLIs, "main.CLIs not defined"
2486 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002487 # NOTE: You should probably run a topology check after this
2488
2489 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2490
2491 description = "Restore a link to ensure that Link Discovery is " + \
2492 "working properly"
2493 main.case( description )
2494
2495 main.step( "Bring link between s3 and s28 back up" )
2496 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2497 main.log.info( "Waiting " + str( linkSleep ) +
2498 " seconds for link up to be discovered" )
2499 time.sleep( linkSleep )
2500 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2501 onpass="Link up successful",
2502 onfail="Failed to bring link up" )
2503 # TODO do some sort of check here
2504
2505 def CASE11( self, main ):
2506 """
2507 Switch Down
2508 """
2509 # NOTE: You should probably run a topology check after this
2510 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002511 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002512 assert main, "main not defined"
2513 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002514 assert main.CLIs, "main.CLIs not defined"
2515 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002516
2517 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2518
2519 description = "Killing a switch to ensure it is discovered correctly"
2520 main.case( description )
2521 switch = main.params[ 'kill' ][ 'switch' ]
2522 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2523
2524 # TODO: Make this switch parameterizable
2525 main.step( "Kill " + switch )
2526 main.log.info( "Deleting " + switch )
2527 main.Mininet1.delSwitch( switch )
2528 main.log.info( "Waiting " + str( switchSleep ) +
2529 " seconds for switch down to be discovered" )
2530 time.sleep( switchSleep )
2531 device = main.ONOScli1.getDevice( dpid=switchDPID )
2532 # Peek at the deleted switch
2533 main.log.warn( str( device ) )
2534 result = main.FALSE
2535 if device and device[ 'available' ] is False:
2536 result = main.TRUE
2537 utilities.assert_equals( expect=main.TRUE, actual=result,
2538 onpass="Kill switch successful",
2539 onfail="Failed to kill switch?" )
2540
2541 def CASE12( self, main ):
2542 """
2543 Switch Up
2544 """
2545 # NOTE: You should probably run a topology check after this
2546 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002547 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002548 assert main, "main not defined"
2549 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002550 assert main.CLIs, "main.CLIs not defined"
2551 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002552 assert ONOS1Port, "ONOS1Port not defined"
2553 assert ONOS2Port, "ONOS2Port not defined"
2554 assert ONOS3Port, "ONOS3Port not defined"
2555 assert ONOS4Port, "ONOS4Port not defined"
2556 assert ONOS5Port, "ONOS5Port not defined"
2557 assert ONOS6Port, "ONOS6Port not defined"
2558 assert ONOS7Port, "ONOS7Port not defined"
2559
2560 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2561 switch = main.params[ 'kill' ][ 'switch' ]
2562 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2563 links = main.params[ 'kill' ][ 'links' ].split()
2564 description = "Adding a switch to ensure it is discovered correctly"
2565 main.case( description )
2566
2567 main.step( "Add back " + switch )
2568 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2569 for peer in links:
2570 main.Mininet1.addLink( switch, peer )
2571 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002572 for i in range( main.numCtrls ):
2573 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002574 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2575 main.log.info( "Waiting " + str( switchSleep ) +
2576 " seconds for switch up to be discovered" )
2577 time.sleep( switchSleep )
2578 device = main.ONOScli1.getDevice( dpid=switchDPID )
2579 # Peek at the deleted switch
2580 main.log.warn( str( device ) )
2581 result = main.FALSE
2582 if device and device[ 'available' ]:
2583 result = main.TRUE
2584 utilities.assert_equals( expect=main.TRUE, actual=result,
2585 onpass="add switch successful",
2586 onfail="Failed to add switch?" )
2587
2588 def CASE13( self, main ):
2589 """
2590 Clean up
2591 """
2592 import os
2593 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002594 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002595 assert main, "main not defined"
2596 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002597 assert main.CLIs, "main.CLIs not defined"
2598 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002599
2600 # printing colors to terminal
2601 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2602 'blue': '\033[94m', 'green': '\033[92m',
2603 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2604 main.case( "Test Cleanup" )
2605 main.step( "Killing tcpdumps" )
2606 main.Mininet2.stopTcpdump()
2607
2608 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002609 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002610 main.step( "Copying MN pcap and ONOS log files to test station" )
2611 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2612 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002613 # NOTE: MN Pcap file is being saved to logdir.
2614 # We scp this file as MN and TestON aren't necessarily the same vm
2615
2616 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002617 # TODO: Load these from params
2618 # NOTE: must end in /
2619 logFolder = "/opt/onos/log/"
2620 logFiles = [ "karaf.log", "karaf.log.1" ]
2621 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002622 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002623 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002624 dstName = main.logdir + "/" + node.name + "-" + f
2625 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2626 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002627 # std*.log's
2628 # NOTE: must end in /
2629 logFolder = "/opt/onos/var/"
2630 logFiles = [ "stderr.log", "stdout.log" ]
2631 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002632 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002633 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002634 dstName = main.logdir + "/" + node.name + "-" + f
2635 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2636 logFolder + f, dstName )
2637 else:
2638 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002639
2640 main.step( "Stopping Mininet" )
2641 mnResult = main.Mininet1.stopNet()
2642 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2643 onpass="Mininet stopped",
2644 onfail="MN cleanup NOT successful" )
2645
2646 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002647 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002648 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2649 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002650
2651 try:
2652 timerLog = open( main.logdir + "/Timers.csv", 'w')
2653 # Overwrite with empty line and close
2654 labels = "Gossip Intents"
2655 data = str( gossipTime )
2656 timerLog.write( labels + "\n" + data )
2657 timerLog.close()
2658 except NameError, e:
2659 main.log.exception(e)
2660
2661 def CASE14( self, main ):
2662 """
2663 start election app on all onos nodes
2664 """
Jon Halle1a3b752015-07-22 13:02:46 -07002665 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002666 assert main, "main not defined"
2667 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002668 assert main.CLIs, "main.CLIs not defined"
2669 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002670
2671 main.case("Start Leadership Election app")
2672 main.step( "Install leadership election app" )
2673 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2674 utilities.assert_equals(
2675 expect=main.TRUE,
2676 actual=appResult,
2677 onpass="Election app installed",
2678 onfail="Something went wrong with installing Leadership election" )
2679
2680 main.step( "Run for election on each node" )
2681 leaderResult = main.TRUE
2682 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002683 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002684 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002685 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002686 leader = cli.electionTestLeader()
2687 if leader is None or leader == main.FALSE:
2688 main.log.error( cli.name + ": Leader for the election app " +
2689 "should be an ONOS node, instead got '" +
2690 str( leader ) + "'" )
2691 leaderResult = main.FALSE
2692 leaders.append( leader )
2693 utilities.assert_equals(
2694 expect=main.TRUE,
2695 actual=leaderResult,
2696 onpass="Successfully ran for leadership",
2697 onfail="Failed to run for leadership" )
2698
2699 main.step( "Check that each node shows the same leader" )
2700 sameLeader = main.TRUE
2701 if len( set( leaders ) ) != 1:
2702 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002703 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002704 str( leaders ) )
2705 utilities.assert_equals(
2706 expect=main.TRUE,
2707 actual=sameLeader,
2708 onpass="Leadership is consistent for the election topic",
2709 onfail="Nodes have different leaders" )
2710
2711 def CASE15( self, main ):
2712 """
2713 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002714 15.1 Run election on each node
2715 15.2 Check that each node has the same leaders and candidates
2716 15.3 Find current leader and withdraw
2717 15.4 Check that a new node was elected leader
2718 15.5 Check that that new leader was the candidate of old leader
2719 15.6 Run for election on old leader
2720 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2721 15.8 Make sure that the old leader was added to the candidate list
2722
2723 old and new variable prefixes refer to data from before vs after
2724 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002725 """
2726 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002727 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002728 assert main, "main not defined"
2729 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002730 assert main.CLIs, "main.CLIs not defined"
2731 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002732
acsmars3a72bde2015-09-02 14:16:22 -07002733 description = "Check that Leadership Election App is still functional"
Jon Hall5cf14d52015-07-16 12:15:19 -07002734 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07002735 # NOTE: Need to re-run since being a canidate is not persistant
2736 # TODO: add check for "Command not found:" in the driver, this
2737 # means the election test app isn't loaded
Jon Hall5cf14d52015-07-16 12:15:19 -07002738
acsmars71adceb2015-08-31 15:09:26 -07002739 oldLeaders = [] # leaders by node before withdrawl from candidates
2740 newLeaders = [] # leaders by node after withdrawl from candidates
2741 oldAllCandidates = [] # list of lists of each nodes' candidates before
2742 newAllCandidates = [] # list of lists of each nodes' candidates after
2743 oldCandidates = [] # list of candidates from node 0 before withdrawl
2744 newCandidates = [] # list of candidates from node 0 after withdrawl
2745 oldLeader = '' # the old leader from oldLeaders, None if not same
2746 newLeader = '' # the new leaders fron newLoeaders, None if not same
2747 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2748 expectNoLeader = False # True when there is only one leader
2749 if main.numCtrls == 1:
2750 expectNoLeader = True
2751
2752 main.step( "Run for election on each node" )
2753 electionResult = main.TRUE
2754
2755 for cli in main.CLIs: # run test election on each node
2756 if cli.electionTestRun() == main.FALSE:
2757 electionResult = main.FALSE
2758
Jon Hall5cf14d52015-07-16 12:15:19 -07002759 utilities.assert_equals(
2760 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002761 actual=electionResult,
2762 onpass="All nodes successfully ran for leadership",
2763 onfail="At least one node failed to run for leadership" )
2764
acsmars3a72bde2015-09-02 14:16:22 -07002765 if electionResult == main.FALSE:
2766 main.log.error(
2767 "Skipping Test Case because Election Test isn't loaded" )
2768 main.skipCase()
2769
acsmars71adceb2015-08-31 15:09:26 -07002770 main.step( "Check that each node shows the same leader and candidates" )
2771 sameResult = main.TRUE
2772 failMessage = "Nodes have different leaders"
2773 for cli in main.CLIs:
2774 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2775 oldAllCandidates.append( node )
2776 oldLeaders.append( node[ 0 ] )
2777 oldCandidates = oldAllCandidates[ 0 ]
2778
2779 # Check that each node has the same leader. Defines oldLeader
2780 if len( set( oldLeaders ) ) != 1:
2781 sameResult = main.FALSE
2782 main.log.error( "More than one leader present:" + str( oldLeaders ) )
2783 oldLeader = None
2784 else:
2785 oldLeader = oldLeaders[ 0 ]
2786
2787 # Check that each node's candidate list is the same
acsmars29233db2015-11-04 11:15:00 -08002788 candidateDiscrepancy = False # Boolean of candidate mismatches
acsmars71adceb2015-08-31 15:09:26 -07002789 for candidates in oldAllCandidates:
2790 if set( candidates ) != set( oldCandidates ):
2791 sameResult = main.FALSE
acsmars29233db2015-11-04 11:15:00 -08002792 candidateDiscrepancy = True
2793
2794 if candidateDiscrepancy:
2795 failMessage += " and candidates"
acsmars71adceb2015-08-31 15:09:26 -07002796
2797 utilities.assert_equals(
2798 expect=main.TRUE,
2799 actual=sameResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002800 onpass="Leadership is consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002801 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002802
2803 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002804 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002805 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002806 if oldLeader is None:
2807 main.log.error( "Leadership isn't consistent." )
2808 withdrawResult = main.FALSE
2809 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002810 for i in range( len( main.CLIs ) ):
acsmars71adceb2015-08-31 15:09:26 -07002811 if oldLeader == main.nodes[ i ].ip_address:
2812 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002813 break
2814 else: # FOR/ELSE statement
2815 main.log.error( "Leader election, could not find current leader" )
2816 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002817 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002818 utilities.assert_equals(
2819 expect=main.TRUE,
2820 actual=withdrawResult,
2821 onpass="Node was withdrawn from election",
2822 onfail="Node was not withdrawn from election" )
2823
acsmars71adceb2015-08-31 15:09:26 -07002824 main.step( "Check that a new node was elected leader" )
2825
Jon Hall5cf14d52015-07-16 12:15:19 -07002826 # FIXME: use threads
acsmars71adceb2015-08-31 15:09:26 -07002827 newLeaderResult = main.TRUE
2828 failMessage = "Nodes have different leaders"
2829
2830 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002831 for cli in main.CLIs:
acsmars71adceb2015-08-31 15:09:26 -07002832 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2833 # elections might no have finished yet
2834 if node[ 0 ] == 'none' and not expectNoLeader:
2835 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2836 "sure elections are complete." )
2837 time.sleep(5)
2838 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2839 # election still isn't done or there is a problem
2840 if node[ 0 ] == 'none':
2841 main.log.error( "No leader was elected on at least 1 node" )
2842 newLeaderResult = main.FALSE
2843 newAllCandidates.append( node )
2844 newLeaders.append( node[ 0 ] )
2845 newCandidates = newAllCandidates[ 0 ]
2846
2847 # Check that each node has the same leader. Defines newLeader
2848 if len( set( newLeaders ) ) != 1:
2849 newLeaderResult = main.FALSE
2850 main.log.error( "Nodes have different leaders: " +
2851 str( newLeaders ) )
2852 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002853 else:
acsmars71adceb2015-08-31 15:09:26 -07002854 newLeader = newLeaders[ 0 ]
2855
2856 # Check that each node's candidate list is the same
2857 for candidates in newAllCandidates:
2858 if set( candidates ) != set( newCandidates ):
2859 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002860 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002861
2862 # Check that the new leader is not the older leader, which was withdrawn
2863 if newLeader == oldLeader:
2864 newLeaderResult = main.FALSE
2865 main.log.error( "All nodes still see old leader: " + oldLeader +
2866 " as the current leader" )
2867
Jon Hall5cf14d52015-07-16 12:15:19 -07002868 utilities.assert_equals(
2869 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002870 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002871 onpass="Leadership election passed",
2872 onfail="Something went wrong with Leadership election" )
2873
acsmars71adceb2015-08-31 15:09:26 -07002874 main.step( "Check that that new leader was the candidate of old leader")
2875 # candidates[ 2 ] should be come the top candidate after withdrawl
2876 correctCandidateResult = main.TRUE
2877 if expectNoLeader:
2878 if newLeader == 'none':
2879 main.log.info( "No leader expected. None found. Pass" )
2880 correctCandidateResult = main.TRUE
2881 else:
2882 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2883 correctCandidateResult = main.FALSE
2884 elif newLeader != oldCandidates[ 2 ]:
2885 correctCandidateResult = main.FALSE
2886 main.log.error( "Candidate " + newLeader + " was elected. " +
2887 oldCandidates[ 2 ] + " should have had priority." )
2888
2889 utilities.assert_equals(
2890 expect=main.TRUE,
2891 actual=correctCandidateResult,
2892 onpass="Correct Candidate Elected",
2893 onfail="Incorrect Candidate Elected" )
2894
Jon Hall5cf14d52015-07-16 12:15:19 -07002895 main.step( "Run for election on old leader( just so everyone " +
2896 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002897 if oldLeaderCLI is not None:
2898 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002899 else:
acsmars71adceb2015-08-31 15:09:26 -07002900 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002901 runResult = main.FALSE
2902 utilities.assert_equals(
2903 expect=main.TRUE,
2904 actual=runResult,
2905 onpass="App re-ran for election",
2906 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002907 main.step(
2908 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002909 # verify leader didn't just change
acsmars71adceb2015-08-31 15:09:26 -07002910 positionResult = main.TRUE
2911 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
2912
2913 # Reset and reuse the new candidate and leaders lists
2914 newAllCandidates = []
2915 newCandidates = []
2916 newLeaders = []
2917 for cli in main.CLIs:
2918 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2919 if oldLeader not in node: # election might no have finished yet
2920 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2921 "be sure elections are complete" )
2922 time.sleep(5)
2923 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2924 if oldLeader not in node: # election still isn't done, errors
2925 main.log.error(
2926 "Old leader was not elected on at least one node" )
2927 positionResult = main.FALSE
2928 newAllCandidates.append( node )
2929 newLeaders.append( node[ 0 ] )
2930 newCandidates = newAllCandidates[ 0 ]
2931
2932 # Check that each node has the same leader. Defines newLeader
2933 if len( set( newLeaders ) ) != 1:
2934 positionResult = main.FALSE
2935 main.log.error( "Nodes have different leaders: " +
2936 str( newLeaders ) )
2937 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002938 else:
acsmars71adceb2015-08-31 15:09:26 -07002939 newLeader = newLeaders[ 0 ]
2940
2941 # Check that each node's candidate list is the same
2942 for candidates in newAllCandidates:
2943 if set( candidates ) != set( newCandidates ):
2944 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002945 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002946
2947 # Check that the re-elected node is last on the candidate List
2948 if oldLeader != newCandidates[ -1 ]:
2949 main.log.error( "Old Leader (" + oldLeader + ") not in the proper position " +
2950 str( newCandidates ) )
2951 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002952
2953 utilities.assert_equals(
2954 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002955 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002956 onpass="Old leader successfully re-ran for election",
2957 onfail="Something went wrong with Leadership election after " +
2958 "the old leader re-ran for election" )
2959
2960 def CASE16( self, main ):
2961 """
2962 Install Distributed Primitives app
2963 """
2964 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002965 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002966 assert main, "main not defined"
2967 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002968 assert main.CLIs, "main.CLIs not defined"
2969 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002970
2971 # Variables for the distributed primitives tests
2972 global pCounterName
2973 global iCounterName
2974 global pCounterValue
2975 global iCounterValue
2976 global onosSet
2977 global onosSetName
2978 pCounterName = "TestON-Partitions"
2979 iCounterName = "TestON-inMemory"
2980 pCounterValue = 0
2981 iCounterValue = 0
2982 onosSet = set([])
2983 onosSetName = "TestON-set"
2984
2985 description = "Install Primitives app"
2986 main.case( description )
2987 main.step( "Install Primitives app" )
2988 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002989 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002990 utilities.assert_equals( expect=main.TRUE,
2991 actual=appResults,
2992 onpass="Primitives app activated",
2993 onfail="Primitives app not activated" )
2994 time.sleep( 5 ) # To allow all nodes to activate
2995
2996 def CASE17( self, main ):
2997 """
2998 Check for basic functionality with distributed primitives
2999 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003000 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003001 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003002 assert main, "main not defined"
3003 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003004 assert main.CLIs, "main.CLIs not defined"
3005 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003006 assert pCounterName, "pCounterName not defined"
3007 assert iCounterName, "iCounterName not defined"
3008 assert onosSetName, "onosSetName not defined"
3009 # NOTE: assert fails if value is 0/None/Empty/False
3010 try:
3011 pCounterValue
3012 except NameError:
3013 main.log.error( "pCounterValue not defined, setting to 0" )
3014 pCounterValue = 0
3015 try:
3016 iCounterValue
3017 except NameError:
3018 main.log.error( "iCounterValue not defined, setting to 0" )
3019 iCounterValue = 0
3020 try:
3021 onosSet
3022 except NameError:
3023 main.log.error( "onosSet not defined, setting to empty Set" )
3024 onosSet = set([])
3025 # Variables for the distributed primitives tests. These are local only
3026 addValue = "a"
3027 addAllValue = "a b c d e f"
3028 retainValue = "c d e f"
3029
3030 description = "Check for basic functionality with distributed " +\
3031 "primitives"
3032 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003033 main.caseExplanation = "Test the methods of the distributed " +\
3034 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003035 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003036 # Partitioned counters
3037 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003038 pCounters = []
3039 threads = []
3040 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003041 for i in range( main.numCtrls ):
3042 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3043 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003044 args=[ pCounterName ] )
3045 pCounterValue += 1
3046 addedPValues.append( pCounterValue )
3047 threads.append( t )
3048 t.start()
3049
3050 for t in threads:
3051 t.join()
3052 pCounters.append( t.result )
3053 # Check that counter incremented numController times
3054 pCounterResults = True
3055 for i in addedPValues:
3056 tmpResult = i in pCounters
3057 pCounterResults = pCounterResults and tmpResult
3058 if not tmpResult:
3059 main.log.error( str( i ) + " is not in partitioned "
3060 "counter incremented results" )
3061 utilities.assert_equals( expect=True,
3062 actual=pCounterResults,
3063 onpass="Default counter incremented",
3064 onfail="Error incrementing default" +
3065 " counter" )
3066
Jon Halle1a3b752015-07-22 13:02:46 -07003067 main.step( "Get then Increment a default counter on each node" )
3068 pCounters = []
3069 threads = []
3070 addedPValues = []
3071 for i in range( main.numCtrls ):
3072 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3073 name="counterGetAndAdd-" + str( i ),
3074 args=[ pCounterName ] )
3075 addedPValues.append( pCounterValue )
3076 pCounterValue += 1
3077 threads.append( t )
3078 t.start()
3079
3080 for t in threads:
3081 t.join()
3082 pCounters.append( t.result )
3083 # Check that counter incremented numController times
3084 pCounterResults = True
3085 for i in addedPValues:
3086 tmpResult = i in pCounters
3087 pCounterResults = pCounterResults and tmpResult
3088 if not tmpResult:
3089 main.log.error( str( i ) + " is not in partitioned "
3090 "counter incremented results" )
3091 utilities.assert_equals( expect=True,
3092 actual=pCounterResults,
3093 onpass="Default counter incremented",
3094 onfail="Error incrementing default" +
3095 " counter" )
3096
3097 main.step( "Counters we added have the correct values" )
3098 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3099 utilities.assert_equals( expect=main.TRUE,
3100 actual=incrementCheck,
3101 onpass="Added counters are correct",
3102 onfail="Added counters are incorrect" )
3103
3104 main.step( "Add -8 to then get a default counter on each node" )
3105 pCounters = []
3106 threads = []
3107 addedPValues = []
3108 for i in range( main.numCtrls ):
3109 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3110 name="counterIncrement-" + str( i ),
3111 args=[ pCounterName ],
3112 kwargs={ "delta": -8 } )
3113 pCounterValue += -8
3114 addedPValues.append( pCounterValue )
3115 threads.append( t )
3116 t.start()
3117
3118 for t in threads:
3119 t.join()
3120 pCounters.append( t.result )
3121 # Check that counter incremented numController times
3122 pCounterResults = True
3123 for i in addedPValues:
3124 tmpResult = i in pCounters
3125 pCounterResults = pCounterResults and tmpResult
3126 if not tmpResult:
3127 main.log.error( str( i ) + " is not in partitioned "
3128 "counter incremented results" )
3129 utilities.assert_equals( expect=True,
3130 actual=pCounterResults,
3131 onpass="Default counter incremented",
3132 onfail="Error incrementing default" +
3133 " counter" )
3134
3135 main.step( "Add 5 to then get a default counter on each node" )
3136 pCounters = []
3137 threads = []
3138 addedPValues = []
3139 for i in range( main.numCtrls ):
3140 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3141 name="counterIncrement-" + str( i ),
3142 args=[ pCounterName ],
3143 kwargs={ "delta": 5 } )
3144 pCounterValue += 5
3145 addedPValues.append( pCounterValue )
3146 threads.append( t )
3147 t.start()
3148
3149 for t in threads:
3150 t.join()
3151 pCounters.append( t.result )
3152 # Check that counter incremented numController times
3153 pCounterResults = True
3154 for i in addedPValues:
3155 tmpResult = i in pCounters
3156 pCounterResults = pCounterResults and tmpResult
3157 if not tmpResult:
3158 main.log.error( str( i ) + " is not in partitioned "
3159 "counter incremented results" )
3160 utilities.assert_equals( expect=True,
3161 actual=pCounterResults,
3162 onpass="Default counter incremented",
3163 onfail="Error incrementing default" +
3164 " counter" )
3165
3166 main.step( "Get then add 5 to a default counter on each node" )
3167 pCounters = []
3168 threads = []
3169 addedPValues = []
3170 for i in range( main.numCtrls ):
3171 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3172 name="counterIncrement-" + str( i ),
3173 args=[ pCounterName ],
3174 kwargs={ "delta": 5 } )
3175 addedPValues.append( pCounterValue )
3176 pCounterValue += 5
3177 threads.append( t )
3178 t.start()
3179
3180 for t in threads:
3181 t.join()
3182 pCounters.append( t.result )
3183 # Check that counter incremented numController times
3184 pCounterResults = True
3185 for i in addedPValues:
3186 tmpResult = i in pCounters
3187 pCounterResults = pCounterResults and tmpResult
3188 if not tmpResult:
3189 main.log.error( str( i ) + " is not in partitioned "
3190 "counter incremented results" )
3191 utilities.assert_equals( expect=True,
3192 actual=pCounterResults,
3193 onpass="Default counter incremented",
3194 onfail="Error incrementing default" +
3195 " counter" )
3196
3197 main.step( "Counters we added have the correct values" )
3198 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3199 utilities.assert_equals( expect=main.TRUE,
3200 actual=incrementCheck,
3201 onpass="Added counters are correct",
3202 onfail="Added counters are incorrect" )
3203
3204 # In-Memory counters
3205 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003206 iCounters = []
3207 addedIValues = []
3208 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003209 for i in range( main.numCtrls ):
3210 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003211 name="icounterIncrement-" + str( i ),
3212 args=[ iCounterName ],
3213 kwargs={ "inMemory": True } )
3214 iCounterValue += 1
3215 addedIValues.append( iCounterValue )
3216 threads.append( t )
3217 t.start()
3218
3219 for t in threads:
3220 t.join()
3221 iCounters.append( t.result )
3222 # Check that counter incremented numController times
3223 iCounterResults = True
3224 for i in addedIValues:
3225 tmpResult = i in iCounters
3226 iCounterResults = iCounterResults and tmpResult
3227 if not tmpResult:
3228 main.log.error( str( i ) + " is not in the in-memory "
3229 "counter incremented results" )
3230 utilities.assert_equals( expect=True,
3231 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003232 onpass="In-memory counter incremented",
3233 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003234 " counter" )
3235
Jon Halle1a3b752015-07-22 13:02:46 -07003236 main.step( "Get then Increment a in-memory counter on each node" )
3237 iCounters = []
3238 threads = []
3239 addedIValues = []
3240 for i in range( main.numCtrls ):
3241 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3242 name="counterGetAndAdd-" + str( i ),
3243 args=[ iCounterName ],
3244 kwargs={ "inMemory": True } )
3245 addedIValues.append( iCounterValue )
3246 iCounterValue += 1
3247 threads.append( t )
3248 t.start()
3249
3250 for t in threads:
3251 t.join()
3252 iCounters.append( t.result )
3253 # Check that counter incremented numController times
3254 iCounterResults = True
3255 for i in addedIValues:
3256 tmpResult = i in iCounters
3257 iCounterResults = iCounterResults and tmpResult
3258 if not tmpResult:
3259 main.log.error( str( i ) + " is not in in-memory "
3260 "counter incremented results" )
3261 utilities.assert_equals( expect=True,
3262 actual=iCounterResults,
3263 onpass="In-memory counter incremented",
3264 onfail="Error incrementing in-memory" +
3265 " counter" )
3266
3267 main.step( "Counters we added have the correct values" )
3268 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3269 utilities.assert_equals( expect=main.TRUE,
3270 actual=incrementCheck,
3271 onpass="Added counters are correct",
3272 onfail="Added counters are incorrect" )
3273
3274 main.step( "Add -8 to then get a in-memory counter on each node" )
3275 iCounters = []
3276 threads = []
3277 addedIValues = []
3278 for i in range( main.numCtrls ):
3279 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3280 name="counterIncrement-" + str( i ),
3281 args=[ iCounterName ],
3282 kwargs={ "delta": -8, "inMemory": True } )
3283 iCounterValue += -8
3284 addedIValues.append( iCounterValue )
3285 threads.append( t )
3286 t.start()
3287
3288 for t in threads:
3289 t.join()
3290 iCounters.append( t.result )
3291 # Check that counter incremented numController times
3292 iCounterResults = True
3293 for i in addedIValues:
3294 tmpResult = i in iCounters
3295 iCounterResults = iCounterResults and tmpResult
3296 if not tmpResult:
3297 main.log.error( str( i ) + " is not in in-memory "
3298 "counter incremented results" )
3299 utilities.assert_equals( expect=True,
3300 actual=pCounterResults,
3301 onpass="In-memory counter incremented",
3302 onfail="Error incrementing in-memory" +
3303 " counter" )
3304
3305 main.step( "Add 5 to then get a in-memory counter on each node" )
3306 iCounters = []
3307 threads = []
3308 addedIValues = []
3309 for i in range( main.numCtrls ):
3310 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3311 name="counterIncrement-" + str( i ),
3312 args=[ iCounterName ],
3313 kwargs={ "delta": 5, "inMemory": True } )
3314 iCounterValue += 5
3315 addedIValues.append( iCounterValue )
3316 threads.append( t )
3317 t.start()
3318
3319 for t in threads:
3320 t.join()
3321 iCounters.append( t.result )
3322 # Check that counter incremented numController times
3323 iCounterResults = True
3324 for i in addedIValues:
3325 tmpResult = i in iCounters
3326 iCounterResults = iCounterResults and tmpResult
3327 if not tmpResult:
3328 main.log.error( str( i ) + " is not in in-memory "
3329 "counter incremented results" )
3330 utilities.assert_equals( expect=True,
3331 actual=pCounterResults,
3332 onpass="In-memory counter incremented",
3333 onfail="Error incrementing in-memory" +
3334 " counter" )
3335
3336 main.step( "Get then add 5 to a in-memory counter on each node" )
3337 iCounters = []
3338 threads = []
3339 addedIValues = []
3340 for i in range( main.numCtrls ):
3341 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3342 name="counterIncrement-" + str( i ),
3343 args=[ iCounterName ],
3344 kwargs={ "delta": 5, "inMemory": True } )
3345 addedIValues.append( iCounterValue )
3346 iCounterValue += 5
3347 threads.append( t )
3348 t.start()
3349
3350 for t in threads:
3351 t.join()
3352 iCounters.append( t.result )
3353 # Check that counter incremented numController times
3354 iCounterResults = True
3355 for i in addedIValues:
3356 tmpResult = i in iCounters
3357 iCounterResults = iCounterResults and tmpResult
3358 if not tmpResult:
3359 main.log.error( str( i ) + " is not in in-memory "
3360 "counter incremented results" )
3361 utilities.assert_equals( expect=True,
3362 actual=iCounterResults,
3363 onpass="In-memory counter incremented",
3364 onfail="Error incrementing in-memory" +
3365 " counter" )
3366
3367 main.step( "Counters we added have the correct values" )
3368 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3369 utilities.assert_equals( expect=main.TRUE,
3370 actual=incrementCheck,
3371 onpass="Added counters are correct",
3372 onfail="Added counters are incorrect" )
3373
Jon Hall5cf14d52015-07-16 12:15:19 -07003374 main.step( "Check counters are consistant across nodes" )
Jon Hall57b50432015-10-22 10:20:10 -07003375 onosCounters, consistentCounterResults = main.Counters.consistentCheck()
Jon Hall5cf14d52015-07-16 12:15:19 -07003376 utilities.assert_equals( expect=main.TRUE,
3377 actual=consistentCounterResults,
3378 onpass="ONOS counters are consistent " +
3379 "across nodes",
3380 onfail="ONOS Counters are inconsistent " +
3381 "across nodes" )
3382
3383 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003384 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3385 incrementCheck = incrementCheck and \
3386 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003387 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003388 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003389 onpass="Added counters are correct",
3390 onfail="Added counters are incorrect" )
3391 # DISTRIBUTED SETS
3392 main.step( "Distributed Set get" )
3393 size = len( onosSet )
3394 getResponses = []
3395 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003396 for i in range( main.numCtrls ):
3397 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003398 name="setTestGet-" + str( i ),
3399 args=[ onosSetName ] )
3400 threads.append( t )
3401 t.start()
3402 for t in threads:
3403 t.join()
3404 getResponses.append( t.result )
3405
3406 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003407 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003408 if isinstance( getResponses[ i ], list):
3409 current = set( getResponses[ i ] )
3410 if len( current ) == len( getResponses[ i ] ):
3411 # no repeats
3412 if onosSet != current:
3413 main.log.error( "ONOS" + str( i + 1 ) +
3414 " has incorrect view" +
3415 " of set " + onosSetName + ":\n" +
3416 str( getResponses[ i ] ) )
3417 main.log.debug( "Expected: " + str( onosSet ) )
3418 main.log.debug( "Actual: " + str( current ) )
3419 getResults = main.FALSE
3420 else:
3421 # error, set is not a set
3422 main.log.error( "ONOS" + str( i + 1 ) +
3423 " has repeat elements in" +
3424 " set " + onosSetName + ":\n" +
3425 str( getResponses[ i ] ) )
3426 getResults = main.FALSE
3427 elif getResponses[ i ] == main.ERROR:
3428 getResults = main.FALSE
3429 utilities.assert_equals( expect=main.TRUE,
3430 actual=getResults,
3431 onpass="Set elements are correct",
3432 onfail="Set elements are incorrect" )
3433
3434 main.step( "Distributed Set size" )
3435 sizeResponses = []
3436 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003437 for i in range( main.numCtrls ):
3438 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003439 name="setTestSize-" + str( i ),
3440 args=[ onosSetName ] )
3441 threads.append( t )
3442 t.start()
3443 for t in threads:
3444 t.join()
3445 sizeResponses.append( t.result )
3446
3447 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003448 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003449 if size != sizeResponses[ i ]:
3450 sizeResults = main.FALSE
3451 main.log.error( "ONOS" + str( i + 1 ) +
3452 " expected a size of " + str( size ) +
3453 " for set " + onosSetName +
3454 " but got " + str( sizeResponses[ i ] ) )
3455 utilities.assert_equals( expect=main.TRUE,
3456 actual=sizeResults,
3457 onpass="Set sizes are correct",
3458 onfail="Set sizes are incorrect" )
3459
3460 main.step( "Distributed Set add()" )
3461 onosSet.add( addValue )
3462 addResponses = []
3463 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003464 for i in range( main.numCtrls ):
3465 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003466 name="setTestAdd-" + str( i ),
3467 args=[ onosSetName, addValue ] )
3468 threads.append( t )
3469 t.start()
3470 for t in threads:
3471 t.join()
3472 addResponses.append( t.result )
3473
3474 # main.TRUE = successfully changed the set
3475 # main.FALSE = action resulted in no change in set
3476 # main.ERROR - Some error in executing the function
3477 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003478 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003479 if addResponses[ i ] == main.TRUE:
3480 # All is well
3481 pass
3482 elif addResponses[ i ] == main.FALSE:
3483 # Already in set, probably fine
3484 pass
3485 elif addResponses[ i ] == main.ERROR:
3486 # Error in execution
3487 addResults = main.FALSE
3488 else:
3489 # unexpected result
3490 addResults = main.FALSE
3491 if addResults != main.TRUE:
3492 main.log.error( "Error executing set add" )
3493
3494 # Check if set is still correct
3495 size = len( onosSet )
3496 getResponses = []
3497 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003498 for i in range( main.numCtrls ):
3499 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003500 name="setTestGet-" + str( i ),
3501 args=[ onosSetName ] )
3502 threads.append( t )
3503 t.start()
3504 for t in threads:
3505 t.join()
3506 getResponses.append( t.result )
3507 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003508 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003509 if isinstance( getResponses[ i ], list):
3510 current = set( getResponses[ i ] )
3511 if len( current ) == len( getResponses[ i ] ):
3512 # no repeats
3513 if onosSet != current:
3514 main.log.error( "ONOS" + str( i + 1 ) +
3515 " has incorrect view" +
3516 " of set " + onosSetName + ":\n" +
3517 str( getResponses[ i ] ) )
3518 main.log.debug( "Expected: " + str( onosSet ) )
3519 main.log.debug( "Actual: " + str( current ) )
3520 getResults = main.FALSE
3521 else:
3522 # error, set is not a set
3523 main.log.error( "ONOS" + str( i + 1 ) +
3524 " has repeat elements in" +
3525 " set " + onosSetName + ":\n" +
3526 str( getResponses[ i ] ) )
3527 getResults = main.FALSE
3528 elif getResponses[ i ] == main.ERROR:
3529 getResults = main.FALSE
3530 sizeResponses = []
3531 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003532 for i in range( main.numCtrls ):
3533 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003534 name="setTestSize-" + str( i ),
3535 args=[ onosSetName ] )
3536 threads.append( t )
3537 t.start()
3538 for t in threads:
3539 t.join()
3540 sizeResponses.append( t.result )
3541 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003542 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003543 if size != sizeResponses[ i ]:
3544 sizeResults = main.FALSE
3545 main.log.error( "ONOS" + str( i + 1 ) +
3546 " expected a size of " + str( size ) +
3547 " for set " + onosSetName +
3548 " but got " + str( sizeResponses[ i ] ) )
3549 addResults = addResults and getResults and sizeResults
3550 utilities.assert_equals( expect=main.TRUE,
3551 actual=addResults,
3552 onpass="Set add correct",
3553 onfail="Set add was incorrect" )
3554
3555 main.step( "Distributed Set addAll()" )
3556 onosSet.update( addAllValue.split() )
3557 addResponses = []
3558 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003559 for i in range( main.numCtrls ):
3560 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003561 name="setTestAddAll-" + str( i ),
3562 args=[ onosSetName, addAllValue ] )
3563 threads.append( t )
3564 t.start()
3565 for t in threads:
3566 t.join()
3567 addResponses.append( t.result )
3568
3569 # main.TRUE = successfully changed the set
3570 # main.FALSE = action resulted in no change in set
3571 # main.ERROR - Some error in executing the function
3572 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003573 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003574 if addResponses[ i ] == main.TRUE:
3575 # All is well
3576 pass
3577 elif addResponses[ i ] == main.FALSE:
3578 # Already in set, probably fine
3579 pass
3580 elif addResponses[ i ] == main.ERROR:
3581 # Error in execution
3582 addAllResults = main.FALSE
3583 else:
3584 # unexpected result
3585 addAllResults = main.FALSE
3586 if addAllResults != main.TRUE:
3587 main.log.error( "Error executing set addAll" )
3588
3589 # Check if set is still correct
3590 size = len( onosSet )
3591 getResponses = []
3592 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003593 for i in range( main.numCtrls ):
3594 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003595 name="setTestGet-" + str( i ),
3596 args=[ onosSetName ] )
3597 threads.append( t )
3598 t.start()
3599 for t in threads:
3600 t.join()
3601 getResponses.append( t.result )
3602 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003603 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003604 if isinstance( getResponses[ i ], list):
3605 current = set( getResponses[ i ] )
3606 if len( current ) == len( getResponses[ i ] ):
3607 # no repeats
3608 if onosSet != current:
3609 main.log.error( "ONOS" + str( i + 1 ) +
3610 " has incorrect view" +
3611 " of set " + onosSetName + ":\n" +
3612 str( getResponses[ i ] ) )
3613 main.log.debug( "Expected: " + str( onosSet ) )
3614 main.log.debug( "Actual: " + str( current ) )
3615 getResults = main.FALSE
3616 else:
3617 # error, set is not a set
3618 main.log.error( "ONOS" + str( i + 1 ) +
3619 " has repeat elements in" +
3620 " set " + onosSetName + ":\n" +
3621 str( getResponses[ i ] ) )
3622 getResults = main.FALSE
3623 elif getResponses[ i ] == main.ERROR:
3624 getResults = main.FALSE
3625 sizeResponses = []
3626 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003627 for i in range( main.numCtrls ):
3628 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003629 name="setTestSize-" + str( i ),
3630 args=[ onosSetName ] )
3631 threads.append( t )
3632 t.start()
3633 for t in threads:
3634 t.join()
3635 sizeResponses.append( t.result )
3636 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003637 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003638 if size != sizeResponses[ i ]:
3639 sizeResults = main.FALSE
3640 main.log.error( "ONOS" + str( i + 1 ) +
3641 " expected a size of " + str( size ) +
3642 " for set " + onosSetName +
3643 " but got " + str( sizeResponses[ i ] ) )
3644 addAllResults = addAllResults and getResults and sizeResults
3645 utilities.assert_equals( expect=main.TRUE,
3646 actual=addAllResults,
3647 onpass="Set addAll correct",
3648 onfail="Set addAll was incorrect" )
3649
3650 main.step( "Distributed Set contains()" )
3651 containsResponses = []
3652 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003653 for i in range( main.numCtrls ):
3654 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003655 name="setContains-" + str( i ),
3656 args=[ onosSetName ],
3657 kwargs={ "values": addValue } )
3658 threads.append( t )
3659 t.start()
3660 for t in threads:
3661 t.join()
3662 # NOTE: This is the tuple
3663 containsResponses.append( t.result )
3664
3665 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003666 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003667 if containsResponses[ i ] == main.ERROR:
3668 containsResults = main.FALSE
3669 else:
3670 containsResults = containsResults and\
3671 containsResponses[ i ][ 1 ]
3672 utilities.assert_equals( expect=main.TRUE,
3673 actual=containsResults,
3674 onpass="Set contains is functional",
3675 onfail="Set contains failed" )
3676
3677 main.step( "Distributed Set containsAll()" )
3678 containsAllResponses = []
3679 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003680 for i in range( main.numCtrls ):
3681 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003682 name="setContainsAll-" + str( i ),
3683 args=[ onosSetName ],
3684 kwargs={ "values": addAllValue } )
3685 threads.append( t )
3686 t.start()
3687 for t in threads:
3688 t.join()
3689 # NOTE: This is the tuple
3690 containsAllResponses.append( t.result )
3691
3692 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003693 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003694 if containsResponses[ i ] == main.ERROR:
3695 containsResults = main.FALSE
3696 else:
3697 containsResults = containsResults and\
3698 containsResponses[ i ][ 1 ]
3699 utilities.assert_equals( expect=main.TRUE,
3700 actual=containsAllResults,
3701 onpass="Set containsAll is functional",
3702 onfail="Set containsAll failed" )
3703
3704 main.step( "Distributed Set remove()" )
3705 onosSet.remove( addValue )
3706 removeResponses = []
3707 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003708 for i in range( main.numCtrls ):
3709 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003710 name="setTestRemove-" + str( i ),
3711 args=[ onosSetName, addValue ] )
3712 threads.append( t )
3713 t.start()
3714 for t in threads:
3715 t.join()
3716 removeResponses.append( t.result )
3717
3718 # main.TRUE = successfully changed the set
3719 # main.FALSE = action resulted in no change in set
3720 # main.ERROR - Some error in executing the function
3721 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003722 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003723 if removeResponses[ i ] == main.TRUE:
3724 # All is well
3725 pass
3726 elif removeResponses[ i ] == main.FALSE:
3727 # not in set, probably fine
3728 pass
3729 elif removeResponses[ i ] == main.ERROR:
3730 # Error in execution
3731 removeResults = main.FALSE
3732 else:
3733 # unexpected result
3734 removeResults = main.FALSE
3735 if removeResults != main.TRUE:
3736 main.log.error( "Error executing set remove" )
3737
3738 # Check if set is still correct
3739 size = len( onosSet )
3740 getResponses = []
3741 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003742 for i in range( main.numCtrls ):
3743 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003744 name="setTestGet-" + str( i ),
3745 args=[ onosSetName ] )
3746 threads.append( t )
3747 t.start()
3748 for t in threads:
3749 t.join()
3750 getResponses.append( t.result )
3751 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003752 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003753 if isinstance( getResponses[ i ], list):
3754 current = set( getResponses[ i ] )
3755 if len( current ) == len( getResponses[ i ] ):
3756 # no repeats
3757 if onosSet != current:
3758 main.log.error( "ONOS" + str( i + 1 ) +
3759 " has incorrect view" +
3760 " of set " + onosSetName + ":\n" +
3761 str( getResponses[ i ] ) )
3762 main.log.debug( "Expected: " + str( onosSet ) )
3763 main.log.debug( "Actual: " + str( current ) )
3764 getResults = main.FALSE
3765 else:
3766 # error, set is not a set
3767 main.log.error( "ONOS" + str( i + 1 ) +
3768 " has repeat elements in" +
3769 " set " + onosSetName + ":\n" +
3770 str( getResponses[ i ] ) )
3771 getResults = main.FALSE
3772 elif getResponses[ i ] == main.ERROR:
3773 getResults = main.FALSE
3774 sizeResponses = []
3775 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003776 for i in range( main.numCtrls ):
3777 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003778 name="setTestSize-" + str( i ),
3779 args=[ onosSetName ] )
3780 threads.append( t )
3781 t.start()
3782 for t in threads:
3783 t.join()
3784 sizeResponses.append( t.result )
3785 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003786 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003787 if size != sizeResponses[ i ]:
3788 sizeResults = main.FALSE
3789 main.log.error( "ONOS" + str( i + 1 ) +
3790 " expected a size of " + str( size ) +
3791 " for set " + onosSetName +
3792 " but got " + str( sizeResponses[ i ] ) )
3793 removeResults = removeResults and getResults and sizeResults
3794 utilities.assert_equals( expect=main.TRUE,
3795 actual=removeResults,
3796 onpass="Set remove correct",
3797 onfail="Set remove was incorrect" )
3798
3799 main.step( "Distributed Set removeAll()" )
3800 onosSet.difference_update( addAllValue.split() )
3801 removeAllResponses = []
3802 threads = []
3803 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003804 for i in range( main.numCtrls ):
3805 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003806 name="setTestRemoveAll-" + str( i ),
3807 args=[ onosSetName, addAllValue ] )
3808 threads.append( t )
3809 t.start()
3810 for t in threads:
3811 t.join()
3812 removeAllResponses.append( t.result )
3813 except Exception, e:
3814 main.log.exception(e)
3815
3816 # main.TRUE = successfully changed the set
3817 # main.FALSE = action resulted in no change in set
3818 # main.ERROR - Some error in executing the function
3819 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003820 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003821 if removeAllResponses[ i ] == main.TRUE:
3822 # All is well
3823 pass
3824 elif removeAllResponses[ i ] == main.FALSE:
3825 # not in set, probably fine
3826 pass
3827 elif removeAllResponses[ i ] == main.ERROR:
3828 # Error in execution
3829 removeAllResults = main.FALSE
3830 else:
3831 # unexpected result
3832 removeAllResults = main.FALSE
3833 if removeAllResults != main.TRUE:
3834 main.log.error( "Error executing set removeAll" )
3835
3836 # Check if set is still correct
3837 size = len( onosSet )
3838 getResponses = []
3839 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003840 for i in range( main.numCtrls ):
3841 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003842 name="setTestGet-" + str( i ),
3843 args=[ onosSetName ] )
3844 threads.append( t )
3845 t.start()
3846 for t in threads:
3847 t.join()
3848 getResponses.append( t.result )
3849 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003850 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003851 if isinstance( getResponses[ i ], list):
3852 current = set( getResponses[ i ] )
3853 if len( current ) == len( getResponses[ i ] ):
3854 # no repeats
3855 if onosSet != current:
3856 main.log.error( "ONOS" + str( i + 1 ) +
3857 " has incorrect view" +
3858 " of set " + onosSetName + ":\n" +
3859 str( getResponses[ i ] ) )
3860 main.log.debug( "Expected: " + str( onosSet ) )
3861 main.log.debug( "Actual: " + str( current ) )
3862 getResults = main.FALSE
3863 else:
3864 # error, set is not a set
3865 main.log.error( "ONOS" + str( i + 1 ) +
3866 " has repeat elements in" +
3867 " set " + onosSetName + ":\n" +
3868 str( getResponses[ i ] ) )
3869 getResults = main.FALSE
3870 elif getResponses[ i ] == main.ERROR:
3871 getResults = main.FALSE
3872 sizeResponses = []
3873 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003874 for i in range( main.numCtrls ):
3875 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003876 name="setTestSize-" + str( i ),
3877 args=[ onosSetName ] )
3878 threads.append( t )
3879 t.start()
3880 for t in threads:
3881 t.join()
3882 sizeResponses.append( t.result )
3883 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003884 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003885 if size != sizeResponses[ i ]:
3886 sizeResults = main.FALSE
3887 main.log.error( "ONOS" + str( i + 1 ) +
3888 " expected a size of " + str( size ) +
3889 " for set " + onosSetName +
3890 " but got " + str( sizeResponses[ i ] ) )
3891 removeAllResults = removeAllResults and getResults and sizeResults
3892 utilities.assert_equals( expect=main.TRUE,
3893 actual=removeAllResults,
3894 onpass="Set removeAll correct",
3895 onfail="Set removeAll was incorrect" )
3896
3897 main.step( "Distributed Set addAll()" )
3898 onosSet.update( addAllValue.split() )
3899 addResponses = []
3900 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003901 for i in range( main.numCtrls ):
3902 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003903 name="setTestAddAll-" + str( i ),
3904 args=[ onosSetName, addAllValue ] )
3905 threads.append( t )
3906 t.start()
3907 for t in threads:
3908 t.join()
3909 addResponses.append( t.result )
3910
3911 # main.TRUE = successfully changed the set
3912 # main.FALSE = action resulted in no change in set
3913 # main.ERROR - Some error in executing the function
3914 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003915 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003916 if addResponses[ i ] == main.TRUE:
3917 # All is well
3918 pass
3919 elif addResponses[ i ] == main.FALSE:
3920 # Already in set, probably fine
3921 pass
3922 elif addResponses[ i ] == main.ERROR:
3923 # Error in execution
3924 addAllResults = main.FALSE
3925 else:
3926 # unexpected result
3927 addAllResults = main.FALSE
3928 if addAllResults != main.TRUE:
3929 main.log.error( "Error executing set addAll" )
3930
3931 # Check if set is still correct
3932 size = len( onosSet )
3933 getResponses = []
3934 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003935 for i in range( main.numCtrls ):
3936 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003937 name="setTestGet-" + str( i ),
3938 args=[ onosSetName ] )
3939 threads.append( t )
3940 t.start()
3941 for t in threads:
3942 t.join()
3943 getResponses.append( t.result )
3944 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003945 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003946 if isinstance( getResponses[ i ], list):
3947 current = set( getResponses[ i ] )
3948 if len( current ) == len( getResponses[ i ] ):
3949 # no repeats
3950 if onosSet != current:
3951 main.log.error( "ONOS" + str( i + 1 ) +
3952 " has incorrect view" +
3953 " of set " + onosSetName + ":\n" +
3954 str( getResponses[ i ] ) )
3955 main.log.debug( "Expected: " + str( onosSet ) )
3956 main.log.debug( "Actual: " + str( current ) )
3957 getResults = main.FALSE
3958 else:
3959 # error, set is not a set
3960 main.log.error( "ONOS" + str( i + 1 ) +
3961 " has repeat elements in" +
3962 " set " + onosSetName + ":\n" +
3963 str( getResponses[ i ] ) )
3964 getResults = main.FALSE
3965 elif getResponses[ i ] == main.ERROR:
3966 getResults = main.FALSE
3967 sizeResponses = []
3968 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003969 for i in range( main.numCtrls ):
3970 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003971 name="setTestSize-" + str( i ),
3972 args=[ onosSetName ] )
3973 threads.append( t )
3974 t.start()
3975 for t in threads:
3976 t.join()
3977 sizeResponses.append( t.result )
3978 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003979 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003980 if size != sizeResponses[ i ]:
3981 sizeResults = main.FALSE
3982 main.log.error( "ONOS" + str( i + 1 ) +
3983 " expected a size of " + str( size ) +
3984 " for set " + onosSetName +
3985 " but got " + str( sizeResponses[ i ] ) )
3986 addAllResults = addAllResults and getResults and sizeResults
3987 utilities.assert_equals( expect=main.TRUE,
3988 actual=addAllResults,
3989 onpass="Set addAll correct",
3990 onfail="Set addAll was incorrect" )
3991
3992 main.step( "Distributed Set clear()" )
3993 onosSet.clear()
3994 clearResponses = []
3995 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003996 for i in range( main.numCtrls ):
3997 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003998 name="setTestClear-" + str( i ),
3999 args=[ onosSetName, " "], # Values doesn't matter
4000 kwargs={ "clear": True } )
4001 threads.append( t )
4002 t.start()
4003 for t in threads:
4004 t.join()
4005 clearResponses.append( t.result )
4006
4007 # main.TRUE = successfully changed the set
4008 # main.FALSE = action resulted in no change in set
4009 # main.ERROR - Some error in executing the function
4010 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004011 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004012 if clearResponses[ i ] == main.TRUE:
4013 # All is well
4014 pass
4015 elif clearResponses[ i ] == main.FALSE:
4016 # Nothing set, probably fine
4017 pass
4018 elif clearResponses[ i ] == main.ERROR:
4019 # Error in execution
4020 clearResults = main.FALSE
4021 else:
4022 # unexpected result
4023 clearResults = main.FALSE
4024 if clearResults != main.TRUE:
4025 main.log.error( "Error executing set clear" )
4026
4027 # Check if set is still correct
4028 size = len( onosSet )
4029 getResponses = []
4030 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004031 for i in range( main.numCtrls ):
4032 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004033 name="setTestGet-" + str( i ),
4034 args=[ onosSetName ] )
4035 threads.append( t )
4036 t.start()
4037 for t in threads:
4038 t.join()
4039 getResponses.append( t.result )
4040 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004041 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004042 if isinstance( getResponses[ i ], list):
4043 current = set( getResponses[ i ] )
4044 if len( current ) == len( getResponses[ i ] ):
4045 # no repeats
4046 if onosSet != current:
4047 main.log.error( "ONOS" + str( i + 1 ) +
4048 " has incorrect view" +
4049 " of set " + onosSetName + ":\n" +
4050 str( getResponses[ i ] ) )
4051 main.log.debug( "Expected: " + str( onosSet ) )
4052 main.log.debug( "Actual: " + str( current ) )
4053 getResults = main.FALSE
4054 else:
4055 # error, set is not a set
4056 main.log.error( "ONOS" + str( i + 1 ) +
4057 " has repeat elements in" +
4058 " set " + onosSetName + ":\n" +
4059 str( getResponses[ i ] ) )
4060 getResults = main.FALSE
4061 elif getResponses[ i ] == main.ERROR:
4062 getResults = main.FALSE
4063 sizeResponses = []
4064 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004065 for i in range( main.numCtrls ):
4066 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004067 name="setTestSize-" + str( i ),
4068 args=[ onosSetName ] )
4069 threads.append( t )
4070 t.start()
4071 for t in threads:
4072 t.join()
4073 sizeResponses.append( t.result )
4074 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004075 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004076 if size != sizeResponses[ i ]:
4077 sizeResults = main.FALSE
4078 main.log.error( "ONOS" + str( i + 1 ) +
4079 " expected a size of " + str( size ) +
4080 " for set " + onosSetName +
4081 " but got " + str( sizeResponses[ i ] ) )
4082 clearResults = clearResults and getResults and sizeResults
4083 utilities.assert_equals( expect=main.TRUE,
4084 actual=clearResults,
4085 onpass="Set clear correct",
4086 onfail="Set clear was incorrect" )
4087
4088 main.step( "Distributed Set addAll()" )
4089 onosSet.update( addAllValue.split() )
4090 addResponses = []
4091 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004092 for i in range( main.numCtrls ):
4093 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004094 name="setTestAddAll-" + str( i ),
4095 args=[ onosSetName, addAllValue ] )
4096 threads.append( t )
4097 t.start()
4098 for t in threads:
4099 t.join()
4100 addResponses.append( t.result )
4101
4102 # main.TRUE = successfully changed the set
4103 # main.FALSE = action resulted in no change in set
4104 # main.ERROR - Some error in executing the function
4105 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004106 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004107 if addResponses[ i ] == main.TRUE:
4108 # All is well
4109 pass
4110 elif addResponses[ i ] == main.FALSE:
4111 # Already in set, probably fine
4112 pass
4113 elif addResponses[ i ] == main.ERROR:
4114 # Error in execution
4115 addAllResults = main.FALSE
4116 else:
4117 # unexpected result
4118 addAllResults = main.FALSE
4119 if addAllResults != main.TRUE:
4120 main.log.error( "Error executing set addAll" )
4121
4122 # Check if set is still correct
4123 size = len( onosSet )
4124 getResponses = []
4125 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004126 for i in range( main.numCtrls ):
4127 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004128 name="setTestGet-" + str( i ),
4129 args=[ onosSetName ] )
4130 threads.append( t )
4131 t.start()
4132 for t in threads:
4133 t.join()
4134 getResponses.append( t.result )
4135 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004136 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004137 if isinstance( getResponses[ i ], list):
4138 current = set( getResponses[ i ] )
4139 if len( current ) == len( getResponses[ i ] ):
4140 # no repeats
4141 if onosSet != current:
4142 main.log.error( "ONOS" + str( i + 1 ) +
4143 " has incorrect view" +
4144 " of set " + onosSetName + ":\n" +
4145 str( getResponses[ i ] ) )
4146 main.log.debug( "Expected: " + str( onosSet ) )
4147 main.log.debug( "Actual: " + str( current ) )
4148 getResults = main.FALSE
4149 else:
4150 # error, set is not a set
4151 main.log.error( "ONOS" + str( i + 1 ) +
4152 " has repeat elements in" +
4153 " set " + onosSetName + ":\n" +
4154 str( getResponses[ i ] ) )
4155 getResults = main.FALSE
4156 elif getResponses[ i ] == main.ERROR:
4157 getResults = main.FALSE
4158 sizeResponses = []
4159 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004160 for i in range( main.numCtrls ):
4161 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004162 name="setTestSize-" + str( i ),
4163 args=[ onosSetName ] )
4164 threads.append( t )
4165 t.start()
4166 for t in threads:
4167 t.join()
4168 sizeResponses.append( t.result )
4169 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004170 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004171 if size != sizeResponses[ i ]:
4172 sizeResults = main.FALSE
4173 main.log.error( "ONOS" + str( i + 1 ) +
4174 " expected a size of " + str( size ) +
4175 " for set " + onosSetName +
4176 " but got " + str( sizeResponses[ i ] ) )
4177 addAllResults = addAllResults and getResults and sizeResults
4178 utilities.assert_equals( expect=main.TRUE,
4179 actual=addAllResults,
4180 onpass="Set addAll correct",
4181 onfail="Set addAll was incorrect" )
4182
4183 main.step( "Distributed Set retain()" )
4184 onosSet.intersection_update( retainValue.split() )
4185 retainResponses = []
4186 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004187 for i in range( main.numCtrls ):
4188 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004189 name="setTestRetain-" + str( i ),
4190 args=[ onosSetName, retainValue ],
4191 kwargs={ "retain": True } )
4192 threads.append( t )
4193 t.start()
4194 for t in threads:
4195 t.join()
4196 retainResponses.append( t.result )
4197
4198 # main.TRUE = successfully changed the set
4199 # main.FALSE = action resulted in no change in set
4200 # main.ERROR - Some error in executing the function
4201 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004202 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004203 if retainResponses[ i ] == main.TRUE:
4204 # All is well
4205 pass
4206 elif retainResponses[ i ] == main.FALSE:
4207 # Already in set, probably fine
4208 pass
4209 elif retainResponses[ i ] == main.ERROR:
4210 # Error in execution
4211 retainResults = main.FALSE
4212 else:
4213 # unexpected result
4214 retainResults = main.FALSE
4215 if retainResults != main.TRUE:
4216 main.log.error( "Error executing set retain" )
4217
4218 # Check if set is still correct
4219 size = len( onosSet )
4220 getResponses = []
4221 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004222 for i in range( main.numCtrls ):
4223 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004224 name="setTestGet-" + str( i ),
4225 args=[ onosSetName ] )
4226 threads.append( t )
4227 t.start()
4228 for t in threads:
4229 t.join()
4230 getResponses.append( t.result )
4231 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004232 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004233 if isinstance( getResponses[ i ], list):
4234 current = set( getResponses[ i ] )
4235 if len( current ) == len( getResponses[ i ] ):
4236 # no repeats
4237 if onosSet != current:
4238 main.log.error( "ONOS" + str( i + 1 ) +
4239 " has incorrect view" +
4240 " of set " + onosSetName + ":\n" +
4241 str( getResponses[ i ] ) )
4242 main.log.debug( "Expected: " + str( onosSet ) )
4243 main.log.debug( "Actual: " + str( current ) )
4244 getResults = main.FALSE
4245 else:
4246 # error, set is not a set
4247 main.log.error( "ONOS" + str( i + 1 ) +
4248 " has repeat elements in" +
4249 " set " + onosSetName + ":\n" +
4250 str( getResponses[ i ] ) )
4251 getResults = main.FALSE
4252 elif getResponses[ i ] == main.ERROR:
4253 getResults = main.FALSE
4254 sizeResponses = []
4255 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004256 for i in range( main.numCtrls ):
4257 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004258 name="setTestSize-" + str( i ),
4259 args=[ onosSetName ] )
4260 threads.append( t )
4261 t.start()
4262 for t in threads:
4263 t.join()
4264 sizeResponses.append( t.result )
4265 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004266 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004267 if size != sizeResponses[ i ]:
4268 sizeResults = main.FALSE
4269 main.log.error( "ONOS" + str( i + 1 ) +
4270 " expected a size of " +
4271 str( size ) + " for set " + onosSetName +
4272 " but got " + str( sizeResponses[ i ] ) )
4273 retainResults = retainResults and getResults and sizeResults
4274 utilities.assert_equals( expect=main.TRUE,
4275 actual=retainResults,
4276 onpass="Set retain correct",
4277 onfail="Set retain was incorrect" )
4278
Jon Hall2a5002c2015-08-21 16:49:11 -07004279 # Transactional maps
4280 main.step( "Partitioned Transactional maps put" )
4281 tMapValue = "Testing"
4282 numKeys = 100
4283 putResult = True
4284 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4285 if len( putResponses ) == 100:
4286 for i in putResponses:
4287 if putResponses[ i ][ 'value' ] != tMapValue:
4288 putResult = False
4289 else:
4290 putResult = False
4291 if not putResult:
4292 main.log.debug( "Put response values: " + str( putResponses ) )
4293 utilities.assert_equals( expect=True,
4294 actual=putResult,
4295 onpass="Partitioned Transactional Map put successful",
4296 onfail="Partitioned Transactional Map put values are incorrect" )
4297
4298 main.step( "Partitioned Transactional maps get" )
4299 getCheck = True
4300 for n in range( 1, numKeys + 1 ):
4301 getResponses = []
4302 threads = []
4303 valueCheck = True
4304 for i in range( main.numCtrls ):
4305 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4306 name="TMap-get-" + str( i ),
4307 args=[ "Key" + str ( n ) ] )
4308 threads.append( t )
4309 t.start()
4310 for t in threads:
4311 t.join()
4312 getResponses.append( t.result )
4313 for node in getResponses:
4314 if node != tMapValue:
4315 valueCheck = False
4316 if not valueCheck:
4317 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4318 main.log.warn( getResponses )
4319 getCheck = getCheck and valueCheck
4320 utilities.assert_equals( expect=True,
4321 actual=getCheck,
4322 onpass="Partitioned Transactional Map get values were correct",
4323 onfail="Partitioned Transactional Map values incorrect" )
4324
4325 main.step( "In-memory Transactional maps put" )
4326 tMapValue = "Testing"
4327 numKeys = 100
4328 putResult = True
4329 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4330 if len( putResponses ) == 100:
4331 for i in putResponses:
4332 if putResponses[ i ][ 'value' ] != tMapValue:
4333 putResult = False
4334 else:
4335 putResult = False
4336 if not putResult:
4337 main.log.debug( "Put response values: " + str( putResponses ) )
4338 utilities.assert_equals( expect=True,
4339 actual=putResult,
4340 onpass="In-Memory Transactional Map put successful",
4341 onfail="In-Memory Transactional Map put values are incorrect" )
4342
4343 main.step( "In-Memory Transactional maps get" )
4344 getCheck = True
4345 for n in range( 1, numKeys + 1 ):
4346 getResponses = []
4347 threads = []
4348 valueCheck = True
4349 for i in range( main.numCtrls ):
4350 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4351 name="TMap-get-" + str( i ),
4352 args=[ "Key" + str ( n ) ],
4353 kwargs={ "inMemory": True } )
4354 threads.append( t )
4355 t.start()
4356 for t in threads:
4357 t.join()
4358 getResponses.append( t.result )
4359 for node in getResponses:
4360 if node != tMapValue:
4361 valueCheck = False
4362 if not valueCheck:
4363 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4364 main.log.warn( getResponses )
4365 getCheck = getCheck and valueCheck
4366 utilities.assert_equals( expect=True,
4367 actual=getCheck,
4368 onpass="In-Memory Transactional Map get values were correct",
4369 onfail="In-Memory Transactional Map values incorrect" )