blob: 624b7201f12b75084cfbc1e57668a8cc04f0c954 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
28class HAsanity:
29
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080052 import time
Jon Hall5cf14d52015-07-16 12:15:19 -070053 main.log.info( "ONOS HA Sanity test - initialization" )
54 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070055 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070056 "installing ONOS, starting Mininet and ONOS" +\
57 "cli sessions."
58 # TODO: save all the timers and output them for plotting
59
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070071 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070072 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070073 global ONOS1Port
74 global ONOS2Port
75 global ONOS3Port
76 global ONOS4Port
77 global ONOS5Port
78 global ONOS6Port
79 global ONOS7Port
80
81 # FIXME: just get controller port from params?
82 # TODO: do we really need all these?
83 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
84 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
85 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
86 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
87 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
88 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
89 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
90
Jon Halle1a3b752015-07-22 13:02:46 -070091 try:
92 fileName = "Counters"
93 path = main.params[ 'imports' ][ 'path' ]
94 main.Counters = imp.load_source( fileName,
95 path + fileName + ".py" )
96 except Exception as e:
97 main.log.exception( e )
98 main.cleanup()
99 main.exit()
100
101 main.CLIs = []
102 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700103 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700104 for i in range( 1, main.numCtrls + 1 ):
105 try:
106 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
107 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
108 ipList.append( main.nodes[ -1 ].ip_address )
109 except AttributeError:
110 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700111
112 main.step( "Create cell file" )
113 cellAppString = main.params[ 'ENV' ][ 'appString' ]
114 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
115 main.Mininet1.ip_address,
116 cellAppString, ipList )
117 main.step( "Applying cell variable to environment" )
118 cellResult = main.ONOSbench.setCell( cellName )
119 verifyResult = main.ONOSbench.verifyCell()
120
121 # FIXME:this is short term fix
122 main.log.info( "Removing raft logs" )
123 main.ONOSbench.onosRemoveRaftLogs()
124
125 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700126 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700127 main.ONOSbench.onosUninstall( node.ip_address )
128
129 # Make sure ONOS is DEAD
130 main.log.info( "Killing any ONOS processes" )
131 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700132 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700133 killed = main.ONOSbench.onosKill( node.ip_address )
134 killResults = killResults and killed
135
136 cleanInstallResult = main.TRUE
137 gitPullResult = main.TRUE
138
139 main.step( "Starting Mininet" )
140 # scp topo file to mininet
141 # TODO: move to params?
142 topoName = "obelisk.py"
143 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700144 main.ONOSbench.scp( main.Mininet1,
145 filePath + topoName,
146 main.Mininet1.home,
147 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700148 mnResult = main.Mininet1.startNet( )
149 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
150 onpass="Mininet Started",
151 onfail="Error starting Mininet" )
152
153 main.step( "Git checkout and pull " + gitBranch )
154 if PULLCODE:
155 main.ONOSbench.gitCheckout( gitBranch )
156 gitPullResult = main.ONOSbench.gitPull()
157 # values of 1 or 3 are good
158 utilities.assert_lesser( expect=0, actual=gitPullResult,
159 onpass="Git pull successful",
160 onfail="Git pull failed" )
161 main.ONOSbench.getVersion( report=True )
162
163 main.step( "Using mvn clean install" )
164 cleanInstallResult = main.TRUE
165 if PULLCODE and gitPullResult == main.TRUE:
166 cleanInstallResult = main.ONOSbench.cleanInstall()
167 else:
168 main.log.warn( "Did not pull new code so skipping mvn " +
169 "clean install" )
170 utilities.assert_equals( expect=main.TRUE,
171 actual=cleanInstallResult,
172 onpass="MCI successful",
173 onfail="MCI failed" )
174 # GRAPHS
175 # NOTE: important params here:
176 # job = name of Jenkins job
177 # Plot Name = Plot-HA, only can be used if multiple plots
178 # index = The number of the graph under plot name
179 job = "HAsanity"
180 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700181 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700182 graphs = '<ac:structured-macro ac:name="html">\n'
183 graphs += '<ac:plain-text-body><![CDATA[\n'
184 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800185 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700186 '&width=500&height=300"' +\
187 'noborder="0" width="500" height="300" scrolling="yes" ' +\
188 'seamless="seamless"></iframe>\n'
189 graphs += ']]></ac:plain-text-body>\n'
190 graphs += '</ac:structured-macro>\n'
191 main.log.wiki(graphs)
192
193 main.step( "Creating ONOS package" )
194 packageResult = main.ONOSbench.onosPackage()
195 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
196 onpass="ONOS package successful",
197 onfail="ONOS package failed" )
198
199 main.step( "Installing ONOS package" )
200 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700201 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700202 tmpResult = main.ONOSbench.onosInstall( options="-f",
203 node=node.ip_address )
204 onosInstallResult = onosInstallResult and tmpResult
205 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
206 onpass="ONOS install successful",
207 onfail="ONOS install failed" )
208
209 main.step( "Checking if ONOS is up yet" )
210 for i in range( 2 ):
211 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700212 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700213 started = main.ONOSbench.isup( node.ip_address )
214 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800215 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700216 onosIsupResult = onosIsupResult and started
217 if onosIsupResult == main.TRUE:
218 break
219 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
220 onpass="ONOS startup successful",
221 onfail="ONOS startup failed" )
222
223 main.log.step( "Starting ONOS CLI sessions" )
224 cliResults = main.TRUE
225 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700226 for i in range( main.numCtrls ):
227 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700228 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700229 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700230 threads.append( t )
231 t.start()
232
233 for t in threads:
234 t.join()
235 cliResults = cliResults and t.result
236 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
237 onpass="ONOS cli startup successful",
238 onfail="ONOS cli startup failed" )
239
240 if main.params[ 'tcpdump' ].lower() == "true":
241 main.step( "Start Packet Capture MN" )
242 main.Mininet2.startTcpdump(
243 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
244 + "-MN.pcap",
245 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
246 port=main.params[ 'MNtcpdump' ][ 'port' ] )
247
248 main.step( "App Ids check" )
Jon Hallf3d16e72015-12-16 17:45:08 -0800249 time.sleep(60)
Jon Hall5cf14d52015-07-16 12:15:19 -0700250 appCheck = main.TRUE
251 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700252 for i in range( main.numCtrls ):
253 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700254 name="appToIDCheck-" + str( i ),
255 args=[] )
256 threads.append( t )
257 t.start()
258
259 for t in threads:
260 t.join()
261 appCheck = appCheck and t.result
262 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700263 main.log.warn( main.CLIs[0].apps() )
264 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700265 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
266 onpass="App Ids seem to be correct",
267 onfail="Something is wrong with app Ids" )
268
269 if cliResults == main.FALSE:
270 main.log.error( "Failed to start ONOS, stopping test" )
271 main.cleanup()
272 main.exit()
273
274 def CASE2( self, main ):
275 """
276 Assign devices to controllers
277 """
278 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700279 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700280 assert main, "main not defined"
281 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700282 assert main.CLIs, "main.CLIs not defined"
283 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700284 assert ONOS1Port, "ONOS1Port not defined"
285 assert ONOS2Port, "ONOS2Port not defined"
286 assert ONOS3Port, "ONOS3Port not defined"
287 assert ONOS4Port, "ONOS4Port not defined"
288 assert ONOS5Port, "ONOS5Port not defined"
289 assert ONOS6Port, "ONOS6Port not defined"
290 assert ONOS7Port, "ONOS7Port not defined"
291
292 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700293 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700294 "and check that an ONOS node becomes the " +\
295 "master of the device."
296 main.step( "Assign switches to controllers" )
297
298 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700299 for i in range( main.numCtrls ):
300 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700301 swList = []
302 for i in range( 1, 29 ):
303 swList.append( "s" + str( i ) )
304 main.Mininet1.assignSwController( sw=swList, ip=ipList )
305
306 mastershipCheck = main.TRUE
307 for i in range( 1, 29 ):
308 response = main.Mininet1.getSwController( "s" + str( i ) )
309 try:
310 main.log.info( str( response ) )
311 except Exception:
312 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700313 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700314 if re.search( "tcp:" + node.ip_address, response ):
315 mastershipCheck = mastershipCheck and main.TRUE
316 else:
317 main.log.error( "Error, node " + node.ip_address + " is " +
318 "not in the list of controllers s" +
319 str( i ) + " is connecting to." )
320 mastershipCheck = main.FALSE
321 utilities.assert_equals(
322 expect=main.TRUE,
323 actual=mastershipCheck,
324 onpass="Switch mastership assigned correctly",
325 onfail="Switches not assigned correctly to controllers" )
326
327 def CASE21( self, main ):
328 """
329 Assign mastership to controllers
330 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700331 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700332 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700333 assert main, "main not defined"
334 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700335 assert main.CLIs, "main.CLIs not defined"
336 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700337 assert ONOS1Port, "ONOS1Port not defined"
338 assert ONOS2Port, "ONOS2Port not defined"
339 assert ONOS3Port, "ONOS3Port not defined"
340 assert ONOS4Port, "ONOS4Port not defined"
341 assert ONOS5Port, "ONOS5Port not defined"
342 assert ONOS6Port, "ONOS6Port not defined"
343 assert ONOS7Port, "ONOS7Port not defined"
344
345 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700346 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700347 "device. Then manually assign" +\
348 " mastership to specific ONOS nodes using" +\
349 " 'device-role'"
350 main.step( "Assign mastership of switches to specific controllers" )
351 # Manually assign mastership to the controller we want
352 roleCall = main.TRUE
353
354 ipList = [ ]
355 deviceList = []
356 try:
357 # Assign mastership to specific controllers. This assignment was
358 # determined for a 7 node cluser, but will work with any sized
359 # cluster
360 for i in range( 1, 29 ): # switches 1 through 28
361 # set up correct variables:
362 if i == 1:
363 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700364 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700365 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
366 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700367 c = 1 % main.numCtrls
368 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700369 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
370 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700371 c = 1 % main.numCtrls
372 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hall5cf14d52015-07-16 12:15:19 -0700373 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
374 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700375 c = 3 % main.numCtrls
376 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hall5cf14d52015-07-16 12:15:19 -0700377 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
378 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700379 c = 2 % main.numCtrls
380 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700381 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
382 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700383 c = 2 % main.numCtrls
384 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hall5cf14d52015-07-16 12:15:19 -0700385 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
386 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700387 c = 5 % main.numCtrls
388 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hall5cf14d52015-07-16 12:15:19 -0700389 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
390 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700391 c = 4 % main.numCtrls
392 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700393 dpid = '3' + str( i ).zfill( 3 )
394 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
395 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700396 c = 6 % main.numCtrls
397 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700398 dpid = '6' + str( i ).zfill( 3 )
399 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
400 elif i == 28:
401 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700402 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hall5cf14d52015-07-16 12:15:19 -0700403 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
404 else:
405 main.log.error( "You didn't write an else statement for " +
406 "switch s" + str( i ) )
407 roleCall = main.FALSE
408 # Assign switch
409 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
410 # TODO: make this controller dynamic
411 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
412 ip )
413 ipList.append( ip )
414 deviceList.append( deviceId )
415 except ( AttributeError, AssertionError ):
416 main.log.exception( "Something is wrong with ONOS device view" )
417 main.log.info( main.ONOScli1.devices() )
418 utilities.assert_equals(
419 expect=main.TRUE,
420 actual=roleCall,
421 onpass="Re-assigned switch mastership to designated controller",
422 onfail="Something wrong with deviceRole calls" )
423
424 main.step( "Check mastership was correctly assigned" )
425 roleCheck = main.TRUE
426 # NOTE: This is due to the fact that device mastership change is not
427 # atomic and is actually a multi step process
428 time.sleep( 5 )
429 for i in range( len( ipList ) ):
430 ip = ipList[i]
431 deviceId = deviceList[i]
432 # Check assignment
433 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
434 if ip in master:
435 roleCheck = roleCheck and main.TRUE
436 else:
437 roleCheck = roleCheck and main.FALSE
438 main.log.error( "Error, controller " + ip + " is not" +
439 " master " + "of device " +
440 str( deviceId ) + ". Master is " +
441 repr( master ) + "." )
442 utilities.assert_equals(
443 expect=main.TRUE,
444 actual=roleCheck,
445 onpass="Switches were successfully reassigned to designated " +
446 "controller",
447 onfail="Switches were not successfully reassigned" )
448
449 def CASE3( self, main ):
450 """
451 Assign intents
452 """
453 import time
454 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700455 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700456 assert main, "main not defined"
457 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700458 assert main.CLIs, "main.CLIs not defined"
459 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700460 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700461 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700462 "assign predetermined host-to-host intents." +\
463 " After installation, check that the intent" +\
464 " is distributed to all nodes and the state" +\
465 " is INSTALLED"
466
467 # install onos-app-fwd
468 main.step( "Install reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700469 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700470 utilities.assert_equals( expect=main.TRUE, actual=installResults,
471 onpass="Install fwd successful",
472 onfail="Install fwd failed" )
473
474 main.step( "Check app ids" )
475 appCheck = main.TRUE
476 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700477 for i in range( main.numCtrls ):
478 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700479 name="appToIDCheck-" + str( i ),
480 args=[] )
481 threads.append( t )
482 t.start()
483
484 for t in threads:
485 t.join()
486 appCheck = appCheck and t.result
487 if appCheck != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700488 main.log.warn( main.CLIs[0].apps() )
489 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700490 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
491 onpass="App Ids seem to be correct",
492 onfail="Something is wrong with app Ids" )
493
494 main.step( "Discovering Hosts( Via pingall for now )" )
495 # FIXME: Once we have a host discovery mechanism, use that instead
496 # REACTIVE FWD test
497 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700498 passMsg = "Reactive Pingall test passed"
499 time1 = time.time()
500 pingResult = main.Mininet1.pingall()
501 time2 = time.time()
502 if not pingResult:
503 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700504 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700505 passMsg += " on the second try"
506 utilities.assert_equals(
507 expect=main.TRUE,
508 actual=pingResult,
509 onpass= passMsg,
510 onfail="Reactive Pingall failed, " +
511 "one or more ping pairs failed" )
512 main.log.info( "Time for pingall: %2f seconds" %
513 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700514 # timeout for fwd flows
515 time.sleep( 11 )
516 # uninstall onos-app-fwd
517 main.step( "Uninstall reactive forwarding app" )
Jon Halle1a3b752015-07-22 13:02:46 -0700518 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700519 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
520 onpass="Uninstall fwd successful",
521 onfail="Uninstall fwd failed" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700522
523 main.step( "Check app ids" )
524 threads = []
525 appCheck2 = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700526 for i in range( main.numCtrls ):
527 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700528 name="appToIDCheck-" + str( i ),
529 args=[] )
530 threads.append( t )
531 t.start()
532
533 for t in threads:
534 t.join()
535 appCheck2 = appCheck2 and t.result
536 if appCheck2 != main.TRUE:
Jon Halle1a3b752015-07-22 13:02:46 -0700537 main.log.warn( main.CLIs[0].apps() )
538 main.log.warn( main.CLIs[0].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700539 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
540 onpass="App Ids seem to be correct",
541 onfail="Something is wrong with app Ids" )
542
543 main.step( "Add host intents via cli" )
544 intentIds = []
Jon Hall6e709752016-02-01 13:38:46 -0800545 # TODO: move the host numbers to params
546 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700547 intentAddResult = True
548 hostResult = main.TRUE
549 for i in range( 8, 18 ):
550 main.log.info( "Adding host intent between h" + str( i ) +
551 " and h" + str( i + 10 ) )
552 host1 = "00:00:00:00:00:" + \
553 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
554 host2 = "00:00:00:00:00:" + \
555 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
556 # NOTE: getHost can return None
557 host1Dict = main.ONOScli1.getHost( host1 )
558 host2Dict = main.ONOScli1.getHost( host2 )
559 host1Id = None
560 host2Id = None
561 if host1Dict and host2Dict:
562 host1Id = host1Dict.get( 'id', None )
563 host2Id = host2Dict.get( 'id', None )
564 if host1Id and host2Id:
Jon Halle1a3b752015-07-22 13:02:46 -0700565 nodeNum = ( i % main.numCtrls )
566 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700567 if tmpId:
568 main.log.info( "Added intent with id: " + tmpId )
569 intentIds.append( tmpId )
570 else:
571 main.log.error( "addHostIntent returned: " +
572 repr( tmpId ) )
573 else:
574 main.log.error( "Error, getHost() failed for h" + str( i ) +
575 " and/or h" + str( i + 10 ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700576 hosts = main.CLIs[ 0 ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700577 main.log.warn( "Hosts output: " )
578 try:
579 main.log.warn( json.dumps( json.loads( hosts ),
580 sort_keys=True,
581 indent=4,
582 separators=( ',', ': ' ) ) )
583 except ( ValueError, TypeError ):
584 main.log.warn( repr( hosts ) )
585 hostResult = main.FALSE
586 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
587 onpass="Found a host id for each host",
588 onfail="Error looking up host ids" )
589
590 intentStart = time.time()
591 onosIds = main.ONOScli1.getAllIntentsId()
592 main.log.info( "Submitted intents: " + str( intentIds ) )
593 main.log.info( "Intents in ONOS: " + str( onosIds ) )
594 for intent in intentIds:
595 if intent in onosIds:
596 pass # intent submitted is in onos
597 else:
598 intentAddResult = False
599 if intentAddResult:
600 intentStop = time.time()
601 else:
602 intentStop = None
603 # Print the intent states
604 intents = main.ONOScli1.intents()
605 intentStates = []
606 installedCheck = True
607 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
608 count = 0
609 try:
610 for intent in json.loads( intents ):
611 state = intent.get( 'state', None )
612 if "INSTALLED" not in state:
613 installedCheck = False
614 intentId = intent.get( 'id', None )
615 intentStates.append( ( intentId, state ) )
616 except ( ValueError, TypeError ):
617 main.log.exception( "Error parsing intents" )
618 # add submitted intents not in the store
619 tmplist = [ i for i, s in intentStates ]
620 missingIntents = False
621 for i in intentIds:
622 if i not in tmplist:
623 intentStates.append( ( i, " - " ) )
624 missingIntents = True
625 intentStates.sort()
626 for i, s in intentStates:
627 count += 1
628 main.log.info( "%-6s%-15s%-15s" %
629 ( str( count ), str( i ), str( s ) ) )
630 leaders = main.ONOScli1.leaders()
631 try:
632 missing = False
633 if leaders:
634 parsedLeaders = json.loads( leaders )
635 main.log.warn( json.dumps( parsedLeaders,
636 sort_keys=True,
637 indent=4,
638 separators=( ',', ': ' ) ) )
639 # check for all intent partitions
640 topics = []
641 for i in range( 14 ):
642 topics.append( "intent-partition-" + str( i ) )
643 main.log.debug( topics )
644 ONOStopics = [ j['topic'] for j in parsedLeaders ]
645 for topic in topics:
646 if topic not in ONOStopics:
647 main.log.error( "Error: " + topic +
648 " not in leaders" )
649 missing = True
650 else:
651 main.log.error( "leaders() returned None" )
652 except ( ValueError, TypeError ):
653 main.log.exception( "Error parsing leaders" )
654 main.log.error( repr( leaders ) )
655 # Check all nodes
656 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700657 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700658 response = node.leaders( jsonFormat=False)
659 main.log.warn( str( node.name ) + " leaders output: \n" +
660 str( response ) )
661
662 partitions = main.ONOScli1.partitions()
663 try:
664 if partitions :
665 parsedPartitions = json.loads( partitions )
666 main.log.warn( json.dumps( parsedPartitions,
667 sort_keys=True,
668 indent=4,
669 separators=( ',', ': ' ) ) )
670 # TODO check for a leader in all paritions
671 # TODO check for consistency among nodes
672 else:
673 main.log.error( "partitions() returned None" )
674 except ( ValueError, TypeError ):
675 main.log.exception( "Error parsing partitions" )
676 main.log.error( repr( partitions ) )
677 pendingMap = main.ONOScli1.pendingMap()
678 try:
679 if pendingMap :
680 parsedPending = json.loads( pendingMap )
681 main.log.warn( json.dumps( parsedPending,
682 sort_keys=True,
683 indent=4,
684 separators=( ',', ': ' ) ) )
685 # TODO check something here?
686 else:
687 main.log.error( "pendingMap() returned None" )
688 except ( ValueError, TypeError ):
689 main.log.exception( "Error parsing pending map" )
690 main.log.error( repr( pendingMap ) )
691
692 intentAddResult = bool( intentAddResult and not missingIntents and
693 installedCheck )
694 if not intentAddResult:
695 main.log.error( "Error in pushing host intents to ONOS" )
696
697 main.step( "Intent Anti-Entropy dispersion" )
698 for i in range(100):
699 correct = True
700 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700701 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700702 onosIds = []
703 ids = cli.getAllIntentsId()
704 onosIds.append( ids )
705 main.log.debug( "Intents in " + cli.name + ": " +
706 str( sorted( onosIds ) ) )
707 if sorted( ids ) != sorted( intentIds ):
708 main.log.warn( "Set of intent IDs doesn't match" )
709 correct = False
710 break
711 else:
712 intents = json.loads( cli.intents() )
713 for intent in intents:
714 if intent[ 'state' ] != "INSTALLED":
715 main.log.warn( "Intent " + intent[ 'id' ] +
716 " is " + intent[ 'state' ] )
717 correct = False
718 break
719 if correct:
720 break
721 else:
722 time.sleep(1)
723 if not intentStop:
724 intentStop = time.time()
725 global gossipTime
726 gossipTime = intentStop - intentStart
727 main.log.info( "It took about " + str( gossipTime ) +
728 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700729 gossipPeriod = int( main.params['timers']['gossip'] )
730 maxGossipTime = gossipPeriod * len( main.nodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700731 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700732 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700733 onpass="ECM anti-entropy for intents worked within " +
734 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700735 onfail="Intent ECM anti-entropy took too long. " +
736 "Expected time:{}, Actual time:{}".format( maxGossipTime,
737 gossipTime ) )
738 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700739 intentAddResult = True
740
741 if not intentAddResult or "key" in pendingMap:
742 import time
743 installedCheck = True
744 main.log.info( "Sleeping 60 seconds to see if intents are found" )
745 time.sleep( 60 )
746 onosIds = main.ONOScli1.getAllIntentsId()
747 main.log.info( "Submitted intents: " + str( intentIds ) )
748 main.log.info( "Intents in ONOS: " + str( onosIds ) )
749 # Print the intent states
750 intents = main.ONOScli1.intents()
751 intentStates = []
752 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
753 count = 0
754 try:
755 for intent in json.loads( intents ):
756 # Iter through intents of a node
757 state = intent.get( 'state', None )
758 if "INSTALLED" not in state:
759 installedCheck = False
760 intentId = intent.get( 'id', None )
761 intentStates.append( ( intentId, state ) )
762 except ( ValueError, TypeError ):
763 main.log.exception( "Error parsing intents" )
764 # add submitted intents not in the store
765 tmplist = [ i for i, s in intentStates ]
766 for i in intentIds:
767 if i not in tmplist:
768 intentStates.append( ( i, " - " ) )
769 intentStates.sort()
770 for i, s in intentStates:
771 count += 1
772 main.log.info( "%-6s%-15s%-15s" %
773 ( str( count ), str( i ), str( s ) ) )
774 leaders = main.ONOScli1.leaders()
775 try:
776 missing = False
777 if leaders:
778 parsedLeaders = json.loads( leaders )
779 main.log.warn( json.dumps( parsedLeaders,
780 sort_keys=True,
781 indent=4,
782 separators=( ',', ': ' ) ) )
783 # check for all intent partitions
784 # check for election
785 topics = []
786 for i in range( 14 ):
787 topics.append( "intent-partition-" + str( i ) )
788 # FIXME: this should only be after we start the app
789 topics.append( "org.onosproject.election" )
790 main.log.debug( topics )
791 ONOStopics = [ j['topic'] for j in parsedLeaders ]
792 for topic in topics:
793 if topic not in ONOStopics:
794 main.log.error( "Error: " + topic +
795 " not in leaders" )
796 missing = True
797 else:
798 main.log.error( "leaders() returned None" )
799 except ( ValueError, TypeError ):
800 main.log.exception( "Error parsing leaders" )
801 main.log.error( repr( leaders ) )
802 # Check all nodes
803 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -0700804 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700805 response = node.leaders( jsonFormat=False)
806 main.log.warn( str( node.name ) + " leaders output: \n" +
807 str( response ) )
808
809 partitions = main.ONOScli1.partitions()
810 try:
811 if partitions :
812 parsedPartitions = json.loads( partitions )
813 main.log.warn( json.dumps( parsedPartitions,
814 sort_keys=True,
815 indent=4,
816 separators=( ',', ': ' ) ) )
817 # TODO check for a leader in all paritions
818 # TODO check for consistency among nodes
819 else:
820 main.log.error( "partitions() returned None" )
821 except ( ValueError, TypeError ):
822 main.log.exception( "Error parsing partitions" )
823 main.log.error( repr( partitions ) )
824 pendingMap = main.ONOScli1.pendingMap()
825 try:
826 if pendingMap :
827 parsedPending = json.loads( pendingMap )
828 main.log.warn( json.dumps( parsedPending,
829 sort_keys=True,
830 indent=4,
831 separators=( ',', ': ' ) ) )
832 # TODO check something here?
833 else:
834 main.log.error( "pendingMap() returned None" )
835 except ( ValueError, TypeError ):
836 main.log.exception( "Error parsing pending map" )
837 main.log.error( repr( pendingMap ) )
838
839 def CASE4( self, main ):
840 """
841 Ping across added host intents
842 """
843 import json
844 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700845 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700846 assert main, "main not defined"
847 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700848 assert main.CLIs, "main.CLIs not defined"
849 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700850 main.case( "Verify connectivity by sendind traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700851 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700852 "functionality and check the state of " +\
853 "the intent"
854 main.step( "Ping across added host intents" )
855 PingResult = main.TRUE
856 for i in range( 8, 18 ):
857 ping = main.Mininet1.pingHost( src="h" + str( i ),
858 target="h" + str( i + 10 ) )
859 PingResult = PingResult and ping
860 if ping == main.FALSE:
861 main.log.warn( "Ping failed between h" + str( i ) +
862 " and h" + str( i + 10 ) )
863 elif ping == main.TRUE:
864 main.log.info( "Ping test passed!" )
865 # Don't set PingResult or you'd override failures
866 if PingResult == main.FALSE:
867 main.log.error(
868 "Intents have not been installed correctly, pings failed." )
869 # TODO: pretty print
870 main.log.warn( "ONOS1 intents: " )
871 try:
872 tmpIntents = main.ONOScli1.intents()
873 main.log.warn( json.dumps( json.loads( tmpIntents ),
874 sort_keys=True,
875 indent=4,
876 separators=( ',', ': ' ) ) )
877 except ( ValueError, TypeError ):
878 main.log.warn( repr( tmpIntents ) )
879 utilities.assert_equals(
880 expect=main.TRUE,
881 actual=PingResult,
882 onpass="Intents have been installed correctly and pings work",
883 onfail="Intents have not been installed correctly, pings failed." )
884
885 main.step( "Check Intent state" )
886 installedCheck = False
887 loopCount = 0
888 while not installedCheck and loopCount < 40:
889 installedCheck = True
890 # Print the intent states
891 intents = main.ONOScli1.intents()
892 intentStates = []
893 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
894 count = 0
895 # Iter through intents of a node
896 try:
897 for intent in json.loads( intents ):
898 state = intent.get( 'state', None )
899 if "INSTALLED" not in state:
900 installedCheck = False
901 intentId = intent.get( 'id', None )
902 intentStates.append( ( intentId, state ) )
903 except ( ValueError, TypeError ):
904 main.log.exception( "Error parsing intents." )
905 # Print states
906 intentStates.sort()
907 for i, s in intentStates:
908 count += 1
909 main.log.info( "%-6s%-15s%-15s" %
910 ( str( count ), str( i ), str( s ) ) )
911 if not installedCheck:
912 time.sleep( 1 )
913 loopCount += 1
914 utilities.assert_equals( expect=True, actual=installedCheck,
915 onpass="Intents are all INSTALLED",
916 onfail="Intents are not all in " +
917 "INSTALLED state" )
918
919 main.step( "Check leadership of topics" )
920 leaders = main.ONOScli1.leaders()
921 topicCheck = main.TRUE
922 try:
923 if leaders:
924 parsedLeaders = json.loads( leaders )
925 main.log.warn( json.dumps( parsedLeaders,
926 sort_keys=True,
927 indent=4,
928 separators=( ',', ': ' ) ) )
929 # check for all intent partitions
930 # check for election
931 # TODO: Look at Devices as topics now that it uses this system
932 topics = []
933 for i in range( 14 ):
934 topics.append( "intent-partition-" + str( i ) )
935 # FIXME: this should only be after we start the app
936 # FIXME: topics.append( "org.onosproject.election" )
937 # Print leaders output
938 main.log.debug( topics )
939 ONOStopics = [ j['topic'] for j in parsedLeaders ]
940 for topic in topics:
941 if topic not in ONOStopics:
942 main.log.error( "Error: " + topic +
943 " not in leaders" )
944 topicCheck = main.FALSE
945 else:
946 main.log.error( "leaders() returned None" )
947 topicCheck = main.FALSE
948 except ( ValueError, TypeError ):
949 topicCheck = main.FALSE
950 main.log.exception( "Error parsing leaders" )
951 main.log.error( repr( leaders ) )
952 # TODO: Check for a leader of these topics
953 # Check all nodes
954 if topicCheck:
Jon Halle1a3b752015-07-22 13:02:46 -0700955 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -0700956 response = node.leaders( jsonFormat=False)
957 main.log.warn( str( node.name ) + " leaders output: \n" +
958 str( response ) )
959
960 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
961 onpass="intent Partitions is in leaders",
962 onfail="Some topics were lost " )
963 # Print partitions
964 partitions = main.ONOScli1.partitions()
965 try:
966 if partitions :
967 parsedPartitions = json.loads( partitions )
968 main.log.warn( json.dumps( parsedPartitions,
969 sort_keys=True,
970 indent=4,
971 separators=( ',', ': ' ) ) )
972 # TODO check for a leader in all paritions
973 # TODO check for consistency among nodes
974 else:
975 main.log.error( "partitions() returned None" )
976 except ( ValueError, TypeError ):
977 main.log.exception( "Error parsing partitions" )
978 main.log.error( repr( partitions ) )
979 # Print Pending Map
980 pendingMap = main.ONOScli1.pendingMap()
981 try:
982 if pendingMap :
983 parsedPending = json.loads( pendingMap )
984 main.log.warn( json.dumps( parsedPending,
985 sort_keys=True,
986 indent=4,
987 separators=( ',', ': ' ) ) )
988 # TODO check something here?
989 else:
990 main.log.error( "pendingMap() returned None" )
991 except ( ValueError, TypeError ):
992 main.log.exception( "Error parsing pending map" )
993 main.log.error( repr( pendingMap ) )
994
995 if not installedCheck:
996 main.log.info( "Waiting 60 seconds to see if the state of " +
997 "intents change" )
998 time.sleep( 60 )
999 # Print the intent states
1000 intents = main.ONOScli1.intents()
1001 intentStates = []
1002 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1003 count = 0
1004 # Iter through intents of a node
1005 try:
1006 for intent in json.loads( intents ):
1007 state = intent.get( 'state', None )
1008 if "INSTALLED" not in state:
1009 installedCheck = False
1010 intentId = intent.get( 'id', None )
1011 intentStates.append( ( intentId, state ) )
1012 except ( ValueError, TypeError ):
1013 main.log.exception( "Error parsing intents." )
1014 intentStates.sort()
1015 for i, s in intentStates:
1016 count += 1
1017 main.log.info( "%-6s%-15s%-15s" %
1018 ( str( count ), str( i ), str( s ) ) )
1019 leaders = main.ONOScli1.leaders()
1020 try:
1021 missing = False
1022 if leaders:
1023 parsedLeaders = json.loads( leaders )
1024 main.log.warn( json.dumps( parsedLeaders,
1025 sort_keys=True,
1026 indent=4,
1027 separators=( ',', ': ' ) ) )
1028 # check for all intent partitions
1029 # check for election
1030 topics = []
1031 for i in range( 14 ):
1032 topics.append( "intent-partition-" + str( i ) )
1033 # FIXME: this should only be after we start the app
1034 topics.append( "org.onosproject.election" )
1035 main.log.debug( topics )
1036 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1037 for topic in topics:
1038 if topic not in ONOStopics:
1039 main.log.error( "Error: " + topic +
1040 " not in leaders" )
1041 missing = True
1042 else:
1043 main.log.error( "leaders() returned None" )
1044 except ( ValueError, TypeError ):
1045 main.log.exception( "Error parsing leaders" )
1046 main.log.error( repr( leaders ) )
1047 if missing:
Jon Halle1a3b752015-07-22 13:02:46 -07001048 for node in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07001049 response = node.leaders( jsonFormat=False)
1050 main.log.warn( str( node.name ) + " leaders output: \n" +
1051 str( response ) )
1052
1053 partitions = main.ONOScli1.partitions()
1054 try:
1055 if partitions :
1056 parsedPartitions = json.loads( partitions )
1057 main.log.warn( json.dumps( parsedPartitions,
1058 sort_keys=True,
1059 indent=4,
1060 separators=( ',', ': ' ) ) )
1061 # TODO check for a leader in all paritions
1062 # TODO check for consistency among nodes
1063 else:
1064 main.log.error( "partitions() returned None" )
1065 except ( ValueError, TypeError ):
1066 main.log.exception( "Error parsing partitions" )
1067 main.log.error( repr( partitions ) )
1068 pendingMap = main.ONOScli1.pendingMap()
1069 try:
1070 if pendingMap :
1071 parsedPending = json.loads( pendingMap )
1072 main.log.warn( json.dumps( parsedPending,
1073 sort_keys=True,
1074 indent=4,
1075 separators=( ',', ': ' ) ) )
1076 # TODO check something here?
1077 else:
1078 main.log.error( "pendingMap() returned None" )
1079 except ( ValueError, TypeError ):
1080 main.log.exception( "Error parsing pending map" )
1081 main.log.error( repr( pendingMap ) )
1082 # Print flowrules
Jon Halle1a3b752015-07-22 13:02:46 -07001083 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001084 main.step( "Wait a minute then ping again" )
1085 # the wait is above
1086 PingResult = main.TRUE
1087 for i in range( 8, 18 ):
1088 ping = main.Mininet1.pingHost( src="h" + str( i ),
1089 target="h" + str( i + 10 ) )
1090 PingResult = PingResult and ping
1091 if ping == main.FALSE:
1092 main.log.warn( "Ping failed between h" + str( i ) +
1093 " and h" + str( i + 10 ) )
1094 elif ping == main.TRUE:
1095 main.log.info( "Ping test passed!" )
1096 # Don't set PingResult or you'd override failures
1097 if PingResult == main.FALSE:
1098 main.log.error(
1099 "Intents have not been installed correctly, pings failed." )
1100 # TODO: pretty print
1101 main.log.warn( "ONOS1 intents: " )
1102 try:
1103 tmpIntents = main.ONOScli1.intents()
1104 main.log.warn( json.dumps( json.loads( tmpIntents ),
1105 sort_keys=True,
1106 indent=4,
1107 separators=( ',', ': ' ) ) )
1108 except ( ValueError, TypeError ):
1109 main.log.warn( repr( tmpIntents ) )
1110 utilities.assert_equals(
1111 expect=main.TRUE,
1112 actual=PingResult,
1113 onpass="Intents have been installed correctly and pings work",
1114 onfail="Intents have not been installed correctly, pings failed." )
1115
1116 def CASE5( self, main ):
1117 """
1118 Reading state of ONOS
1119 """
1120 import json
1121 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001122 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001123 assert main, "main not defined"
1124 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001125 assert main.CLIs, "main.CLIs not defined"
1126 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001127
1128 main.case( "Setting up and gathering data for current state" )
1129 # The general idea for this test case is to pull the state of
1130 # ( intents,flows, topology,... ) from each ONOS node
1131 # We can then compare them with each other and also with past states
1132
1133 main.step( "Check that each switch has a master" )
1134 global mastershipState
1135 mastershipState = '[]'
1136
1137 # Assert that each device has a master
1138 rolesNotNull = main.TRUE
1139 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001140 for i in range( main.numCtrls ):
1141 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001142 name="rolesNotNull-" + str( i ),
1143 args=[] )
1144 threads.append( t )
1145 t.start()
1146
1147 for t in threads:
1148 t.join()
1149 rolesNotNull = rolesNotNull and t.result
1150 utilities.assert_equals(
1151 expect=main.TRUE,
1152 actual=rolesNotNull,
1153 onpass="Each device has a master",
1154 onfail="Some devices don't have a master assigned" )
1155
1156 main.step( "Get the Mastership of each switch from each controller" )
1157 ONOSMastership = []
1158 mastershipCheck = main.FALSE
1159 consistentMastership = True
1160 rolesResults = True
1161 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001162 for i in range( main.numCtrls ):
1163 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001164 name="roles-" + str( i ),
1165 args=[] )
1166 threads.append( t )
1167 t.start()
1168
1169 for t in threads:
1170 t.join()
1171 ONOSMastership.append( t.result )
1172
Jon Halle1a3b752015-07-22 13:02:46 -07001173 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001174 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1175 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1176 " roles" )
1177 main.log.warn(
1178 "ONOS" + str( i + 1 ) + " mastership response: " +
1179 repr( ONOSMastership[i] ) )
1180 rolesResults = False
1181 utilities.assert_equals(
1182 expect=True,
1183 actual=rolesResults,
1184 onpass="No error in reading roles output",
1185 onfail="Error in reading roles from ONOS" )
1186
1187 main.step( "Check for consistency in roles from each controller" )
1188 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1189 main.log.info(
1190 "Switch roles are consistent across all ONOS nodes" )
1191 else:
1192 consistentMastership = False
1193 utilities.assert_equals(
1194 expect=True,
1195 actual=consistentMastership,
1196 onpass="Switch roles are consistent across all ONOS nodes",
1197 onfail="ONOS nodes have different views of switch roles" )
1198
1199 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001200 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001201 try:
1202 main.log.warn(
1203 "ONOS" + str( i + 1 ) + " roles: ",
1204 json.dumps(
1205 json.loads( ONOSMastership[ i ] ),
1206 sort_keys=True,
1207 indent=4,
1208 separators=( ',', ': ' ) ) )
1209 except ( ValueError, TypeError ):
1210 main.log.warn( repr( ONOSMastership[ i ] ) )
1211 elif rolesResults and consistentMastership:
1212 mastershipCheck = main.TRUE
1213 mastershipState = ONOSMastership[ 0 ]
1214
1215 main.step( "Get the intents from each controller" )
1216 global intentState
1217 intentState = []
1218 ONOSIntents = []
1219 intentCheck = main.FALSE
1220 consistentIntents = True
1221 intentsResults = True
1222 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001223 for i in range( main.numCtrls ):
1224 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001225 name="intents-" + str( i ),
1226 args=[],
1227 kwargs={ 'jsonFormat': True } )
1228 threads.append( t )
1229 t.start()
1230
1231 for t in threads:
1232 t.join()
1233 ONOSIntents.append( t.result )
1234
Jon Halle1a3b752015-07-22 13:02:46 -07001235 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001236 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1237 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1238 " intents" )
1239 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1240 repr( ONOSIntents[ i ] ) )
1241 intentsResults = False
1242 utilities.assert_equals(
1243 expect=True,
1244 actual=intentsResults,
1245 onpass="No error in reading intents output",
1246 onfail="Error in reading intents from ONOS" )
1247
1248 main.step( "Check for consistency in Intents from each controller" )
1249 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1250 main.log.info( "Intents are consistent across all ONOS " +
1251 "nodes" )
1252 else:
1253 consistentIntents = False
1254 main.log.error( "Intents not consistent" )
1255 utilities.assert_equals(
1256 expect=True,
1257 actual=consistentIntents,
1258 onpass="Intents are consistent across all ONOS nodes",
1259 onfail="ONOS nodes have different views of intents" )
1260
1261 if intentsResults:
1262 # Try to make it easy to figure out what is happening
1263 #
1264 # Intent ONOS1 ONOS2 ...
1265 # 0x01 INSTALLED INSTALLING
1266 # ... ... ...
1267 # ... ... ...
1268 title = " Id"
Jon Halle1a3b752015-07-22 13:02:46 -07001269 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001270 title += " " * 10 + "ONOS" + str( n + 1 )
1271 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001272 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001273 keys = []
1274 try:
1275 # Get the set of all intent keys
1276 for nodeStr in ONOSIntents:
1277 node = json.loads( nodeStr )
1278 for intent in node:
1279 keys.append( intent.get( 'id' ) )
1280 keys = set( keys )
1281 # For each intent key, print the state on each node
1282 for key in keys:
1283 row = "%-13s" % key
1284 for nodeStr in ONOSIntents:
1285 node = json.loads( nodeStr )
1286 for intent in node:
1287 if intent.get( 'id', "Error" ) == key:
1288 row += "%-15s" % intent.get( 'state' )
1289 main.log.warn( row )
1290 # End of intent state table
1291 except ValueError as e:
1292 main.log.exception( e )
1293 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1294
1295 if intentsResults and not consistentIntents:
1296 # print the json objects
1297 n = len(ONOSIntents)
1298 main.log.debug( "ONOS" + str( n ) + " intents: " )
1299 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1300 sort_keys=True,
1301 indent=4,
1302 separators=( ',', ': ' ) ) )
Jon Halle1a3b752015-07-22 13:02:46 -07001303 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001304 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1305 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " )
1306 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1307 sort_keys=True,
1308 indent=4,
1309 separators=( ',', ': ' ) ) )
1310 else:
Jon Halle1a3b752015-07-22 13:02:46 -07001311 main.log.debug( main.nodes[ i ].name + " intents match ONOS" +
Jon Hall5cf14d52015-07-16 12:15:19 -07001312 str( n ) + " intents" )
1313 elif intentsResults and consistentIntents:
1314 intentCheck = main.TRUE
1315 intentState = ONOSIntents[ 0 ]
1316
1317 main.step( "Get the flows from each controller" )
1318 global flowState
1319 flowState = []
1320 ONOSFlows = []
1321 ONOSFlowsJson = []
1322 flowCheck = main.FALSE
1323 consistentFlows = True
1324 flowsResults = True
1325 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001326 for i in range( main.numCtrls ):
1327 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001328 name="flows-" + str( i ),
1329 args=[],
1330 kwargs={ 'jsonFormat': True } )
1331 threads.append( t )
1332 t.start()
1333
1334 # NOTE: Flows command can take some time to run
1335 time.sleep(30)
1336 for t in threads:
1337 t.join()
1338 result = t.result
1339 ONOSFlows.append( result )
1340
Jon Halle1a3b752015-07-22 13:02:46 -07001341 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001342 num = str( i + 1 )
1343 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1344 main.log.error( "Error in getting ONOS" + num + " flows" )
1345 main.log.warn( "ONOS" + num + " flows response: " +
1346 repr( ONOSFlows[ i ] ) )
1347 flowsResults = False
1348 ONOSFlowsJson.append( None )
1349 else:
1350 try:
1351 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1352 except ( ValueError, TypeError ):
1353 # FIXME: change this to log.error?
1354 main.log.exception( "Error in parsing ONOS" + num +
1355 " response as json." )
1356 main.log.error( repr( ONOSFlows[ i ] ) )
1357 ONOSFlowsJson.append( None )
1358 flowsResults = False
1359 utilities.assert_equals(
1360 expect=True,
1361 actual=flowsResults,
1362 onpass="No error in reading flows output",
1363 onfail="Error in reading flows from ONOS" )
1364
1365 main.step( "Check for consistency in Flows from each controller" )
1366 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1367 if all( tmp ):
1368 main.log.info( "Flow count is consistent across all ONOS nodes" )
1369 else:
1370 consistentFlows = False
1371 utilities.assert_equals(
1372 expect=True,
1373 actual=consistentFlows,
1374 onpass="The flow count is consistent across all ONOS nodes",
1375 onfail="ONOS nodes have different flow counts" )
1376
1377 if flowsResults and not consistentFlows:
Jon Halle1a3b752015-07-22 13:02:46 -07001378 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001379 try:
1380 main.log.warn(
1381 "ONOS" + str( i + 1 ) + " flows: " +
1382 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1383 indent=4, separators=( ',', ': ' ) ) )
1384 except ( ValueError, TypeError ):
1385 main.log.warn(
1386 "ONOS" + str( i + 1 ) + " flows: " +
1387 repr( ONOSFlows[ i ] ) )
1388 elif flowsResults and consistentFlows:
1389 flowCheck = main.TRUE
1390 flowState = ONOSFlows[ 0 ]
1391
1392 main.step( "Get the OF Table entries" )
1393 global flows
1394 flows = []
1395 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001396 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001397 if flowCheck == main.FALSE:
1398 for table in flows:
1399 main.log.warn( table )
1400 # TODO: Compare switch flow tables with ONOS flow tables
1401
1402 main.step( "Start continuous pings" )
1403 main.Mininet2.pingLong(
1404 src=main.params[ 'PING' ][ 'source1' ],
1405 target=main.params[ 'PING' ][ 'target1' ],
1406 pingTime=500 )
1407 main.Mininet2.pingLong(
1408 src=main.params[ 'PING' ][ 'source2' ],
1409 target=main.params[ 'PING' ][ 'target2' ],
1410 pingTime=500 )
1411 main.Mininet2.pingLong(
1412 src=main.params[ 'PING' ][ 'source3' ],
1413 target=main.params[ 'PING' ][ 'target3' ],
1414 pingTime=500 )
1415 main.Mininet2.pingLong(
1416 src=main.params[ 'PING' ][ 'source4' ],
1417 target=main.params[ 'PING' ][ 'target4' ],
1418 pingTime=500 )
1419 main.Mininet2.pingLong(
1420 src=main.params[ 'PING' ][ 'source5' ],
1421 target=main.params[ 'PING' ][ 'target5' ],
1422 pingTime=500 )
1423 main.Mininet2.pingLong(
1424 src=main.params[ 'PING' ][ 'source6' ],
1425 target=main.params[ 'PING' ][ 'target6' ],
1426 pingTime=500 )
1427 main.Mininet2.pingLong(
1428 src=main.params[ 'PING' ][ 'source7' ],
1429 target=main.params[ 'PING' ][ 'target7' ],
1430 pingTime=500 )
1431 main.Mininet2.pingLong(
1432 src=main.params[ 'PING' ][ 'source8' ],
1433 target=main.params[ 'PING' ][ 'target8' ],
1434 pingTime=500 )
1435 main.Mininet2.pingLong(
1436 src=main.params[ 'PING' ][ 'source9' ],
1437 target=main.params[ 'PING' ][ 'target9' ],
1438 pingTime=500 )
1439 main.Mininet2.pingLong(
1440 src=main.params[ 'PING' ][ 'source10' ],
1441 target=main.params[ 'PING' ][ 'target10' ],
1442 pingTime=500 )
1443
1444 main.step( "Collecting topology information from ONOS" )
1445 devices = []
1446 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001447 for i in range( main.numCtrls ):
1448 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001449 name="devices-" + str( i ),
1450 args=[ ] )
1451 threads.append( t )
1452 t.start()
1453
1454 for t in threads:
1455 t.join()
1456 devices.append( t.result )
1457 hosts = []
1458 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001459 for i in range( main.numCtrls ):
1460 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001461 name="hosts-" + str( i ),
1462 args=[ ] )
1463 threads.append( t )
1464 t.start()
1465
1466 for t in threads:
1467 t.join()
1468 try:
1469 hosts.append( json.loads( t.result ) )
1470 except ( ValueError, TypeError ):
1471 # FIXME: better handling of this, print which node
1472 # Maybe use thread name?
1473 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001474 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001475 hosts.append( None )
1476
1477 ports = []
1478 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001479 for i in range( main.numCtrls ):
1480 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001481 name="ports-" + str( i ),
1482 args=[ ] )
1483 threads.append( t )
1484 t.start()
1485
1486 for t in threads:
1487 t.join()
1488 ports.append( t.result )
1489 links = []
1490 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001491 for i in range( main.numCtrls ):
1492 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001493 name="links-" + str( i ),
1494 args=[ ] )
1495 threads.append( t )
1496 t.start()
1497
1498 for t in threads:
1499 t.join()
1500 links.append( t.result )
1501 clusters = []
1502 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001503 for i in range( main.numCtrls ):
1504 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001505 name="clusters-" + str( i ),
1506 args=[ ] )
1507 threads.append( t )
1508 t.start()
1509
1510 for t in threads:
1511 t.join()
1512 clusters.append( t.result )
1513 # Compare json objects for hosts and dataplane clusters
1514
1515 # hosts
1516 main.step( "Host view is consistent across ONOS nodes" )
1517 consistentHostsResult = main.TRUE
1518 for controller in range( len( hosts ) ):
1519 controllerStr = str( controller + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001520 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001521 if hosts[ controller ] == hosts[ 0 ]:
1522 continue
1523 else: # hosts not consistent
1524 main.log.error( "hosts from ONOS" +
1525 controllerStr +
1526 " is inconsistent with ONOS1" )
1527 main.log.warn( repr( hosts[ controller ] ) )
1528 consistentHostsResult = main.FALSE
1529
1530 else:
1531 main.log.error( "Error in getting ONOS hosts from ONOS" +
1532 controllerStr )
1533 consistentHostsResult = main.FALSE
1534 main.log.warn( "ONOS" + controllerStr +
1535 " hosts response: " +
1536 repr( hosts[ controller ] ) )
1537 utilities.assert_equals(
1538 expect=main.TRUE,
1539 actual=consistentHostsResult,
1540 onpass="Hosts view is consistent across all ONOS nodes",
1541 onfail="ONOS nodes have different views of hosts" )
1542
1543 main.step( "Each host has an IP address" )
1544 ipResult = main.TRUE
1545 for controller in range( 0, len( hosts ) ):
1546 controllerStr = str( controller + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001547 if hosts[ controller ]:
1548 for host in hosts[ controller ]:
1549 if not host.get( 'ipAddresses', [ ] ):
1550 main.log.error( "Error with host ips on controller" +
1551 controllerStr + ": " + str( host ) )
1552 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001553 utilities.assert_equals(
1554 expect=main.TRUE,
1555 actual=ipResult,
1556 onpass="The ips of the hosts aren't empty",
1557 onfail="The ip of at least one host is missing" )
1558
1559 # Strongly connected clusters of devices
1560 main.step( "Cluster view is consistent across ONOS nodes" )
1561 consistentClustersResult = main.TRUE
1562 for controller in range( len( clusters ) ):
1563 controllerStr = str( controller + 1 )
1564 if "Error" not in clusters[ controller ]:
1565 if clusters[ controller ] == clusters[ 0 ]:
1566 continue
1567 else: # clusters not consistent
1568 main.log.error( "clusters from ONOS" + controllerStr +
1569 " is inconsistent with ONOS1" )
1570 consistentClustersResult = main.FALSE
1571
1572 else:
1573 main.log.error( "Error in getting dataplane clusters " +
1574 "from ONOS" + controllerStr )
1575 consistentClustersResult = main.FALSE
1576 main.log.warn( "ONOS" + controllerStr +
1577 " clusters response: " +
1578 repr( clusters[ controller ] ) )
1579 utilities.assert_equals(
1580 expect=main.TRUE,
1581 actual=consistentClustersResult,
1582 onpass="Clusters view is consistent across all ONOS nodes",
1583 onfail="ONOS nodes have different views of clusters" )
1584 # there should always only be one cluster
1585 main.step( "Cluster view correct across ONOS nodes" )
1586 try:
1587 numClusters = len( json.loads( clusters[ 0 ] ) )
1588 except ( ValueError, TypeError ):
1589 main.log.exception( "Error parsing clusters[0]: " +
1590 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001591 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001592 clusterResults = main.FALSE
1593 if numClusters == 1:
1594 clusterResults = main.TRUE
1595 utilities.assert_equals(
1596 expect=1,
1597 actual=numClusters,
1598 onpass="ONOS shows 1 SCC",
1599 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1600
1601 main.step( "Comparing ONOS topology to MN" )
1602 devicesResults = main.TRUE
1603 linksResults = main.TRUE
1604 hostsResults = main.TRUE
1605 mnSwitches = main.Mininet1.getSwitches()
1606 mnLinks = main.Mininet1.getLinks()
1607 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001608 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001609 controllerStr = str( controller + 1 )
1610 if devices[ controller ] and ports[ controller ] and\
1611 "Error" not in devices[ controller ] and\
1612 "Error" not in ports[ controller ]:
1613
1614 currentDevicesResult = main.Mininet1.compareSwitches(
1615 mnSwitches,
1616 json.loads( devices[ controller ] ),
1617 json.loads( ports[ controller ] ) )
1618 else:
1619 currentDevicesResult = main.FALSE
1620 utilities.assert_equals( expect=main.TRUE,
1621 actual=currentDevicesResult,
1622 onpass="ONOS" + controllerStr +
1623 " Switches view is correct",
1624 onfail="ONOS" + controllerStr +
1625 " Switches view is incorrect" )
1626 if links[ controller ] and "Error" not in links[ controller ]:
1627 currentLinksResult = main.Mininet1.compareLinks(
1628 mnSwitches, mnLinks,
1629 json.loads( links[ controller ] ) )
1630 else:
1631 currentLinksResult = main.FALSE
1632 utilities.assert_equals( expect=main.TRUE,
1633 actual=currentLinksResult,
1634 onpass="ONOS" + controllerStr +
1635 " links view is correct",
1636 onfail="ONOS" + controllerStr +
1637 " links view is incorrect" )
1638
Jon Hall657cdf62015-12-17 14:40:51 -08001639 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001640 currentHostsResult = main.Mininet1.compareHosts(
1641 mnHosts,
1642 hosts[ controller ] )
1643 else:
1644 currentHostsResult = main.FALSE
1645 utilities.assert_equals( expect=main.TRUE,
1646 actual=currentHostsResult,
1647 onpass="ONOS" + controllerStr +
1648 " hosts exist in Mininet",
1649 onfail="ONOS" + controllerStr +
1650 " hosts don't match Mininet" )
1651
1652 devicesResults = devicesResults and currentDevicesResult
1653 linksResults = linksResults and currentLinksResult
1654 hostsResults = hostsResults and currentHostsResult
1655
1656 main.step( "Device information is correct" )
1657 utilities.assert_equals(
1658 expect=main.TRUE,
1659 actual=devicesResults,
1660 onpass="Device information is correct",
1661 onfail="Device information is incorrect" )
1662
1663 main.step( "Links are correct" )
1664 utilities.assert_equals(
1665 expect=main.TRUE,
1666 actual=linksResults,
1667 onpass="Link are correct",
1668 onfail="Links are incorrect" )
1669
1670 main.step( "Hosts are correct" )
1671 utilities.assert_equals(
1672 expect=main.TRUE,
1673 actual=hostsResults,
1674 onpass="Hosts are correct",
1675 onfail="Hosts are incorrect" )
1676
1677 def CASE6( self, main ):
1678 """
1679 The Failure case. Since this is the Sanity test, we do nothing.
1680 """
1681 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001682 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001683 assert main, "main not defined"
1684 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001685 assert main.CLIs, "main.CLIs not defined"
1686 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001687 main.case( "Wait 60 seconds instead of inducing a failure" )
1688 time.sleep( 60 )
1689 utilities.assert_equals(
1690 expect=main.TRUE,
1691 actual=main.TRUE,
1692 onpass="Sleeping 60 seconds",
1693 onfail="Something is terribly wrong with my math" )
1694
1695 def CASE7( self, main ):
1696 """
1697 Check state after ONOS failure
1698 """
1699 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001700 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001701 assert main, "main not defined"
1702 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001703 assert main.CLIs, "main.CLIs not defined"
1704 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001705 main.case( "Running ONOS Constant State Tests" )
1706
1707 main.step( "Check that each switch has a master" )
1708 # Assert that each device has a master
1709 rolesNotNull = main.TRUE
1710 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001711 for i in range( main.numCtrls ):
1712 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001713 name="rolesNotNull-" + str( i ),
1714 args=[ ] )
1715 threads.append( t )
1716 t.start()
1717
1718 for t in threads:
1719 t.join()
1720 rolesNotNull = rolesNotNull and t.result
1721 utilities.assert_equals(
1722 expect=main.TRUE,
1723 actual=rolesNotNull,
1724 onpass="Each device has a master",
1725 onfail="Some devices don't have a master assigned" )
1726
1727 main.step( "Read device roles from ONOS" )
1728 ONOSMastership = []
1729 mastershipCheck = main.FALSE
1730 consistentMastership = True
1731 rolesResults = True
1732 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001733 for i in range( main.numCtrls ):
1734 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001735 name="roles-" + str( i ),
1736 args=[] )
1737 threads.append( t )
1738 t.start()
1739
1740 for t in threads:
1741 t.join()
1742 ONOSMastership.append( t.result )
1743
Jon Halle1a3b752015-07-22 13:02:46 -07001744 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001745 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1746 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1747 " roles" )
1748 main.log.warn(
1749 "ONOS" + str( i + 1 ) + " mastership response: " +
1750 repr( ONOSMastership[i] ) )
1751 rolesResults = False
1752 utilities.assert_equals(
1753 expect=True,
1754 actual=rolesResults,
1755 onpass="No error in reading roles output",
1756 onfail="Error in reading roles from ONOS" )
1757
1758 main.step( "Check for consistency in roles from each controller" )
1759 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1760 main.log.info(
1761 "Switch roles are consistent across all ONOS nodes" )
1762 else:
1763 consistentMastership = False
1764 utilities.assert_equals(
1765 expect=True,
1766 actual=consistentMastership,
1767 onpass="Switch roles are consistent across all ONOS nodes",
1768 onfail="ONOS nodes have different views of switch roles" )
1769
1770 if rolesResults and not consistentMastership:
Jon Halle1a3b752015-07-22 13:02:46 -07001771 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001772 main.log.warn(
1773 "ONOS" + str( i + 1 ) + " roles: ",
1774 json.dumps(
1775 json.loads( ONOSMastership[ i ] ),
1776 sort_keys=True,
1777 indent=4,
1778 separators=( ',', ': ' ) ) )
1779 elif rolesResults and not consistentMastership:
1780 mastershipCheck = main.TRUE
1781
1782 description2 = "Compare switch roles from before failure"
1783 main.step( description2 )
1784 try:
1785 currentJson = json.loads( ONOSMastership[0] )
1786 oldJson = json.loads( mastershipState )
1787 except ( ValueError, TypeError ):
1788 main.log.exception( "Something is wrong with parsing " +
1789 "ONOSMastership[0] or mastershipState" )
1790 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1791 main.log.error( "mastershipState" + repr( mastershipState ) )
1792 main.cleanup()
1793 main.exit()
1794 mastershipCheck = main.TRUE
1795 for i in range( 1, 29 ):
1796 switchDPID = str(
1797 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1798 current = [ switch[ 'master' ] for switch in currentJson
1799 if switchDPID in switch[ 'id' ] ]
1800 old = [ switch[ 'master' ] for switch in oldJson
1801 if switchDPID in switch[ 'id' ] ]
1802 if current == old:
1803 mastershipCheck = mastershipCheck and main.TRUE
1804 else:
1805 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1806 mastershipCheck = main.FALSE
1807 utilities.assert_equals(
1808 expect=main.TRUE,
1809 actual=mastershipCheck,
1810 onpass="Mastership of Switches was not changed",
1811 onfail="Mastership of some switches changed" )
1812 mastershipCheck = mastershipCheck and consistentMastership
1813
1814 main.step( "Get the intents and compare across all nodes" )
1815 ONOSIntents = []
1816 intentCheck = main.FALSE
1817 consistentIntents = True
1818 intentsResults = True
1819 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001820 for i in range( main.numCtrls ):
1821 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001822 name="intents-" + str( i ),
1823 args=[],
1824 kwargs={ 'jsonFormat': True } )
1825 threads.append( t )
1826 t.start()
1827
1828 for t in threads:
1829 t.join()
1830 ONOSIntents.append( t.result )
1831
Jon Halle1a3b752015-07-22 13:02:46 -07001832 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001833 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1834 main.log.error( "Error in getting ONOS" + str( i + 1 ) +
1835 " intents" )
1836 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1837 repr( ONOSIntents[ i ] ) )
1838 intentsResults = False
1839 utilities.assert_equals(
1840 expect=True,
1841 actual=intentsResults,
1842 onpass="No error in reading intents output",
1843 onfail="Error in reading intents from ONOS" )
1844
1845 main.step( "Check for consistency in Intents from each controller" )
1846 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1847 main.log.info( "Intents are consistent across all ONOS " +
1848 "nodes" )
1849 else:
1850 consistentIntents = False
1851
1852 # Try to make it easy to figure out what is happening
1853 #
1854 # Intent ONOS1 ONOS2 ...
1855 # 0x01 INSTALLED INSTALLING
1856 # ... ... ...
1857 # ... ... ...
1858 title = " ID"
Jon Halle1a3b752015-07-22 13:02:46 -07001859 for n in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001860 title += " " * 10 + "ONOS" + str( n + 1 )
1861 main.log.warn( title )
1862 # get all intent keys in the cluster
1863 keys = []
1864 for nodeStr in ONOSIntents:
1865 node = json.loads( nodeStr )
1866 for intent in node:
1867 keys.append( intent.get( 'id' ) )
1868 keys = set( keys )
1869 for key in keys:
1870 row = "%-13s" % key
1871 for nodeStr in ONOSIntents:
1872 node = json.loads( nodeStr )
1873 for intent in node:
1874 if intent.get( 'id' ) == key:
1875 row += "%-15s" % intent.get( 'state' )
1876 main.log.warn( row )
1877 # End table view
1878
1879 utilities.assert_equals(
1880 expect=True,
1881 actual=consistentIntents,
1882 onpass="Intents are consistent across all ONOS nodes",
1883 onfail="ONOS nodes have different views of intents" )
1884 intentStates = []
1885 for node in ONOSIntents: # Iter through ONOS nodes
1886 nodeStates = []
1887 # Iter through intents of a node
1888 try:
1889 for intent in json.loads( node ):
1890 nodeStates.append( intent[ 'state' ] )
1891 except ( ValueError, TypeError ):
1892 main.log.exception( "Error in parsing intents" )
1893 main.log.error( repr( node ) )
1894 intentStates.append( nodeStates )
1895 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1896 main.log.info( dict( out ) )
1897
1898 if intentsResults and not consistentIntents:
Jon Halle1a3b752015-07-22 13:02:46 -07001899 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001900 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1901 main.log.warn( json.dumps(
1902 json.loads( ONOSIntents[ i ] ),
1903 sort_keys=True,
1904 indent=4,
1905 separators=( ',', ': ' ) ) )
1906 elif intentsResults and consistentIntents:
1907 intentCheck = main.TRUE
1908
1909 # NOTE: Store has no durability, so intents are lost across system
1910 # restarts
1911 main.step( "Compare current intents with intents before the failure" )
1912 # NOTE: this requires case 5 to pass for intentState to be set.
1913 # maybe we should stop the test if that fails?
1914 sameIntents = main.FALSE
1915 if intentState and intentState == ONOSIntents[ 0 ]:
1916 sameIntents = main.TRUE
1917 main.log.info( "Intents are consistent with before failure" )
1918 # TODO: possibly the states have changed? we may need to figure out
1919 # what the acceptable states are
1920 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1921 sameIntents = main.TRUE
1922 try:
1923 before = json.loads( intentState )
1924 after = json.loads( ONOSIntents[ 0 ] )
1925 for intent in before:
1926 if intent not in after:
1927 sameIntents = main.FALSE
1928 main.log.debug( "Intent is not currently in ONOS " +
1929 "(at least in the same form):" )
1930 main.log.debug( json.dumps( intent ) )
1931 except ( ValueError, TypeError ):
1932 main.log.exception( "Exception printing intents" )
1933 main.log.debug( repr( ONOSIntents[0] ) )
1934 main.log.debug( repr( intentState ) )
1935 if sameIntents == main.FALSE:
1936 try:
1937 main.log.debug( "ONOS intents before: " )
1938 main.log.debug( json.dumps( json.loads( intentState ),
1939 sort_keys=True, indent=4,
1940 separators=( ',', ': ' ) ) )
1941 main.log.debug( "Current ONOS intents: " )
1942 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1943 sort_keys=True, indent=4,
1944 separators=( ',', ': ' ) ) )
1945 except ( ValueError, TypeError ):
1946 main.log.exception( "Exception printing intents" )
1947 main.log.debug( repr( ONOSIntents[0] ) )
1948 main.log.debug( repr( intentState ) )
1949 utilities.assert_equals(
1950 expect=main.TRUE,
1951 actual=sameIntents,
1952 onpass="Intents are consistent with before failure",
1953 onfail="The Intents changed during failure" )
1954 intentCheck = intentCheck and sameIntents
1955
1956 main.step( "Get the OF Table entries and compare to before " +
1957 "component failure" )
1958 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07001959 for i in range( 28 ):
1960 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08001961 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
1962 FlowTables = FlowTables and main.Mininet1.flowTableComp( flows[i], tmpFlows )
Jon Hall5cf14d52015-07-16 12:15:19 -07001963 if FlowTables == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08001964 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
1965
Jon Hall5cf14d52015-07-16 12:15:19 -07001966 utilities.assert_equals(
1967 expect=main.TRUE,
1968 actual=FlowTables,
1969 onpass="No changes were found in the flow tables",
1970 onfail="Changes were found in the flow tables" )
1971
1972 main.Mininet2.pingLongKill()
1973 '''
1974 main.step( "Check the continuous pings to ensure that no packets " +
1975 "were dropped during component failure" )
1976 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1977 main.params[ 'TESTONIP' ] )
1978 LossInPings = main.FALSE
1979 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1980 for i in range( 8, 18 ):
1981 main.log.info(
1982 "Checking for a loss in pings along flow from s" +
1983 str( i ) )
1984 LossInPings = main.Mininet2.checkForLoss(
1985 "/tmp/ping.h" +
1986 str( i ) ) or LossInPings
1987 if LossInPings == main.TRUE:
1988 main.log.info( "Loss in ping detected" )
1989 elif LossInPings == main.ERROR:
1990 main.log.info( "There are multiple mininet process running" )
1991 elif LossInPings == main.FALSE:
1992 main.log.info( "No Loss in the pings" )
1993 main.log.info( "No loss of dataplane connectivity" )
1994 utilities.assert_equals(
1995 expect=main.FALSE,
1996 actual=LossInPings,
1997 onpass="No Loss of connectivity",
1998 onfail="Loss of dataplane connectivity detected" )
1999 '''
2000
2001 main.step( "Leadership Election is still functional" )
2002 # Test of LeadershipElection
2003 # NOTE: this only works for the sanity test. In case of failures,
2004 # leader will likely change
Jon Halle1a3b752015-07-22 13:02:46 -07002005 leader = main.nodes[ 0 ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002006 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07002007 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002008 leaderN = cli.electionTestLeader()
2009 # verify leader is ONOS1
2010 if leaderN == leader:
2011 # all is well
2012 # NOTE: In failure scenario, this could be a new node, maybe
2013 # check != ONOS1
2014 pass
2015 elif leaderN == main.FALSE:
2016 # error in response
2017 main.log.error( "Something is wrong with " +
2018 "electionTestLeader function, check the" +
2019 " error logs" )
2020 leaderResult = main.FALSE
2021 elif leader != leaderN:
2022 leaderResult = main.FALSE
2023 main.log.error( cli.name + " sees " + str( leaderN ) +
2024 " as the leader of the election app. " +
2025 "Leader should be " + str( leader ) )
2026 utilities.assert_equals(
2027 expect=main.TRUE,
2028 actual=leaderResult,
2029 onpass="Leadership election passed",
2030 onfail="Something went wrong with Leadership election" )
2031
2032 def CASE8( self, main ):
2033 """
2034 Compare topo
2035 """
2036 import json
2037 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002038 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002039 assert main, "main not defined"
2040 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002041 assert main.CLIs, "main.CLIs not defined"
2042 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002043
2044 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002045 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002046 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002047 topoResult = main.FALSE
2048 elapsed = 0
2049 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002050 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002051 startTime = time.time()
2052 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002053 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002054 devicesResults = main.TRUE
2055 linksResults = main.TRUE
2056 hostsResults = main.TRUE
2057 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002058 count += 1
2059 cliStart = time.time()
2060 devices = []
2061 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002062 for i in range( main.numCtrls ):
2063 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07002064 name="devices-" + str( i ),
2065 args=[ ] )
2066 threads.append( t )
2067 t.start()
2068
2069 for t in threads:
2070 t.join()
2071 devices.append( t.result )
2072 hosts = []
2073 ipResult = main.TRUE
2074 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002075 for i in range( main.numCtrls ):
Jon Halld8f6de82015-12-17 17:04:34 -08002076 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002077 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002078 args=[ main.CLIs[i].hosts, [ None ] ],
2079 kwargs= { 'sleep': 5, 'attempts': 5,
2080 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002081 threads.append( t )
2082 t.start()
2083
2084 for t in threads:
2085 t.join()
2086 try:
2087 hosts.append( json.loads( t.result ) )
2088 except ( ValueError, TypeError ):
2089 main.log.exception( "Error parsing hosts results" )
2090 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002091 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002092 for controller in range( 0, len( hosts ) ):
2093 controllerStr = str( controller + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002094 if hosts[ controller ]:
2095 for host in hosts[ controller ]:
2096 if host is None or host.get( 'ipAddresses', [] ) == []:
2097 main.log.error(
2098 "Error with host ipAddresses on controller" +
2099 controllerStr + ": " + str( host ) )
2100 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002101 ports = []
2102 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002103 for i in range( main.numCtrls ):
2104 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07002105 name="ports-" + str( i ),
2106 args=[ ] )
2107 threads.append( t )
2108 t.start()
2109
2110 for t in threads:
2111 t.join()
2112 ports.append( t.result )
2113 links = []
2114 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002115 for i in range( main.numCtrls ):
2116 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07002117 name="links-" + str( i ),
2118 args=[ ] )
2119 threads.append( t )
2120 t.start()
2121
2122 for t in threads:
2123 t.join()
2124 links.append( t.result )
2125 clusters = []
2126 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002127 for i in range( main.numCtrls ):
2128 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07002129 name="clusters-" + str( i ),
2130 args=[ ] )
2131 threads.append( t )
2132 t.start()
2133
2134 for t in threads:
2135 t.join()
2136 clusters.append( t.result )
2137
2138 elapsed = time.time() - startTime
2139 cliTime = time.time() - cliStart
2140 print "Elapsed time: " + str( elapsed )
2141 print "CLI time: " + str( cliTime )
2142
2143 mnSwitches = main.Mininet1.getSwitches()
2144 mnLinks = main.Mininet1.getLinks()
2145 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07002146 for controller in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002147 controllerStr = str( controller + 1 )
2148 if devices[ controller ] and ports[ controller ] and\
2149 "Error" not in devices[ controller ] and\
2150 "Error" not in ports[ controller ]:
2151
Jon Hallc6793552016-01-19 14:18:37 -08002152 try:
2153 currentDevicesResult = main.Mininet1.compareSwitches(
2154 mnSwitches,
2155 json.loads( devices[ controller ] ),
2156 json.loads( ports[ controller ] ) )
2157 except ( TypeError, ValueError ) as e:
2158 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2159 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002160 else:
2161 currentDevicesResult = main.FALSE
2162 utilities.assert_equals( expect=main.TRUE,
2163 actual=currentDevicesResult,
2164 onpass="ONOS" + controllerStr +
2165 " Switches view is correct",
2166 onfail="ONOS" + controllerStr +
2167 " Switches view is incorrect" )
2168
2169 if links[ controller ] and "Error" not in links[ controller ]:
2170 currentLinksResult = main.Mininet1.compareLinks(
2171 mnSwitches, mnLinks,
2172 json.loads( links[ controller ] ) )
2173 else:
2174 currentLinksResult = main.FALSE
2175 utilities.assert_equals( expect=main.TRUE,
2176 actual=currentLinksResult,
2177 onpass="ONOS" + controllerStr +
2178 " links view is correct",
2179 onfail="ONOS" + controllerStr +
2180 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002181 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002182 currentHostsResult = main.Mininet1.compareHosts(
2183 mnHosts,
2184 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002185 elif hosts[ controller ] == []:
2186 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002187 else:
2188 currentHostsResult = main.FALSE
2189 utilities.assert_equals( expect=main.TRUE,
2190 actual=currentHostsResult,
2191 onpass="ONOS" + controllerStr +
2192 " hosts exist in Mininet",
2193 onfail="ONOS" + controllerStr +
2194 " hosts don't match Mininet" )
2195 # CHECKING HOST ATTACHMENT POINTS
2196 hostAttachment = True
2197 zeroHosts = False
2198 # FIXME: topo-HA/obelisk specific mappings:
2199 # key is mac and value is dpid
2200 mappings = {}
2201 for i in range( 1, 29 ): # hosts 1 through 28
2202 # set up correct variables:
2203 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2204 if i == 1:
2205 deviceId = "1000".zfill(16)
2206 elif i == 2:
2207 deviceId = "2000".zfill(16)
2208 elif i == 3:
2209 deviceId = "3000".zfill(16)
2210 elif i == 4:
2211 deviceId = "3004".zfill(16)
2212 elif i == 5:
2213 deviceId = "5000".zfill(16)
2214 elif i == 6:
2215 deviceId = "6000".zfill(16)
2216 elif i == 7:
2217 deviceId = "6007".zfill(16)
2218 elif i >= 8 and i <= 17:
2219 dpid = '3' + str( i ).zfill( 3 )
2220 deviceId = dpid.zfill(16)
2221 elif i >= 18 and i <= 27:
2222 dpid = '6' + str( i ).zfill( 3 )
2223 deviceId = dpid.zfill(16)
2224 elif i == 28:
2225 deviceId = "2800".zfill(16)
2226 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002227 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002228 if hosts[ controller ] == []:
2229 main.log.warn( "There are no hosts discovered" )
2230 zeroHosts = True
2231 else:
2232 for host in hosts[ controller ]:
2233 mac = None
2234 location = None
2235 device = None
2236 port = None
2237 try:
2238 mac = host.get( 'mac' )
2239 assert mac, "mac field could not be found for this host object"
2240
2241 location = host.get( 'location' )
2242 assert location, "location field could not be found for this host object"
2243
2244 # Trim the protocol identifier off deviceId
2245 device = str( location.get( 'elementId' ) ).split(':')[1]
2246 assert device, "elementId field could not be found for this host location object"
2247
2248 port = location.get( 'port' )
2249 assert port, "port field could not be found for this host location object"
2250
2251 # Now check if this matches where they should be
2252 if mac and device and port:
2253 if str( port ) != "1":
2254 main.log.error( "The attachment port is incorrect for " +
2255 "host " + str( mac ) +
2256 ". Expected: 1 Actual: " + str( port) )
2257 hostAttachment = False
2258 if device != mappings[ str( mac ) ]:
2259 main.log.error( "The attachment device is incorrect for " +
2260 "host " + str( mac ) +
2261 ". Expected: " + mappings[ str( mac ) ] +
2262 " Actual: " + device )
2263 hostAttachment = False
2264 else:
2265 hostAttachment = False
2266 except AssertionError:
2267 main.log.exception( "Json object not as expected" )
2268 main.log.error( repr( host ) )
2269 hostAttachment = False
2270 else:
2271 main.log.error( "No hosts json output or \"Error\"" +
2272 " in output. hosts = " +
2273 repr( hosts[ controller ] ) )
2274 if zeroHosts is False:
2275 hostAttachment = True
2276
2277 # END CHECKING HOST ATTACHMENT POINTS
2278 devicesResults = devicesResults and currentDevicesResult
2279 linksResults = linksResults and currentLinksResult
2280 hostsResults = hostsResults and currentHostsResult
2281 hostAttachmentResults = hostAttachmentResults and\
2282 hostAttachment
2283 topoResult = ( devicesResults and linksResults
2284 and hostsResults and ipResult and
2285 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002286 utilities.assert_equals( expect=True,
2287 actual=topoResult,
2288 onpass="ONOS topology matches Mininet",
2289 onfail="ONOS topology don't match Mininet" )
2290 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002291
2292 # Compare json objects for hosts and dataplane clusters
2293
2294 # hosts
2295 main.step( "Hosts view is consistent across all ONOS nodes" )
2296 consistentHostsResult = main.TRUE
2297 for controller in range( len( hosts ) ):
2298 controllerStr = str( controller + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002299 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002300 if hosts[ controller ] == hosts[ 0 ]:
2301 continue
2302 else: # hosts not consistent
2303 main.log.error( "hosts from ONOS" + controllerStr +
2304 " is inconsistent with ONOS1" )
2305 main.log.warn( repr( hosts[ controller ] ) )
2306 consistentHostsResult = main.FALSE
2307
2308 else:
2309 main.log.error( "Error in getting ONOS hosts from ONOS" +
2310 controllerStr )
2311 consistentHostsResult = main.FALSE
2312 main.log.warn( "ONOS" + controllerStr +
2313 " hosts response: " +
2314 repr( hosts[ controller ] ) )
2315 utilities.assert_equals(
2316 expect=main.TRUE,
2317 actual=consistentHostsResult,
2318 onpass="Hosts view is consistent across all ONOS nodes",
2319 onfail="ONOS nodes have different views of hosts" )
2320
2321 main.step( "Hosts information is correct" )
2322 hostsResults = hostsResults and ipResult
2323 utilities.assert_equals(
2324 expect=main.TRUE,
2325 actual=hostsResults,
2326 onpass="Host information is correct",
2327 onfail="Host information is incorrect" )
2328
2329 main.step( "Host attachment points to the network" )
2330 utilities.assert_equals(
2331 expect=True,
2332 actual=hostAttachmentResults,
2333 onpass="Hosts are correctly attached to the network",
2334 onfail="ONOS did not correctly attach hosts to the network" )
2335
2336 # Strongly connected clusters of devices
2337 main.step( "Clusters view is consistent across all ONOS nodes" )
2338 consistentClustersResult = main.TRUE
2339 for controller in range( len( clusters ) ):
2340 controllerStr = str( controller + 1 )
2341 if "Error" not in clusters[ controller ]:
2342 if clusters[ controller ] == clusters[ 0 ]:
2343 continue
2344 else: # clusters not consistent
2345 main.log.error( "clusters from ONOS" +
2346 controllerStr +
2347 " is inconsistent with ONOS1" )
2348 consistentClustersResult = main.FALSE
2349
2350 else:
2351 main.log.error( "Error in getting dataplane clusters " +
2352 "from ONOS" + controllerStr )
2353 consistentClustersResult = main.FALSE
2354 main.log.warn( "ONOS" + controllerStr +
2355 " clusters response: " +
2356 repr( clusters[ controller ] ) )
2357 utilities.assert_equals(
2358 expect=main.TRUE,
2359 actual=consistentClustersResult,
2360 onpass="Clusters view is consistent across all ONOS nodes",
2361 onfail="ONOS nodes have different views of clusters" )
2362
2363 main.step( "There is only one SCC" )
2364 # there should always only be one cluster
2365 try:
2366 numClusters = len( json.loads( clusters[ 0 ] ) )
2367 except ( ValueError, TypeError ):
2368 main.log.exception( "Error parsing clusters[0]: " +
2369 repr( clusters[0] ) )
2370 clusterResults = main.FALSE
2371 if numClusters == 1:
2372 clusterResults = main.TRUE
2373 utilities.assert_equals(
2374 expect=1,
2375 actual=numClusters,
2376 onpass="ONOS shows 1 SCC",
2377 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2378
2379 topoResult = ( devicesResults and linksResults
2380 and hostsResults and consistentHostsResult
2381 and consistentClustersResult and clusterResults
2382 and ipResult and hostAttachmentResults )
2383
2384 topoResult = topoResult and int( count <= 2 )
2385 note = "note it takes about " + str( int( cliTime ) ) + \
2386 " seconds for the test to make all the cli calls to fetch " +\
2387 "the topology from each ONOS instance"
2388 main.log.info(
2389 "Very crass estimate for topology discovery/convergence( " +
2390 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2391 str( count ) + " tries" )
2392
2393 main.step( "Device information is correct" )
2394 utilities.assert_equals(
2395 expect=main.TRUE,
2396 actual=devicesResults,
2397 onpass="Device information is correct",
2398 onfail="Device information is incorrect" )
2399
2400 main.step( "Links are correct" )
2401 utilities.assert_equals(
2402 expect=main.TRUE,
2403 actual=linksResults,
2404 onpass="Link are correct",
2405 onfail="Links are incorrect" )
2406
2407 main.step( "Hosts are correct" )
2408 utilities.assert_equals(
2409 expect=main.TRUE,
2410 actual=hostsResults,
2411 onpass="Hosts are correct",
2412 onfail="Hosts are incorrect" )
2413
2414 # FIXME: move this to an ONOS state case
2415 main.step( "Checking ONOS nodes" )
2416 nodesOutput = []
2417 nodeResults = main.TRUE
2418 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07002419 for i in range( main.numCtrls ):
2420 t = main.Thread( target=main.CLIs[i].nodes,
Jon Hall5cf14d52015-07-16 12:15:19 -07002421 name="nodes-" + str( i ),
2422 args=[ ] )
2423 threads.append( t )
2424 t.start()
2425
2426 for t in threads:
2427 t.join()
2428 nodesOutput.append( t.result )
Jon Halle1a3b752015-07-22 13:02:46 -07002429 ips = [ node.ip_address for node in main.nodes ]
Jon Halle9b1fa32015-12-08 15:32:21 -08002430 ips.sort()
Jon Hall5cf14d52015-07-16 12:15:19 -07002431 for i in nodesOutput:
2432 try:
2433 current = json.loads( i )
Jon Halle9b1fa32015-12-08 15:32:21 -08002434 activeIps = []
2435 currentResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002436 for node in current:
Jon Hallbd182782016-03-28 16:42:22 -07002437 if node['state'] == 'READY':
Jon Halle9b1fa32015-12-08 15:32:21 -08002438 activeIps.append( node['ip'] )
2439 activeIps.sort()
2440 if ips == activeIps:
2441 currentResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002442 except ( ValueError, TypeError ):
2443 main.log.error( "Error parsing nodes output" )
2444 main.log.warn( repr( i ) )
Jon Halle9b1fa32015-12-08 15:32:21 -08002445 currentResult = main.FALSE
2446 nodeResults = nodeResults and currentResult
Jon Hall5cf14d52015-07-16 12:15:19 -07002447 utilities.assert_equals( expect=main.TRUE, actual=nodeResults,
2448 onpass="Nodes check successful",
2449 onfail="Nodes check NOT successful" )
2450
2451 def CASE9( self, main ):
2452 """
2453 Link s3-s28 down
2454 """
2455 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002456 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002457 assert main, "main not defined"
2458 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002459 assert main.CLIs, "main.CLIs not defined"
2460 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002461 # NOTE: You should probably run a topology check after this
2462
2463 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2464
2465 description = "Turn off a link to ensure that Link Discovery " +\
2466 "is working properly"
2467 main.case( description )
2468
2469 main.step( "Kill Link between s3 and s28" )
2470 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2471 main.log.info( "Waiting " + str( linkSleep ) +
2472 " seconds for link down to be discovered" )
2473 time.sleep( linkSleep )
2474 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2475 onpass="Link down successful",
2476 onfail="Failed to bring link down" )
2477 # TODO do some sort of check here
2478
2479 def CASE10( self, main ):
2480 """
2481 Link s3-s28 up
2482 """
2483 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002484 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002485 assert main, "main not defined"
2486 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002487 assert main.CLIs, "main.CLIs not defined"
2488 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002489 # NOTE: You should probably run a topology check after this
2490
2491 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2492
2493 description = "Restore a link to ensure that Link Discovery is " + \
2494 "working properly"
2495 main.case( description )
2496
2497 main.step( "Bring link between s3 and s28 back up" )
2498 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2499 main.log.info( "Waiting " + str( linkSleep ) +
2500 " seconds for link up to be discovered" )
2501 time.sleep( linkSleep )
2502 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2503 onpass="Link up successful",
2504 onfail="Failed to bring link up" )
2505 # TODO do some sort of check here
2506
2507 def CASE11( self, main ):
2508 """
2509 Switch Down
2510 """
2511 # NOTE: You should probably run a topology check after this
2512 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002513 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002514 assert main, "main not defined"
2515 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002516 assert main.CLIs, "main.CLIs not defined"
2517 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002518
2519 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2520
2521 description = "Killing a switch to ensure it is discovered correctly"
2522 main.case( description )
2523 switch = main.params[ 'kill' ][ 'switch' ]
2524 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2525
2526 # TODO: Make this switch parameterizable
2527 main.step( "Kill " + switch )
2528 main.log.info( "Deleting " + switch )
2529 main.Mininet1.delSwitch( switch )
2530 main.log.info( "Waiting " + str( switchSleep ) +
2531 " seconds for switch down to be discovered" )
2532 time.sleep( switchSleep )
2533 device = main.ONOScli1.getDevice( dpid=switchDPID )
2534 # Peek at the deleted switch
2535 main.log.warn( str( device ) )
2536 result = main.FALSE
2537 if device and device[ 'available' ] is False:
2538 result = main.TRUE
2539 utilities.assert_equals( expect=main.TRUE, actual=result,
2540 onpass="Kill switch successful",
2541 onfail="Failed to kill switch?" )
2542
2543 def CASE12( self, main ):
2544 """
2545 Switch Up
2546 """
2547 # NOTE: You should probably run a topology check after this
2548 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002549 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002550 assert main, "main not defined"
2551 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002552 assert main.CLIs, "main.CLIs not defined"
2553 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002554 assert ONOS1Port, "ONOS1Port not defined"
2555 assert ONOS2Port, "ONOS2Port not defined"
2556 assert ONOS3Port, "ONOS3Port not defined"
2557 assert ONOS4Port, "ONOS4Port not defined"
2558 assert ONOS5Port, "ONOS5Port not defined"
2559 assert ONOS6Port, "ONOS6Port not defined"
2560 assert ONOS7Port, "ONOS7Port not defined"
2561
2562 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2563 switch = main.params[ 'kill' ][ 'switch' ]
2564 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2565 links = main.params[ 'kill' ][ 'links' ].split()
2566 description = "Adding a switch to ensure it is discovered correctly"
2567 main.case( description )
2568
2569 main.step( "Add back " + switch )
2570 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2571 for peer in links:
2572 main.Mininet1.addLink( switch, peer )
2573 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -07002574 for i in range( main.numCtrls ):
2575 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002576 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2577 main.log.info( "Waiting " + str( switchSleep ) +
2578 " seconds for switch up to be discovered" )
2579 time.sleep( switchSleep )
2580 device = main.ONOScli1.getDevice( dpid=switchDPID )
2581 # Peek at the deleted switch
2582 main.log.warn( str( device ) )
2583 result = main.FALSE
2584 if device and device[ 'available' ]:
2585 result = main.TRUE
2586 utilities.assert_equals( expect=main.TRUE, actual=result,
2587 onpass="add switch successful",
2588 onfail="Failed to add switch?" )
2589
2590 def CASE13( self, main ):
2591 """
2592 Clean up
2593 """
2594 import os
2595 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002596 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002597 assert main, "main not defined"
2598 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002599 assert main.CLIs, "main.CLIs not defined"
2600 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002601
2602 # printing colors to terminal
2603 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2604 'blue': '\033[94m', 'green': '\033[92m',
2605 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2606 main.case( "Test Cleanup" )
2607 main.step( "Killing tcpdumps" )
2608 main.Mininet2.stopTcpdump()
2609
2610 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002611 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002612 main.step( "Copying MN pcap and ONOS log files to test station" )
2613 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2614 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002615 # NOTE: MN Pcap file is being saved to logdir.
2616 # We scp this file as MN and TestON aren't necessarily the same vm
2617
2618 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002619 # TODO: Load these from params
2620 # NOTE: must end in /
2621 logFolder = "/opt/onos/log/"
2622 logFiles = [ "karaf.log", "karaf.log.1" ]
2623 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002624 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002625 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002626 dstName = main.logdir + "/" + node.name + "-" + f
2627 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2628 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002629 # std*.log's
2630 # NOTE: must end in /
2631 logFolder = "/opt/onos/var/"
2632 logFiles = [ "stderr.log", "stdout.log" ]
2633 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002634 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002635 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002636 dstName = main.logdir + "/" + node.name + "-" + f
2637 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2638 logFolder + f, dstName )
2639 else:
2640 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002641
2642 main.step( "Stopping Mininet" )
2643 mnResult = main.Mininet1.stopNet()
2644 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2645 onpass="Mininet stopped",
2646 onfail="MN cleanup NOT successful" )
2647
2648 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002649 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002650 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2651 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002652
2653 try:
2654 timerLog = open( main.logdir + "/Timers.csv", 'w')
2655 # Overwrite with empty line and close
2656 labels = "Gossip Intents"
2657 data = str( gossipTime )
2658 timerLog.write( labels + "\n" + data )
2659 timerLog.close()
2660 except NameError, e:
2661 main.log.exception(e)
2662
2663 def CASE14( self, main ):
2664 """
2665 start election app on all onos nodes
2666 """
Jon Halle1a3b752015-07-22 13:02:46 -07002667 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002668 assert main, "main not defined"
2669 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002670 assert main.CLIs, "main.CLIs not defined"
2671 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002672
2673 main.case("Start Leadership Election app")
2674 main.step( "Install leadership election app" )
2675 appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
2676 utilities.assert_equals(
2677 expect=main.TRUE,
2678 actual=appResult,
2679 onpass="Election app installed",
2680 onfail="Something went wrong with installing Leadership election" )
2681
2682 main.step( "Run for election on each node" )
2683 leaderResult = main.TRUE
2684 leaders = []
Jon Halle1a3b752015-07-22 13:02:46 -07002685 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002686 cli.electionTestRun()
Jon Halle1a3b752015-07-22 13:02:46 -07002687 for cli in main.CLIs:
Jon Hall5cf14d52015-07-16 12:15:19 -07002688 leader = cli.electionTestLeader()
2689 if leader is None or leader == main.FALSE:
2690 main.log.error( cli.name + ": Leader for the election app " +
2691 "should be an ONOS node, instead got '" +
2692 str( leader ) + "'" )
2693 leaderResult = main.FALSE
2694 leaders.append( leader )
2695 utilities.assert_equals(
2696 expect=main.TRUE,
2697 actual=leaderResult,
2698 onpass="Successfully ran for leadership",
2699 onfail="Failed to run for leadership" )
2700
2701 main.step( "Check that each node shows the same leader" )
2702 sameLeader = main.TRUE
2703 if len( set( leaders ) ) != 1:
2704 sameLeader = main.FALSE
Jon Halle1a3b752015-07-22 13:02:46 -07002705 main.log.error( "Results of electionTestLeader is order of main.CLIs:" +
Jon Hall5cf14d52015-07-16 12:15:19 -07002706 str( leaders ) )
2707 utilities.assert_equals(
2708 expect=main.TRUE,
2709 actual=sameLeader,
2710 onpass="Leadership is consistent for the election topic",
2711 onfail="Nodes have different leaders" )
2712
2713 def CASE15( self, main ):
2714 """
2715 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002716 15.1 Run election on each node
2717 15.2 Check that each node has the same leaders and candidates
2718 15.3 Find current leader and withdraw
2719 15.4 Check that a new node was elected leader
2720 15.5 Check that that new leader was the candidate of old leader
2721 15.6 Run for election on old leader
2722 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2723 15.8 Make sure that the old leader was added to the candidate list
2724
2725 old and new variable prefixes refer to data from before vs after
2726 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002727 """
2728 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002729 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002730 assert main, "main not defined"
2731 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002732 assert main.CLIs, "main.CLIs not defined"
2733 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002734
acsmars3a72bde2015-09-02 14:16:22 -07002735 description = "Check that Leadership Election App is still functional"
Jon Hall5cf14d52015-07-16 12:15:19 -07002736 main.case( description )
acsmars71adceb2015-08-31 15:09:26 -07002737 # NOTE: Need to re-run since being a canidate is not persistant
2738 # TODO: add check for "Command not found:" in the driver, this
2739 # means the election test app isn't loaded
Jon Hall5cf14d52015-07-16 12:15:19 -07002740
acsmars71adceb2015-08-31 15:09:26 -07002741 oldLeaders = [] # leaders by node before withdrawl from candidates
2742 newLeaders = [] # leaders by node after withdrawl from candidates
2743 oldAllCandidates = [] # list of lists of each nodes' candidates before
2744 newAllCandidates = [] # list of lists of each nodes' candidates after
2745 oldCandidates = [] # list of candidates from node 0 before withdrawl
2746 newCandidates = [] # list of candidates from node 0 after withdrawl
2747 oldLeader = '' # the old leader from oldLeaders, None if not same
2748 newLeader = '' # the new leaders fron newLoeaders, None if not same
2749 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2750 expectNoLeader = False # True when there is only one leader
2751 if main.numCtrls == 1:
2752 expectNoLeader = True
2753
2754 main.step( "Run for election on each node" )
2755 electionResult = main.TRUE
2756
2757 for cli in main.CLIs: # run test election on each node
2758 if cli.electionTestRun() == main.FALSE:
2759 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002760 utilities.assert_equals(
2761 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002762 actual=electionResult,
2763 onpass="All nodes successfully ran for leadership",
2764 onfail="At least one node failed to run for leadership" )
2765
acsmars3a72bde2015-09-02 14:16:22 -07002766 if electionResult == main.FALSE:
2767 main.log.error(
2768 "Skipping Test Case because Election Test isn't loaded" )
2769 main.skipCase()
2770
acsmars71adceb2015-08-31 15:09:26 -07002771 main.step( "Check that each node shows the same leader and candidates" )
2772 sameResult = main.TRUE
2773 failMessage = "Nodes have different leaders"
2774 for cli in main.CLIs:
2775 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2776 oldAllCandidates.append( node )
2777 oldLeaders.append( node[ 0 ] )
2778 oldCandidates = oldAllCandidates[ 0 ]
2779
2780 # Check that each node has the same leader. Defines oldLeader
2781 if len( set( oldLeaders ) ) != 1:
2782 sameResult = main.FALSE
2783 main.log.error( "More than one leader present:" + str( oldLeaders ) )
2784 oldLeader = None
2785 else:
2786 oldLeader = oldLeaders[ 0 ]
2787
2788 # Check that each node's candidate list is the same
acsmars29233db2015-11-04 11:15:00 -08002789 candidateDiscrepancy = False # Boolean of candidate mismatches
acsmars71adceb2015-08-31 15:09:26 -07002790 for candidates in oldAllCandidates:
2791 if set( candidates ) != set( oldCandidates ):
2792 sameResult = main.FALSE
acsmars29233db2015-11-04 11:15:00 -08002793 candidateDiscrepancy = True
2794
2795 if candidateDiscrepancy:
2796 failMessage += " and candidates"
acsmars71adceb2015-08-31 15:09:26 -07002797
2798 utilities.assert_equals(
2799 expect=main.TRUE,
2800 actual=sameResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002801 onpass="Leadership is consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002802 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002803
2804 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002805 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002806 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002807 if oldLeader is None:
2808 main.log.error( "Leadership isn't consistent." )
2809 withdrawResult = main.FALSE
2810 # Get the CLI of the oldLeader
Jon Halle1a3b752015-07-22 13:02:46 -07002811 for i in range( len( main.CLIs ) ):
acsmars71adceb2015-08-31 15:09:26 -07002812 if oldLeader == main.nodes[ i ].ip_address:
2813 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002814 break
2815 else: # FOR/ELSE statement
2816 main.log.error( "Leader election, could not find current leader" )
2817 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002818 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002819 utilities.assert_equals(
2820 expect=main.TRUE,
2821 actual=withdrawResult,
2822 onpass="Node was withdrawn from election",
2823 onfail="Node was not withdrawn from election" )
2824
acsmars71adceb2015-08-31 15:09:26 -07002825 main.step( "Check that a new node was elected leader" )
2826
Jon Hall5cf14d52015-07-16 12:15:19 -07002827 # FIXME: use threads
acsmars71adceb2015-08-31 15:09:26 -07002828 newLeaderResult = main.TRUE
2829 failMessage = "Nodes have different leaders"
2830
2831 # Get new leaders and candidates
Jon Halle1a3b752015-07-22 13:02:46 -07002832 for cli in main.CLIs:
acsmars71adceb2015-08-31 15:09:26 -07002833 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2834 # elections might no have finished yet
2835 if node[ 0 ] == 'none' and not expectNoLeader:
2836 main.log.info( "Node has no leader, waiting 5 seconds to be " +
2837 "sure elections are complete." )
2838 time.sleep(5)
2839 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2840 # election still isn't done or there is a problem
2841 if node[ 0 ] == 'none':
2842 main.log.error( "No leader was elected on at least 1 node" )
2843 newLeaderResult = main.FALSE
2844 newAllCandidates.append( node )
2845 newLeaders.append( node[ 0 ] )
2846 newCandidates = newAllCandidates[ 0 ]
2847
2848 # Check that each node has the same leader. Defines newLeader
2849 if len( set( newLeaders ) ) != 1:
2850 newLeaderResult = main.FALSE
2851 main.log.error( "Nodes have different leaders: " +
2852 str( newLeaders ) )
2853 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002854 else:
acsmars71adceb2015-08-31 15:09:26 -07002855 newLeader = newLeaders[ 0 ]
2856
2857 # Check that each node's candidate list is the same
2858 for candidates in newAllCandidates:
2859 if set( candidates ) != set( newCandidates ):
2860 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002861 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002862
2863 # Check that the new leader is not the older leader, which was withdrawn
2864 if newLeader == oldLeader:
2865 newLeaderResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002866 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07002867 " as the current leader" )
2868
Jon Hall5cf14d52015-07-16 12:15:19 -07002869 utilities.assert_equals(
2870 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002871 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002872 onpass="Leadership election passed",
2873 onfail="Something went wrong with Leadership election" )
2874
acsmars71adceb2015-08-31 15:09:26 -07002875 main.step( "Check that that new leader was the candidate of old leader")
2876 # candidates[ 2 ] should be come the top candidate after withdrawl
2877 correctCandidateResult = main.TRUE
2878 if expectNoLeader:
2879 if newLeader == 'none':
2880 main.log.info( "No leader expected. None found. Pass" )
2881 correctCandidateResult = main.TRUE
2882 else:
2883 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2884 correctCandidateResult = main.FALSE
2885 elif newLeader != oldCandidates[ 2 ]:
2886 correctCandidateResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002887 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
2888 newLeader, oldCandidates[ 2 ] ) )
acsmars71adceb2015-08-31 15:09:26 -07002889
2890 utilities.assert_equals(
2891 expect=main.TRUE,
2892 actual=correctCandidateResult,
2893 onpass="Correct Candidate Elected",
2894 onfail="Incorrect Candidate Elected" )
2895
Jon Hall5cf14d52015-07-16 12:15:19 -07002896 main.step( "Run for election on old leader( just so everyone " +
2897 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002898 if oldLeaderCLI is not None:
2899 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002900 else:
acsmars71adceb2015-08-31 15:09:26 -07002901 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002902 runResult = main.FALSE
2903 utilities.assert_equals(
2904 expect=main.TRUE,
2905 actual=runResult,
2906 onpass="App re-ran for election",
2907 onfail="App failed to run for election" )
acsmars71adceb2015-08-31 15:09:26 -07002908 main.step(
2909 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002910 # verify leader didn't just change
acsmars71adceb2015-08-31 15:09:26 -07002911 positionResult = main.TRUE
2912 # Get new leaders and candidates, wait if oldLeader is not a candidate yet
2913
2914 # Reset and reuse the new candidate and leaders lists
2915 newAllCandidates = []
2916 newCandidates = []
2917 newLeaders = []
2918 for cli in main.CLIs:
2919 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2920 if oldLeader not in node: # election might no have finished yet
2921 main.log.info( "Old Leader not elected, waiting 5 seconds to " +
2922 "be sure elections are complete" )
2923 time.sleep(5)
2924 node = cli.specificLeaderCandidate( 'org.onosproject.election' )
2925 if oldLeader not in node: # election still isn't done, errors
2926 main.log.error(
2927 "Old leader was not elected on at least one node" )
2928 positionResult = main.FALSE
2929 newAllCandidates.append( node )
2930 newLeaders.append( node[ 0 ] )
2931 newCandidates = newAllCandidates[ 0 ]
2932
2933 # Check that each node has the same leader. Defines newLeader
2934 if len( set( newLeaders ) ) != 1:
2935 positionResult = main.FALSE
2936 main.log.error( "Nodes have different leaders: " +
2937 str( newLeaders ) )
2938 newLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002939 else:
acsmars71adceb2015-08-31 15:09:26 -07002940 newLeader = newLeaders[ 0 ]
2941
2942 # Check that each node's candidate list is the same
2943 for candidates in newAllCandidates:
2944 if set( candidates ) != set( newCandidates ):
2945 newLeaderResult = main.FALSE
Jon Hallceb4abb2015-09-25 12:03:06 -07002946 main.log.error( "Discrepancy in candidate lists detected" )
acsmars71adceb2015-08-31 15:09:26 -07002947
2948 # Check that the re-elected node is last on the candidate List
2949 if oldLeader != newCandidates[ -1 ]:
Jon Hall6e709752016-02-01 13:38:46 -08002950 main.log.error( "Old Leader (" + str( oldLeader ) + ") not in the proper position " +
acsmars71adceb2015-08-31 15:09:26 -07002951 str( newCandidates ) )
2952 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002953
2954 utilities.assert_equals(
2955 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002956 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002957 onpass="Old leader successfully re-ran for election",
2958 onfail="Something went wrong with Leadership election after " +
2959 "the old leader re-ran for election" )
2960
2961 def CASE16( self, main ):
2962 """
2963 Install Distributed Primitives app
2964 """
2965 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002966 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002967 assert main, "main not defined"
2968 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002969 assert main.CLIs, "main.CLIs not defined"
2970 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002971
2972 # Variables for the distributed primitives tests
2973 global pCounterName
2974 global iCounterName
2975 global pCounterValue
2976 global iCounterValue
2977 global onosSet
2978 global onosSetName
2979 pCounterName = "TestON-Partitions"
2980 iCounterName = "TestON-inMemory"
2981 pCounterValue = 0
2982 iCounterValue = 0
2983 onosSet = set([])
2984 onosSetName = "TestON-set"
2985
2986 description = "Install Primitives app"
2987 main.case( description )
2988 main.step( "Install Primitives app" )
2989 appName = "org.onosproject.distributedprimitives"
Jon Halle1a3b752015-07-22 13:02:46 -07002990 appResults = main.CLIs[0].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002991 utilities.assert_equals( expect=main.TRUE,
2992 actual=appResults,
2993 onpass="Primitives app activated",
2994 onfail="Primitives app not activated" )
2995 time.sleep( 5 ) # To allow all nodes to activate
2996
2997 def CASE17( self, main ):
2998 """
2999 Check for basic functionality with distributed primitives
3000 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003001 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003002 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003003 assert main, "main not defined"
3004 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003005 assert main.CLIs, "main.CLIs not defined"
3006 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003007 assert pCounterName, "pCounterName not defined"
3008 assert iCounterName, "iCounterName not defined"
3009 assert onosSetName, "onosSetName not defined"
3010 # NOTE: assert fails if value is 0/None/Empty/False
3011 try:
3012 pCounterValue
3013 except NameError:
3014 main.log.error( "pCounterValue not defined, setting to 0" )
3015 pCounterValue = 0
3016 try:
3017 iCounterValue
3018 except NameError:
3019 main.log.error( "iCounterValue not defined, setting to 0" )
3020 iCounterValue = 0
3021 try:
3022 onosSet
3023 except NameError:
3024 main.log.error( "onosSet not defined, setting to empty Set" )
3025 onosSet = set([])
3026 # Variables for the distributed primitives tests. These are local only
3027 addValue = "a"
3028 addAllValue = "a b c d e f"
3029 retainValue = "c d e f"
3030
3031 description = "Check for basic functionality with distributed " +\
3032 "primitives"
3033 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003034 main.caseExplanation = "Test the methods of the distributed " +\
3035 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003036 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003037 # Partitioned counters
3038 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003039 pCounters = []
3040 threads = []
3041 addedPValues = []
Jon Halle1a3b752015-07-22 13:02:46 -07003042 for i in range( main.numCtrls ):
3043 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3044 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003045 args=[ pCounterName ] )
3046 pCounterValue += 1
3047 addedPValues.append( pCounterValue )
3048 threads.append( t )
3049 t.start()
3050
3051 for t in threads:
3052 t.join()
3053 pCounters.append( t.result )
3054 # Check that counter incremented numController times
3055 pCounterResults = True
3056 for i in addedPValues:
3057 tmpResult = i in pCounters
3058 pCounterResults = pCounterResults and tmpResult
3059 if not tmpResult:
3060 main.log.error( str( i ) + " is not in partitioned "
3061 "counter incremented results" )
3062 utilities.assert_equals( expect=True,
3063 actual=pCounterResults,
3064 onpass="Default counter incremented",
3065 onfail="Error incrementing default" +
3066 " counter" )
3067
Jon Halle1a3b752015-07-22 13:02:46 -07003068 main.step( "Get then Increment a default counter on each node" )
3069 pCounters = []
3070 threads = []
3071 addedPValues = []
3072 for i in range( main.numCtrls ):
3073 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3074 name="counterGetAndAdd-" + str( i ),
3075 args=[ pCounterName ] )
3076 addedPValues.append( pCounterValue )
3077 pCounterValue += 1
3078 threads.append( t )
3079 t.start()
3080
3081 for t in threads:
3082 t.join()
3083 pCounters.append( t.result )
3084 # Check that counter incremented numController times
3085 pCounterResults = True
3086 for i in addedPValues:
3087 tmpResult = i in pCounters
3088 pCounterResults = pCounterResults and tmpResult
3089 if not tmpResult:
3090 main.log.error( str( i ) + " is not in partitioned "
3091 "counter incremented results" )
3092 utilities.assert_equals( expect=True,
3093 actual=pCounterResults,
3094 onpass="Default counter incremented",
3095 onfail="Error incrementing default" +
3096 " counter" )
3097
3098 main.step( "Counters we added have the correct values" )
3099 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3100 utilities.assert_equals( expect=main.TRUE,
3101 actual=incrementCheck,
3102 onpass="Added counters are correct",
3103 onfail="Added counters are incorrect" )
3104
3105 main.step( "Add -8 to then get a default counter on each node" )
3106 pCounters = []
3107 threads = []
3108 addedPValues = []
3109 for i in range( main.numCtrls ):
3110 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3111 name="counterIncrement-" + str( i ),
3112 args=[ pCounterName ],
3113 kwargs={ "delta": -8 } )
3114 pCounterValue += -8
3115 addedPValues.append( pCounterValue )
3116 threads.append( t )
3117 t.start()
3118
3119 for t in threads:
3120 t.join()
3121 pCounters.append( t.result )
3122 # Check that counter incremented numController times
3123 pCounterResults = True
3124 for i in addedPValues:
3125 tmpResult = i in pCounters
3126 pCounterResults = pCounterResults and tmpResult
3127 if not tmpResult:
3128 main.log.error( str( i ) + " is not in partitioned "
3129 "counter incremented results" )
3130 utilities.assert_equals( expect=True,
3131 actual=pCounterResults,
3132 onpass="Default counter incremented",
3133 onfail="Error incrementing default" +
3134 " counter" )
3135
3136 main.step( "Add 5 to then get a default counter on each node" )
3137 pCounters = []
3138 threads = []
3139 addedPValues = []
3140 for i in range( main.numCtrls ):
3141 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3142 name="counterIncrement-" + str( i ),
3143 args=[ pCounterName ],
3144 kwargs={ "delta": 5 } )
3145 pCounterValue += 5
3146 addedPValues.append( pCounterValue )
3147 threads.append( t )
3148 t.start()
3149
3150 for t in threads:
3151 t.join()
3152 pCounters.append( t.result )
3153 # Check that counter incremented numController times
3154 pCounterResults = True
3155 for i in addedPValues:
3156 tmpResult = i in pCounters
3157 pCounterResults = pCounterResults and tmpResult
3158 if not tmpResult:
3159 main.log.error( str( i ) + " is not in partitioned "
3160 "counter incremented results" )
3161 utilities.assert_equals( expect=True,
3162 actual=pCounterResults,
3163 onpass="Default counter incremented",
3164 onfail="Error incrementing default" +
3165 " counter" )
3166
3167 main.step( "Get then add 5 to a default counter on each node" )
3168 pCounters = []
3169 threads = []
3170 addedPValues = []
3171 for i in range( main.numCtrls ):
3172 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3173 name="counterIncrement-" + str( i ),
3174 args=[ pCounterName ],
3175 kwargs={ "delta": 5 } )
3176 addedPValues.append( pCounterValue )
3177 pCounterValue += 5
3178 threads.append( t )
3179 t.start()
3180
3181 for t in threads:
3182 t.join()
3183 pCounters.append( t.result )
3184 # Check that counter incremented numController times
3185 pCounterResults = True
3186 for i in addedPValues:
3187 tmpResult = i in pCounters
3188 pCounterResults = pCounterResults and tmpResult
3189 if not tmpResult:
3190 main.log.error( str( i ) + " is not in partitioned "
3191 "counter incremented results" )
3192 utilities.assert_equals( expect=True,
3193 actual=pCounterResults,
3194 onpass="Default counter incremented",
3195 onfail="Error incrementing default" +
3196 " counter" )
3197
3198 main.step( "Counters we added have the correct values" )
3199 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue )
3200 utilities.assert_equals( expect=main.TRUE,
3201 actual=incrementCheck,
3202 onpass="Added counters are correct",
3203 onfail="Added counters are incorrect" )
3204
3205 # In-Memory counters
3206 main.step( "Increment and get an in-memory counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003207 iCounters = []
3208 addedIValues = []
3209 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003210 for i in range( main.numCtrls ):
3211 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003212 name="icounterIncrement-" + str( i ),
3213 args=[ iCounterName ],
3214 kwargs={ "inMemory": True } )
3215 iCounterValue += 1
3216 addedIValues.append( iCounterValue )
3217 threads.append( t )
3218 t.start()
3219
3220 for t in threads:
3221 t.join()
3222 iCounters.append( t.result )
3223 # Check that counter incremented numController times
3224 iCounterResults = True
3225 for i in addedIValues:
3226 tmpResult = i in iCounters
3227 iCounterResults = iCounterResults and tmpResult
3228 if not tmpResult:
3229 main.log.error( str( i ) + " is not in the in-memory "
3230 "counter incremented results" )
3231 utilities.assert_equals( expect=True,
3232 actual=iCounterResults,
Jon Halle1a3b752015-07-22 13:02:46 -07003233 onpass="In-memory counter incremented",
3234 onfail="Error incrementing in-memory" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003235 " counter" )
3236
Jon Halle1a3b752015-07-22 13:02:46 -07003237 main.step( "Get then Increment a in-memory counter on each node" )
3238 iCounters = []
3239 threads = []
3240 addedIValues = []
3241 for i in range( main.numCtrls ):
3242 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3243 name="counterGetAndAdd-" + str( i ),
3244 args=[ iCounterName ],
3245 kwargs={ "inMemory": True } )
3246 addedIValues.append( iCounterValue )
3247 iCounterValue += 1
3248 threads.append( t )
3249 t.start()
3250
3251 for t in threads:
3252 t.join()
3253 iCounters.append( t.result )
3254 # Check that counter incremented numController times
3255 iCounterResults = True
3256 for i in addedIValues:
3257 tmpResult = i in iCounters
3258 iCounterResults = iCounterResults and tmpResult
3259 if not tmpResult:
3260 main.log.error( str( i ) + " is not in in-memory "
3261 "counter incremented results" )
3262 utilities.assert_equals( expect=True,
3263 actual=iCounterResults,
3264 onpass="In-memory counter incremented",
3265 onfail="Error incrementing in-memory" +
3266 " counter" )
3267
3268 main.step( "Counters we added have the correct values" )
3269 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3270 utilities.assert_equals( expect=main.TRUE,
3271 actual=incrementCheck,
3272 onpass="Added counters are correct",
3273 onfail="Added counters are incorrect" )
3274
3275 main.step( "Add -8 to then get a in-memory counter on each node" )
3276 iCounters = []
3277 threads = []
3278 addedIValues = []
3279 for i in range( main.numCtrls ):
3280 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3281 name="counterIncrement-" + str( i ),
3282 args=[ iCounterName ],
3283 kwargs={ "delta": -8, "inMemory": True } )
3284 iCounterValue += -8
3285 addedIValues.append( iCounterValue )
3286 threads.append( t )
3287 t.start()
3288
3289 for t in threads:
3290 t.join()
3291 iCounters.append( t.result )
3292 # Check that counter incremented numController times
3293 iCounterResults = True
3294 for i in addedIValues:
3295 tmpResult = i in iCounters
3296 iCounterResults = iCounterResults and tmpResult
3297 if not tmpResult:
3298 main.log.error( str( i ) + " is not in in-memory "
3299 "counter incremented results" )
3300 utilities.assert_equals( expect=True,
3301 actual=pCounterResults,
3302 onpass="In-memory counter incremented",
3303 onfail="Error incrementing in-memory" +
3304 " counter" )
3305
3306 main.step( "Add 5 to then get a in-memory counter on each node" )
3307 iCounters = []
3308 threads = []
3309 addedIValues = []
3310 for i in range( main.numCtrls ):
3311 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3312 name="counterIncrement-" + str( i ),
3313 args=[ iCounterName ],
3314 kwargs={ "delta": 5, "inMemory": True } )
3315 iCounterValue += 5
3316 addedIValues.append( iCounterValue )
3317 threads.append( t )
3318 t.start()
3319
3320 for t in threads:
3321 t.join()
3322 iCounters.append( t.result )
3323 # Check that counter incremented numController times
3324 iCounterResults = True
3325 for i in addedIValues:
3326 tmpResult = i in iCounters
3327 iCounterResults = iCounterResults and tmpResult
3328 if not tmpResult:
3329 main.log.error( str( i ) + " is not in in-memory "
3330 "counter incremented results" )
3331 utilities.assert_equals( expect=True,
3332 actual=pCounterResults,
3333 onpass="In-memory counter incremented",
3334 onfail="Error incrementing in-memory" +
3335 " counter" )
3336
3337 main.step( "Get then add 5 to a in-memory counter on each node" )
3338 iCounters = []
3339 threads = []
3340 addedIValues = []
3341 for i in range( main.numCtrls ):
3342 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3343 name="counterIncrement-" + str( i ),
3344 args=[ iCounterName ],
3345 kwargs={ "delta": 5, "inMemory": True } )
3346 addedIValues.append( iCounterValue )
3347 iCounterValue += 5
3348 threads.append( t )
3349 t.start()
3350
3351 for t in threads:
3352 t.join()
3353 iCounters.append( t.result )
3354 # Check that counter incremented numController times
3355 iCounterResults = True
3356 for i in addedIValues:
3357 tmpResult = i in iCounters
3358 iCounterResults = iCounterResults and tmpResult
3359 if not tmpResult:
3360 main.log.error( str( i ) + " is not in in-memory "
3361 "counter incremented results" )
3362 utilities.assert_equals( expect=True,
3363 actual=iCounterResults,
3364 onpass="In-memory counter incremented",
3365 onfail="Error incrementing in-memory" +
3366 " counter" )
3367
3368 main.step( "Counters we added have the correct values" )
3369 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3370 utilities.assert_equals( expect=main.TRUE,
3371 actual=incrementCheck,
3372 onpass="Added counters are correct",
3373 onfail="Added counters are incorrect" )
3374
Jon Hall5cf14d52015-07-16 12:15:19 -07003375 main.step( "Check counters are consistant across nodes" )
Jon Hall57b50432015-10-22 10:20:10 -07003376 onosCounters, consistentCounterResults = main.Counters.consistentCheck()
Jon Hall5cf14d52015-07-16 12:15:19 -07003377 utilities.assert_equals( expect=main.TRUE,
3378 actual=consistentCounterResults,
3379 onpass="ONOS counters are consistent " +
3380 "across nodes",
3381 onfail="ONOS Counters are inconsistent " +
3382 "across nodes" )
3383
3384 main.step( "Counters we added have the correct values" )
Jon Halle1a3b752015-07-22 13:02:46 -07003385 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue )
3386 incrementCheck = incrementCheck and \
3387 main.Counters.counterCheck( iCounterName, iCounterValue )
Jon Hall5cf14d52015-07-16 12:15:19 -07003388 utilities.assert_equals( expect=main.TRUE,
Jon Halle1a3b752015-07-22 13:02:46 -07003389 actual=incrementCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -07003390 onpass="Added counters are correct",
3391 onfail="Added counters are incorrect" )
3392 # DISTRIBUTED SETS
3393 main.step( "Distributed Set get" )
3394 size = len( onosSet )
3395 getResponses = []
3396 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003397 for i in range( main.numCtrls ):
3398 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003399 name="setTestGet-" + str( i ),
3400 args=[ onosSetName ] )
3401 threads.append( t )
3402 t.start()
3403 for t in threads:
3404 t.join()
3405 getResponses.append( t.result )
3406
3407 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003408 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003409 if isinstance( getResponses[ i ], list):
3410 current = set( getResponses[ i ] )
3411 if len( current ) == len( getResponses[ i ] ):
3412 # no repeats
3413 if onosSet != current:
3414 main.log.error( "ONOS" + str( i + 1 ) +
3415 " has incorrect view" +
3416 " of set " + onosSetName + ":\n" +
3417 str( getResponses[ i ] ) )
3418 main.log.debug( "Expected: " + str( onosSet ) )
3419 main.log.debug( "Actual: " + str( current ) )
3420 getResults = main.FALSE
3421 else:
3422 # error, set is not a set
3423 main.log.error( "ONOS" + str( i + 1 ) +
3424 " has repeat elements in" +
3425 " set " + onosSetName + ":\n" +
3426 str( getResponses[ i ] ) )
3427 getResults = main.FALSE
3428 elif getResponses[ i ] == main.ERROR:
3429 getResults = main.FALSE
3430 utilities.assert_equals( expect=main.TRUE,
3431 actual=getResults,
3432 onpass="Set elements are correct",
3433 onfail="Set elements are incorrect" )
3434
3435 main.step( "Distributed Set size" )
3436 sizeResponses = []
3437 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003438 for i in range( main.numCtrls ):
3439 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003440 name="setTestSize-" + str( i ),
3441 args=[ onosSetName ] )
3442 threads.append( t )
3443 t.start()
3444 for t in threads:
3445 t.join()
3446 sizeResponses.append( t.result )
3447
3448 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003449 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003450 if size != sizeResponses[ i ]:
3451 sizeResults = main.FALSE
3452 main.log.error( "ONOS" + str( i + 1 ) +
3453 " expected a size of " + str( size ) +
3454 " for set " + onosSetName +
3455 " but got " + str( sizeResponses[ i ] ) )
3456 utilities.assert_equals( expect=main.TRUE,
3457 actual=sizeResults,
3458 onpass="Set sizes are correct",
3459 onfail="Set sizes are incorrect" )
3460
3461 main.step( "Distributed Set add()" )
3462 onosSet.add( addValue )
3463 addResponses = []
3464 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003465 for i in range( main.numCtrls ):
3466 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003467 name="setTestAdd-" + str( i ),
3468 args=[ onosSetName, addValue ] )
3469 threads.append( t )
3470 t.start()
3471 for t in threads:
3472 t.join()
3473 addResponses.append( t.result )
3474
3475 # main.TRUE = successfully changed the set
3476 # main.FALSE = action resulted in no change in set
3477 # main.ERROR - Some error in executing the function
3478 addResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003479 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003480 if addResponses[ i ] == main.TRUE:
3481 # All is well
3482 pass
3483 elif addResponses[ i ] == main.FALSE:
3484 # Already in set, probably fine
3485 pass
3486 elif addResponses[ i ] == main.ERROR:
3487 # Error in execution
3488 addResults = main.FALSE
3489 else:
3490 # unexpected result
3491 addResults = main.FALSE
3492 if addResults != main.TRUE:
3493 main.log.error( "Error executing set add" )
3494
3495 # Check if set is still correct
3496 size = len( onosSet )
3497 getResponses = []
3498 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003499 for i in range( main.numCtrls ):
3500 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003501 name="setTestGet-" + str( i ),
3502 args=[ onosSetName ] )
3503 threads.append( t )
3504 t.start()
3505 for t in threads:
3506 t.join()
3507 getResponses.append( t.result )
3508 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003509 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003510 if isinstance( getResponses[ i ], list):
3511 current = set( getResponses[ i ] )
3512 if len( current ) == len( getResponses[ i ] ):
3513 # no repeats
3514 if onosSet != current:
3515 main.log.error( "ONOS" + str( i + 1 ) +
3516 " has incorrect view" +
3517 " of set " + onosSetName + ":\n" +
3518 str( getResponses[ i ] ) )
3519 main.log.debug( "Expected: " + str( onosSet ) )
3520 main.log.debug( "Actual: " + str( current ) )
3521 getResults = main.FALSE
3522 else:
3523 # error, set is not a set
3524 main.log.error( "ONOS" + str( i + 1 ) +
3525 " has repeat elements in" +
3526 " set " + onosSetName + ":\n" +
3527 str( getResponses[ i ] ) )
3528 getResults = main.FALSE
3529 elif getResponses[ i ] == main.ERROR:
3530 getResults = main.FALSE
3531 sizeResponses = []
3532 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003533 for i in range( main.numCtrls ):
3534 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003535 name="setTestSize-" + str( i ),
3536 args=[ onosSetName ] )
3537 threads.append( t )
3538 t.start()
3539 for t in threads:
3540 t.join()
3541 sizeResponses.append( t.result )
3542 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003543 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003544 if size != sizeResponses[ i ]:
3545 sizeResults = main.FALSE
3546 main.log.error( "ONOS" + str( i + 1 ) +
3547 " expected a size of " + str( size ) +
3548 " for set " + onosSetName +
3549 " but got " + str( sizeResponses[ i ] ) )
3550 addResults = addResults and getResults and sizeResults
3551 utilities.assert_equals( expect=main.TRUE,
3552 actual=addResults,
3553 onpass="Set add correct",
3554 onfail="Set add was incorrect" )
3555
3556 main.step( "Distributed Set addAll()" )
3557 onosSet.update( addAllValue.split() )
3558 addResponses = []
3559 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003560 for i in range( main.numCtrls ):
3561 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003562 name="setTestAddAll-" + str( i ),
3563 args=[ onosSetName, addAllValue ] )
3564 threads.append( t )
3565 t.start()
3566 for t in threads:
3567 t.join()
3568 addResponses.append( t.result )
3569
3570 # main.TRUE = successfully changed the set
3571 # main.FALSE = action resulted in no change in set
3572 # main.ERROR - Some error in executing the function
3573 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003574 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003575 if addResponses[ i ] == main.TRUE:
3576 # All is well
3577 pass
3578 elif addResponses[ i ] == main.FALSE:
3579 # Already in set, probably fine
3580 pass
3581 elif addResponses[ i ] == main.ERROR:
3582 # Error in execution
3583 addAllResults = main.FALSE
3584 else:
3585 # unexpected result
3586 addAllResults = main.FALSE
3587 if addAllResults != main.TRUE:
3588 main.log.error( "Error executing set addAll" )
3589
3590 # Check if set is still correct
3591 size = len( onosSet )
3592 getResponses = []
3593 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003594 for i in range( main.numCtrls ):
3595 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003596 name="setTestGet-" + str( i ),
3597 args=[ onosSetName ] )
3598 threads.append( t )
3599 t.start()
3600 for t in threads:
3601 t.join()
3602 getResponses.append( t.result )
3603 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003604 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003605 if isinstance( getResponses[ i ], list):
3606 current = set( getResponses[ i ] )
3607 if len( current ) == len( getResponses[ i ] ):
3608 # no repeats
3609 if onosSet != current:
3610 main.log.error( "ONOS" + str( i + 1 ) +
3611 " has incorrect view" +
3612 " of set " + onosSetName + ":\n" +
3613 str( getResponses[ i ] ) )
3614 main.log.debug( "Expected: " + str( onosSet ) )
3615 main.log.debug( "Actual: " + str( current ) )
3616 getResults = main.FALSE
3617 else:
3618 # error, set is not a set
3619 main.log.error( "ONOS" + str( i + 1 ) +
3620 " has repeat elements in" +
3621 " set " + onosSetName + ":\n" +
3622 str( getResponses[ i ] ) )
3623 getResults = main.FALSE
3624 elif getResponses[ i ] == main.ERROR:
3625 getResults = main.FALSE
3626 sizeResponses = []
3627 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003628 for i in range( main.numCtrls ):
3629 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003630 name="setTestSize-" + str( i ),
3631 args=[ onosSetName ] )
3632 threads.append( t )
3633 t.start()
3634 for t in threads:
3635 t.join()
3636 sizeResponses.append( t.result )
3637 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003638 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003639 if size != sizeResponses[ i ]:
3640 sizeResults = main.FALSE
3641 main.log.error( "ONOS" + str( i + 1 ) +
3642 " expected a size of " + str( size ) +
3643 " for set " + onosSetName +
3644 " but got " + str( sizeResponses[ i ] ) )
3645 addAllResults = addAllResults and getResults and sizeResults
3646 utilities.assert_equals( expect=main.TRUE,
3647 actual=addAllResults,
3648 onpass="Set addAll correct",
3649 onfail="Set addAll was incorrect" )
3650
3651 main.step( "Distributed Set contains()" )
3652 containsResponses = []
3653 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003654 for i in range( main.numCtrls ):
3655 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003656 name="setContains-" + str( i ),
3657 args=[ onosSetName ],
3658 kwargs={ "values": addValue } )
3659 threads.append( t )
3660 t.start()
3661 for t in threads:
3662 t.join()
3663 # NOTE: This is the tuple
3664 containsResponses.append( t.result )
3665
3666 containsResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003667 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003668 if containsResponses[ i ] == main.ERROR:
3669 containsResults = main.FALSE
3670 else:
3671 containsResults = containsResults and\
3672 containsResponses[ i ][ 1 ]
3673 utilities.assert_equals( expect=main.TRUE,
3674 actual=containsResults,
3675 onpass="Set contains is functional",
3676 onfail="Set contains failed" )
3677
3678 main.step( "Distributed Set containsAll()" )
3679 containsAllResponses = []
3680 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003681 for i in range( main.numCtrls ):
3682 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003683 name="setContainsAll-" + str( i ),
3684 args=[ onosSetName ],
3685 kwargs={ "values": addAllValue } )
3686 threads.append( t )
3687 t.start()
3688 for t in threads:
3689 t.join()
3690 # NOTE: This is the tuple
3691 containsAllResponses.append( t.result )
3692
3693 containsAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003694 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003695 if containsResponses[ i ] == main.ERROR:
3696 containsResults = main.FALSE
3697 else:
3698 containsResults = containsResults and\
3699 containsResponses[ i ][ 1 ]
3700 utilities.assert_equals( expect=main.TRUE,
3701 actual=containsAllResults,
3702 onpass="Set containsAll is functional",
3703 onfail="Set containsAll failed" )
3704
3705 main.step( "Distributed Set remove()" )
3706 onosSet.remove( addValue )
3707 removeResponses = []
3708 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003709 for i in range( main.numCtrls ):
3710 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003711 name="setTestRemove-" + str( i ),
3712 args=[ onosSetName, addValue ] )
3713 threads.append( t )
3714 t.start()
3715 for t in threads:
3716 t.join()
3717 removeResponses.append( t.result )
3718
3719 # main.TRUE = successfully changed the set
3720 # main.FALSE = action resulted in no change in set
3721 # main.ERROR - Some error in executing the function
3722 removeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003723 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003724 if removeResponses[ i ] == main.TRUE:
3725 # All is well
3726 pass
3727 elif removeResponses[ i ] == main.FALSE:
3728 # not in set, probably fine
3729 pass
3730 elif removeResponses[ i ] == main.ERROR:
3731 # Error in execution
3732 removeResults = main.FALSE
3733 else:
3734 # unexpected result
3735 removeResults = main.FALSE
3736 if removeResults != main.TRUE:
3737 main.log.error( "Error executing set remove" )
3738
3739 # Check if set is still correct
3740 size = len( onosSet )
3741 getResponses = []
3742 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003743 for i in range( main.numCtrls ):
3744 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003745 name="setTestGet-" + str( i ),
3746 args=[ onosSetName ] )
3747 threads.append( t )
3748 t.start()
3749 for t in threads:
3750 t.join()
3751 getResponses.append( t.result )
3752 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003753 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003754 if isinstance( getResponses[ i ], list):
3755 current = set( getResponses[ i ] )
3756 if len( current ) == len( getResponses[ i ] ):
3757 # no repeats
3758 if onosSet != current:
3759 main.log.error( "ONOS" + str( i + 1 ) +
3760 " has incorrect view" +
3761 " of set " + onosSetName + ":\n" +
3762 str( getResponses[ i ] ) )
3763 main.log.debug( "Expected: " + str( onosSet ) )
3764 main.log.debug( "Actual: " + str( current ) )
3765 getResults = main.FALSE
3766 else:
3767 # error, set is not a set
3768 main.log.error( "ONOS" + str( i + 1 ) +
3769 " has repeat elements in" +
3770 " set " + onosSetName + ":\n" +
3771 str( getResponses[ i ] ) )
3772 getResults = main.FALSE
3773 elif getResponses[ i ] == main.ERROR:
3774 getResults = main.FALSE
3775 sizeResponses = []
3776 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003777 for i in range( main.numCtrls ):
3778 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003779 name="setTestSize-" + str( i ),
3780 args=[ onosSetName ] )
3781 threads.append( t )
3782 t.start()
3783 for t in threads:
3784 t.join()
3785 sizeResponses.append( t.result )
3786 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003787 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003788 if size != sizeResponses[ i ]:
3789 sizeResults = main.FALSE
3790 main.log.error( "ONOS" + str( i + 1 ) +
3791 " expected a size of " + str( size ) +
3792 " for set " + onosSetName +
3793 " but got " + str( sizeResponses[ i ] ) )
3794 removeResults = removeResults and getResults and sizeResults
3795 utilities.assert_equals( expect=main.TRUE,
3796 actual=removeResults,
3797 onpass="Set remove correct",
3798 onfail="Set remove was incorrect" )
3799
3800 main.step( "Distributed Set removeAll()" )
3801 onosSet.difference_update( addAllValue.split() )
3802 removeAllResponses = []
3803 threads = []
3804 try:
Jon Halle1a3b752015-07-22 13:02:46 -07003805 for i in range( main.numCtrls ):
3806 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003807 name="setTestRemoveAll-" + str( i ),
3808 args=[ onosSetName, addAllValue ] )
3809 threads.append( t )
3810 t.start()
3811 for t in threads:
3812 t.join()
3813 removeAllResponses.append( t.result )
3814 except Exception, e:
3815 main.log.exception(e)
3816
3817 # main.TRUE = successfully changed the set
3818 # main.FALSE = action resulted in no change in set
3819 # main.ERROR - Some error in executing the function
3820 removeAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003821 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003822 if removeAllResponses[ i ] == main.TRUE:
3823 # All is well
3824 pass
3825 elif removeAllResponses[ i ] == main.FALSE:
3826 # not in set, probably fine
3827 pass
3828 elif removeAllResponses[ i ] == main.ERROR:
3829 # Error in execution
3830 removeAllResults = main.FALSE
3831 else:
3832 # unexpected result
3833 removeAllResults = main.FALSE
3834 if removeAllResults != main.TRUE:
3835 main.log.error( "Error executing set removeAll" )
3836
3837 # Check if set is still correct
3838 size = len( onosSet )
3839 getResponses = []
3840 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003841 for i in range( main.numCtrls ):
3842 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003843 name="setTestGet-" + str( i ),
3844 args=[ onosSetName ] )
3845 threads.append( t )
3846 t.start()
3847 for t in threads:
3848 t.join()
3849 getResponses.append( t.result )
3850 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003851 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003852 if isinstance( getResponses[ i ], list):
3853 current = set( getResponses[ i ] )
3854 if len( current ) == len( getResponses[ i ] ):
3855 # no repeats
3856 if onosSet != current:
3857 main.log.error( "ONOS" + str( i + 1 ) +
3858 " has incorrect view" +
3859 " of set " + onosSetName + ":\n" +
3860 str( getResponses[ i ] ) )
3861 main.log.debug( "Expected: " + str( onosSet ) )
3862 main.log.debug( "Actual: " + str( current ) )
3863 getResults = main.FALSE
3864 else:
3865 # error, set is not a set
3866 main.log.error( "ONOS" + str( i + 1 ) +
3867 " has repeat elements in" +
3868 " set " + onosSetName + ":\n" +
3869 str( getResponses[ i ] ) )
3870 getResults = main.FALSE
3871 elif getResponses[ i ] == main.ERROR:
3872 getResults = main.FALSE
3873 sizeResponses = []
3874 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003875 for i in range( main.numCtrls ):
3876 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003877 name="setTestSize-" + str( i ),
3878 args=[ onosSetName ] )
3879 threads.append( t )
3880 t.start()
3881 for t in threads:
3882 t.join()
3883 sizeResponses.append( t.result )
3884 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003885 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003886 if size != sizeResponses[ i ]:
3887 sizeResults = main.FALSE
3888 main.log.error( "ONOS" + str( i + 1 ) +
3889 " expected a size of " + str( size ) +
3890 " for set " + onosSetName +
3891 " but got " + str( sizeResponses[ i ] ) )
3892 removeAllResults = removeAllResults and getResults and sizeResults
3893 utilities.assert_equals( expect=main.TRUE,
3894 actual=removeAllResults,
3895 onpass="Set removeAll correct",
3896 onfail="Set removeAll was incorrect" )
3897
3898 main.step( "Distributed Set addAll()" )
3899 onosSet.update( addAllValue.split() )
3900 addResponses = []
3901 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003902 for i in range( main.numCtrls ):
3903 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003904 name="setTestAddAll-" + str( i ),
3905 args=[ onosSetName, addAllValue ] )
3906 threads.append( t )
3907 t.start()
3908 for t in threads:
3909 t.join()
3910 addResponses.append( t.result )
3911
3912 # main.TRUE = successfully changed the set
3913 # main.FALSE = action resulted in no change in set
3914 # main.ERROR - Some error in executing the function
3915 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003916 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003917 if addResponses[ i ] == main.TRUE:
3918 # All is well
3919 pass
3920 elif addResponses[ i ] == main.FALSE:
3921 # Already in set, probably fine
3922 pass
3923 elif addResponses[ i ] == main.ERROR:
3924 # Error in execution
3925 addAllResults = main.FALSE
3926 else:
3927 # unexpected result
3928 addAllResults = main.FALSE
3929 if addAllResults != main.TRUE:
3930 main.log.error( "Error executing set addAll" )
3931
3932 # Check if set is still correct
3933 size = len( onosSet )
3934 getResponses = []
3935 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003936 for i in range( main.numCtrls ):
3937 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003938 name="setTestGet-" + str( i ),
3939 args=[ onosSetName ] )
3940 threads.append( t )
3941 t.start()
3942 for t in threads:
3943 t.join()
3944 getResponses.append( t.result )
3945 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003946 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003947 if isinstance( getResponses[ i ], list):
3948 current = set( getResponses[ i ] )
3949 if len( current ) == len( getResponses[ i ] ):
3950 # no repeats
3951 if onosSet != current:
3952 main.log.error( "ONOS" + str( i + 1 ) +
3953 " has incorrect view" +
3954 " of set " + onosSetName + ":\n" +
3955 str( getResponses[ i ] ) )
3956 main.log.debug( "Expected: " + str( onosSet ) )
3957 main.log.debug( "Actual: " + str( current ) )
3958 getResults = main.FALSE
3959 else:
3960 # error, set is not a set
3961 main.log.error( "ONOS" + str( i + 1 ) +
3962 " has repeat elements in" +
3963 " set " + onosSetName + ":\n" +
3964 str( getResponses[ i ] ) )
3965 getResults = main.FALSE
3966 elif getResponses[ i ] == main.ERROR:
3967 getResults = main.FALSE
3968 sizeResponses = []
3969 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003970 for i in range( main.numCtrls ):
3971 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003972 name="setTestSize-" + str( i ),
3973 args=[ onosSetName ] )
3974 threads.append( t )
3975 t.start()
3976 for t in threads:
3977 t.join()
3978 sizeResponses.append( t.result )
3979 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07003980 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003981 if size != sizeResponses[ i ]:
3982 sizeResults = main.FALSE
3983 main.log.error( "ONOS" + str( i + 1 ) +
3984 " expected a size of " + str( size ) +
3985 " for set " + onosSetName +
3986 " but got " + str( sizeResponses[ i ] ) )
3987 addAllResults = addAllResults and getResults and sizeResults
3988 utilities.assert_equals( expect=main.TRUE,
3989 actual=addAllResults,
3990 onpass="Set addAll correct",
3991 onfail="Set addAll was incorrect" )
3992
3993 main.step( "Distributed Set clear()" )
3994 onosSet.clear()
3995 clearResponses = []
3996 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07003997 for i in range( main.numCtrls ):
3998 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003999 name="setTestClear-" + str( i ),
4000 args=[ onosSetName, " "], # Values doesn't matter
4001 kwargs={ "clear": True } )
4002 threads.append( t )
4003 t.start()
4004 for t in threads:
4005 t.join()
4006 clearResponses.append( t.result )
4007
4008 # main.TRUE = successfully changed the set
4009 # main.FALSE = action resulted in no change in set
4010 # main.ERROR - Some error in executing the function
4011 clearResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004012 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004013 if clearResponses[ i ] == main.TRUE:
4014 # All is well
4015 pass
4016 elif clearResponses[ i ] == main.FALSE:
4017 # Nothing set, probably fine
4018 pass
4019 elif clearResponses[ i ] == main.ERROR:
4020 # Error in execution
4021 clearResults = main.FALSE
4022 else:
4023 # unexpected result
4024 clearResults = main.FALSE
4025 if clearResults != main.TRUE:
4026 main.log.error( "Error executing set clear" )
4027
4028 # Check if set is still correct
4029 size = len( onosSet )
4030 getResponses = []
4031 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004032 for i in range( main.numCtrls ):
4033 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004034 name="setTestGet-" + str( i ),
4035 args=[ onosSetName ] )
4036 threads.append( t )
4037 t.start()
4038 for t in threads:
4039 t.join()
4040 getResponses.append( t.result )
4041 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004042 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004043 if isinstance( getResponses[ i ], list):
4044 current = set( getResponses[ i ] )
4045 if len( current ) == len( getResponses[ i ] ):
4046 # no repeats
4047 if onosSet != current:
4048 main.log.error( "ONOS" + str( i + 1 ) +
4049 " has incorrect view" +
4050 " of set " + onosSetName + ":\n" +
4051 str( getResponses[ i ] ) )
4052 main.log.debug( "Expected: " + str( onosSet ) )
4053 main.log.debug( "Actual: " + str( current ) )
4054 getResults = main.FALSE
4055 else:
4056 # error, set is not a set
4057 main.log.error( "ONOS" + str( i + 1 ) +
4058 " has repeat elements in" +
4059 " set " + onosSetName + ":\n" +
4060 str( getResponses[ i ] ) )
4061 getResults = main.FALSE
4062 elif getResponses[ i ] == main.ERROR:
4063 getResults = main.FALSE
4064 sizeResponses = []
4065 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004066 for i in range( main.numCtrls ):
4067 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004068 name="setTestSize-" + str( i ),
4069 args=[ onosSetName ] )
4070 threads.append( t )
4071 t.start()
4072 for t in threads:
4073 t.join()
4074 sizeResponses.append( t.result )
4075 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004076 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004077 if size != sizeResponses[ i ]:
4078 sizeResults = main.FALSE
4079 main.log.error( "ONOS" + str( i + 1 ) +
4080 " expected a size of " + str( size ) +
4081 " for set " + onosSetName +
4082 " but got " + str( sizeResponses[ i ] ) )
4083 clearResults = clearResults and getResults and sizeResults
4084 utilities.assert_equals( expect=main.TRUE,
4085 actual=clearResults,
4086 onpass="Set clear correct",
4087 onfail="Set clear was incorrect" )
4088
4089 main.step( "Distributed Set addAll()" )
4090 onosSet.update( addAllValue.split() )
4091 addResponses = []
4092 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004093 for i in range( main.numCtrls ):
4094 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004095 name="setTestAddAll-" + str( i ),
4096 args=[ onosSetName, addAllValue ] )
4097 threads.append( t )
4098 t.start()
4099 for t in threads:
4100 t.join()
4101 addResponses.append( t.result )
4102
4103 # main.TRUE = successfully changed the set
4104 # main.FALSE = action resulted in no change in set
4105 # main.ERROR - Some error in executing the function
4106 addAllResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004107 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004108 if addResponses[ i ] == main.TRUE:
4109 # All is well
4110 pass
4111 elif addResponses[ i ] == main.FALSE:
4112 # Already in set, probably fine
4113 pass
4114 elif addResponses[ i ] == main.ERROR:
4115 # Error in execution
4116 addAllResults = main.FALSE
4117 else:
4118 # unexpected result
4119 addAllResults = main.FALSE
4120 if addAllResults != main.TRUE:
4121 main.log.error( "Error executing set addAll" )
4122
4123 # Check if set is still correct
4124 size = len( onosSet )
4125 getResponses = []
4126 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004127 for i in range( main.numCtrls ):
4128 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004129 name="setTestGet-" + str( i ),
4130 args=[ onosSetName ] )
4131 threads.append( t )
4132 t.start()
4133 for t in threads:
4134 t.join()
4135 getResponses.append( t.result )
4136 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004137 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004138 if isinstance( getResponses[ i ], list):
4139 current = set( getResponses[ i ] )
4140 if len( current ) == len( getResponses[ i ] ):
4141 # no repeats
4142 if onosSet != current:
4143 main.log.error( "ONOS" + str( i + 1 ) +
4144 " has incorrect view" +
4145 " of set " + onosSetName + ":\n" +
4146 str( getResponses[ i ] ) )
4147 main.log.debug( "Expected: " + str( onosSet ) )
4148 main.log.debug( "Actual: " + str( current ) )
4149 getResults = main.FALSE
4150 else:
4151 # error, set is not a set
4152 main.log.error( "ONOS" + str( i + 1 ) +
4153 " has repeat elements in" +
4154 " set " + onosSetName + ":\n" +
4155 str( getResponses[ i ] ) )
4156 getResults = main.FALSE
4157 elif getResponses[ i ] == main.ERROR:
4158 getResults = main.FALSE
4159 sizeResponses = []
4160 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004161 for i in range( main.numCtrls ):
4162 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004163 name="setTestSize-" + str( i ),
4164 args=[ onosSetName ] )
4165 threads.append( t )
4166 t.start()
4167 for t in threads:
4168 t.join()
4169 sizeResponses.append( t.result )
4170 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004171 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004172 if size != sizeResponses[ i ]:
4173 sizeResults = main.FALSE
4174 main.log.error( "ONOS" + str( i + 1 ) +
4175 " expected a size of " + str( size ) +
4176 " for set " + onosSetName +
4177 " but got " + str( sizeResponses[ i ] ) )
4178 addAllResults = addAllResults and getResults and sizeResults
4179 utilities.assert_equals( expect=main.TRUE,
4180 actual=addAllResults,
4181 onpass="Set addAll correct",
4182 onfail="Set addAll was incorrect" )
4183
4184 main.step( "Distributed Set retain()" )
4185 onosSet.intersection_update( retainValue.split() )
4186 retainResponses = []
4187 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004188 for i in range( main.numCtrls ):
4189 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004190 name="setTestRetain-" + str( i ),
4191 args=[ onosSetName, retainValue ],
4192 kwargs={ "retain": True } )
4193 threads.append( t )
4194 t.start()
4195 for t in threads:
4196 t.join()
4197 retainResponses.append( t.result )
4198
4199 # main.TRUE = successfully changed the set
4200 # main.FALSE = action resulted in no change in set
4201 # main.ERROR - Some error in executing the function
4202 retainResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004203 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004204 if retainResponses[ i ] == main.TRUE:
4205 # All is well
4206 pass
4207 elif retainResponses[ i ] == main.FALSE:
4208 # Already in set, probably fine
4209 pass
4210 elif retainResponses[ i ] == main.ERROR:
4211 # Error in execution
4212 retainResults = main.FALSE
4213 else:
4214 # unexpected result
4215 retainResults = main.FALSE
4216 if retainResults != main.TRUE:
4217 main.log.error( "Error executing set retain" )
4218
4219 # Check if set is still correct
4220 size = len( onosSet )
4221 getResponses = []
4222 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004223 for i in range( main.numCtrls ):
4224 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004225 name="setTestGet-" + str( i ),
4226 args=[ onosSetName ] )
4227 threads.append( t )
4228 t.start()
4229 for t in threads:
4230 t.join()
4231 getResponses.append( t.result )
4232 getResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004233 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004234 if isinstance( getResponses[ i ], list):
4235 current = set( getResponses[ i ] )
4236 if len( current ) == len( getResponses[ i ] ):
4237 # no repeats
4238 if onosSet != current:
4239 main.log.error( "ONOS" + str( i + 1 ) +
4240 " has incorrect view" +
4241 " of set " + onosSetName + ":\n" +
4242 str( getResponses[ i ] ) )
4243 main.log.debug( "Expected: " + str( onosSet ) )
4244 main.log.debug( "Actual: " + str( current ) )
4245 getResults = main.FALSE
4246 else:
4247 # error, set is not a set
4248 main.log.error( "ONOS" + str( i + 1 ) +
4249 " has repeat elements in" +
4250 " set " + onosSetName + ":\n" +
4251 str( getResponses[ i ] ) )
4252 getResults = main.FALSE
4253 elif getResponses[ i ] == main.ERROR:
4254 getResults = main.FALSE
4255 sizeResponses = []
4256 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07004257 for i in range( main.numCtrls ):
4258 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004259 name="setTestSize-" + str( i ),
4260 args=[ onosSetName ] )
4261 threads.append( t )
4262 t.start()
4263 for t in threads:
4264 t.join()
4265 sizeResponses.append( t.result )
4266 sizeResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07004267 for i in range( main.numCtrls ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004268 if size != sizeResponses[ i ]:
4269 sizeResults = main.FALSE
4270 main.log.error( "ONOS" + str( i + 1 ) +
4271 " expected a size of " +
4272 str( size ) + " for set " + onosSetName +
4273 " but got " + str( sizeResponses[ i ] ) )
4274 retainResults = retainResults and getResults and sizeResults
4275 utilities.assert_equals( expect=main.TRUE,
4276 actual=retainResults,
4277 onpass="Set retain correct",
4278 onfail="Set retain was incorrect" )
4279
Jon Hall2a5002c2015-08-21 16:49:11 -07004280 # Transactional maps
4281 main.step( "Partitioned Transactional maps put" )
4282 tMapValue = "Testing"
4283 numKeys = 100
4284 putResult = True
4285 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue )
4286 if len( putResponses ) == 100:
4287 for i in putResponses:
4288 if putResponses[ i ][ 'value' ] != tMapValue:
4289 putResult = False
4290 else:
4291 putResult = False
4292 if not putResult:
4293 main.log.debug( "Put response values: " + str( putResponses ) )
4294 utilities.assert_equals( expect=True,
4295 actual=putResult,
4296 onpass="Partitioned Transactional Map put successful",
4297 onfail="Partitioned Transactional Map put values are incorrect" )
4298
4299 main.step( "Partitioned Transactional maps get" )
4300 getCheck = True
4301 for n in range( 1, numKeys + 1 ):
4302 getResponses = []
4303 threads = []
4304 valueCheck = True
4305 for i in range( main.numCtrls ):
4306 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4307 name="TMap-get-" + str( i ),
4308 args=[ "Key" + str ( n ) ] )
4309 threads.append( t )
4310 t.start()
4311 for t in threads:
4312 t.join()
4313 getResponses.append( t.result )
4314 for node in getResponses:
4315 if node != tMapValue:
4316 valueCheck = False
4317 if not valueCheck:
4318 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4319 main.log.warn( getResponses )
4320 getCheck = getCheck and valueCheck
4321 utilities.assert_equals( expect=True,
4322 actual=getCheck,
4323 onpass="Partitioned Transactional Map get values were correct",
4324 onfail="Partitioned Transactional Map values incorrect" )
4325
4326 main.step( "In-memory Transactional maps put" )
4327 tMapValue = "Testing"
4328 numKeys = 100
4329 putResult = True
4330 putResponses = main.CLIs[ 0 ].transactionalMapPut( numKeys, tMapValue, inMemory=True )
4331 if len( putResponses ) == 100:
4332 for i in putResponses:
4333 if putResponses[ i ][ 'value' ] != tMapValue:
4334 putResult = False
4335 else:
4336 putResult = False
4337 if not putResult:
4338 main.log.debug( "Put response values: " + str( putResponses ) )
4339 utilities.assert_equals( expect=True,
4340 actual=putResult,
4341 onpass="In-Memory Transactional Map put successful",
4342 onfail="In-Memory Transactional Map put values are incorrect" )
4343
4344 main.step( "In-Memory Transactional maps get" )
4345 getCheck = True
4346 for n in range( 1, numKeys + 1 ):
4347 getResponses = []
4348 threads = []
4349 valueCheck = True
4350 for i in range( main.numCtrls ):
4351 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4352 name="TMap-get-" + str( i ),
4353 args=[ "Key" + str ( n ) ],
4354 kwargs={ "inMemory": True } )
4355 threads.append( t )
4356 t.start()
4357 for t in threads:
4358 t.join()
4359 getResponses.append( t.result )
4360 for node in getResponses:
4361 if node != tMapValue:
4362 valueCheck = False
4363 if not valueCheck:
4364 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4365 main.log.warn( getResponses )
4366 getCheck = getCheck and valueCheck
4367 utilities.assert_equals( expect=True,
4368 actual=getCheck,
4369 onpass="In-Memory Transactional Map get values were correct",
4370 onfail="In-Memory Transactional Map values incorrect" )