blob: afd3f7841e432f11453e86c28a466c82e3c50756 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAclusterRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080051 import time
Jon Halla440e872016-03-31 15:15:50 -070052 import json
Jon Hall5cf14d52015-07-16 12:15:19 -070053 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
54 "initialization" )
55 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070056 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070057 "installing ONOS, starting Mininet and ONOS" +\
58 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070059
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
71 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
79 # These are for csv plotting in jenkins
80 global labels
81 global data
82 labels = []
83 data = []
84
85 # FIXME: just get controller port from params?
86 # TODO: do we really need all these?
87 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
88 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
89 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
90 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
91 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
92 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
93 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
94
Jon Halle1a3b752015-07-22 13:02:46 -070095 try:
Jon Hall53c5e662016-04-13 16:06:56 -070096 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070097 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070098 except Exception as e:
99 main.log.exception( e )
100 main.cleanup()
101 main.exit()
102
103 main.CLIs = []
104 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700105 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700106 for i in range( 1, main.numCtrls + 1 ):
107 try:
108 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
109 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
110 ipList.append( main.nodes[ -1 ].ip_address )
111 except AttributeError:
112 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700113
114 main.step( "Create cell file" )
115 cellAppString = main.params[ 'ENV' ][ 'appString' ]
116 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
117 main.Mininet1.ip_address,
118 cellAppString, ipList )
119 main.step( "Applying cell variable to environment" )
120 cellResult = main.ONOSbench.setCell( cellName )
121 verifyResult = main.ONOSbench.verifyCell()
122
123 # FIXME:this is short term fix
124 main.log.info( "Removing raft logs" )
125 main.ONOSbench.onosRemoveRaftLogs()
126
127 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700128 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700129 main.ONOSbench.onosUninstall( node.ip_address )
130
131 # Make sure ONOS is DEAD
132 main.log.info( "Killing any ONOS processes" )
133 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700134 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700135 killed = main.ONOSbench.onosKill( node.ip_address )
136 killResults = killResults and killed
137
138 cleanInstallResult = main.TRUE
139 gitPullResult = main.TRUE
140
141 main.step( "Starting Mininet" )
142 # scp topo file to mininet
143 # TODO: move to params?
144 topoName = "obelisk.py"
145 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700146 main.ONOSbench.scp( main.Mininet1,
147 filePath + topoName,
148 main.Mininet1.home,
149 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700150 mnResult = main.Mininet1.startNet( )
151 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
152 onpass="Mininet Started",
153 onfail="Error starting Mininet" )
154
155 main.step( "Git checkout and pull " + gitBranch )
156 if PULLCODE:
157 main.ONOSbench.gitCheckout( gitBranch )
158 gitPullResult = main.ONOSbench.gitPull()
159 # values of 1 or 3 are good
160 utilities.assert_lesser( expect=0, actual=gitPullResult,
161 onpass="Git pull successful",
162 onfail="Git pull failed" )
163 main.ONOSbench.getVersion( report=True )
164
165 main.step( "Using mvn clean install" )
166 cleanInstallResult = main.TRUE
167 if PULLCODE and gitPullResult == main.TRUE:
168 cleanInstallResult = main.ONOSbench.cleanInstall()
169 else:
170 main.log.warn( "Did not pull new code so skipping mvn " +
171 "clean install" )
172 utilities.assert_equals( expect=main.TRUE,
173 actual=cleanInstallResult,
174 onpass="MCI successful",
175 onfail="MCI failed" )
176 # GRAPHS
177 # NOTE: important params here:
178 # job = name of Jenkins job
179 # Plot Name = Plot-HA, only can be used if multiple plots
180 # index = The number of the graph under plot name
181 job = "HAclusterRestart"
182 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700183 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700184 graphs = '<ac:structured-macro ac:name="html">\n'
185 graphs += '<ac:plain-text-body><![CDATA[\n'
186 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800187 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700188 '&width=500&height=300"' +\
189 'noborder="0" width="500" height="300" scrolling="yes" ' +\
190 'seamless="seamless"></iframe>\n'
191 graphs += ']]></ac:plain-text-body>\n'
192 graphs += '</ac:structured-macro>\n'
193 main.log.wiki(graphs)
194
195 main.step( "Creating ONOS package" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700196 packageResult = main.ONOSbench.buckBuild()
Jon Hall5cf14d52015-07-16 12:15:19 -0700197 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
198 onpass="ONOS package successful",
199 onfail="ONOS package failed" )
200
201 main.step( "Installing ONOS package" )
202 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700203 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700204 tmpResult = main.ONOSbench.onosInstall( options="-f",
205 node=node.ip_address )
206 onosInstallResult = onosInstallResult and tmpResult
207 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
208 onpass="ONOS install successful",
209 onfail="ONOS install failed" )
210
You Wangf5de25b2017-01-06 15:13:01 -0800211 main.step( "Set up ONOS secure SSH" )
212 secureSshResult = main.TRUE
213 for node in main.nodes:
214 secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=node.ip_address )
215 utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
216 onpass="Test step PASS",
217 onfail="Test step FAIL" )
218
Jon Hall5cf14d52015-07-16 12:15:19 -0700219 main.step( "Checking if ONOS is up yet" )
220 for i in range( 2 ):
221 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700222 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700223 started = main.ONOSbench.isup( node.ip_address )
224 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800225 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700226 onosIsupResult = onosIsupResult and started
227 if onosIsupResult == main.TRUE:
228 break
229 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
230 onpass="ONOS startup successful",
231 onfail="ONOS startup failed" )
232
Jon Hall6509dbf2016-06-21 17:01:17 -0700233 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700234 cliResults = main.TRUE
235 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700236 for i in range( main.numCtrls ):
237 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700238 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700239 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700240 threads.append( t )
241 t.start()
242
243 for t in threads:
244 t.join()
245 cliResults = cliResults and t.result
246 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
247 onpass="ONOS cli startup successful",
248 onfail="ONOS cli startup failed" )
249
Jon Halla440e872016-03-31 15:15:50 -0700250 # Create a list of active nodes for use when some nodes are stopped
251 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
252
Jon Hall5cf14d52015-07-16 12:15:19 -0700253 if main.params[ 'tcpdump' ].lower() == "true":
254 main.step( "Start Packet Capture MN" )
255 main.Mininet2.startTcpdump(
256 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
257 + "-MN.pcap",
258 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
259 port=main.params[ 'MNtcpdump' ][ 'port' ] )
260
Jon Halla440e872016-03-31 15:15:50 -0700261 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700262 nodeResults = utilities.retry( main.HA.nodesCheck,
263 False,
264 args=[main.activeNodes],
265 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700266
Jon Hall41d39f12016-04-11 22:54:35 -0700267 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700268 onpass="Nodes check successful",
269 onfail="Nodes check NOT successful" )
270
271 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700272 for i in main.activeNodes:
273 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700274 main.log.debug( "{} components not ACTIVE: \n{}".format(
275 cli.name,
276 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700277 main.log.error( "Failed to start ONOS, stopping test" )
278 main.cleanup()
279 main.exit()
280
Jon Hall172b7ba2016-04-07 18:12:20 -0700281 main.step( "Activate apps defined in the params file" )
282 # get data from the params
283 apps = main.params.get( 'apps' )
284 if apps:
285 apps = apps.split(',')
286 main.log.warn( apps )
287 activateResult = True
288 for app in apps:
289 main.CLIs[ 0 ].app( app, "Activate" )
290 # TODO: check this worked
291 time.sleep( 10 ) # wait for apps to activate
292 for app in apps:
293 state = main.CLIs[ 0 ].appStatus( app )
294 if state == "ACTIVE":
295 activateResult = activeResult and True
296 else:
297 main.log.error( "{} is in {} state".format( app, state ) )
298 activeResult = False
299 utilities.assert_equals( expect=True,
300 actual=activateResult,
301 onpass="Successfully activated apps",
302 onfail="Failed to activate apps" )
303 else:
304 main.log.warn( "No apps were specified to be loaded after startup" )
305
306 main.step( "Set ONOS configurations" )
307 config = main.params.get( 'ONOS_Configuration' )
308 if config:
309 main.log.debug( config )
310 checkResult = main.TRUE
311 for component in config:
312 for setting in config[component]:
313 value = config[component][setting]
314 check = main.CLIs[ 0 ].setCfg( component, setting, value )
315 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
316 checkResult = check and checkResult
317 utilities.assert_equals( expect=main.TRUE,
318 actual=checkResult,
319 onpass="Successfully set config",
320 onfail="Failed to set config" )
321 else:
322 main.log.warn( "No configurations were specified to be changed after startup" )
323
Jon Hall9d2dcad2016-04-08 10:15:20 -0700324 main.step( "App Ids check" )
325 appCheck = main.TRUE
326 threads = []
327 for i in main.activeNodes:
328 t = main.Thread( target=main.CLIs[i].appToIDCheck,
329 name="appToIDCheck-" + str( i ),
330 args=[] )
331 threads.append( t )
332 t.start()
333
334 for t in threads:
335 t.join()
336 appCheck = appCheck and t.result
337 if appCheck != main.TRUE:
338 node = main.activeNodes[0]
339 main.log.warn( main.CLIs[node].apps() )
340 main.log.warn( main.CLIs[node].appIDs() )
341 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
342 onpass="App Ids seem to be correct",
343 onfail="Something is wrong with app Ids" )
344
Jon Hall5cf14d52015-07-16 12:15:19 -0700345 def CASE2( self, main ):
346 """
347 Assign devices to controllers
348 """
349 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700350 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700351 assert main, "main not defined"
352 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700353 assert main.CLIs, "main.CLIs not defined"
354 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700355 assert ONOS1Port, "ONOS1Port not defined"
356 assert ONOS2Port, "ONOS2Port not defined"
357 assert ONOS3Port, "ONOS3Port not defined"
358 assert ONOS4Port, "ONOS4Port not defined"
359 assert ONOS5Port, "ONOS5Port not defined"
360 assert ONOS6Port, "ONOS6Port not defined"
361 assert ONOS7Port, "ONOS7Port not defined"
362
363 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700364 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700365 "and check that an ONOS node becomes the " +\
366 "master of the device."
367 main.step( "Assign switches to controllers" )
368
369 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700370 for i in range( main.numCtrls ):
371 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 swList = []
373 for i in range( 1, 29 ):
374 swList.append( "s" + str( i ) )
375 main.Mininet1.assignSwController( sw=swList, ip=ipList )
376
377 mastershipCheck = main.TRUE
378 for i in range( 1, 29 ):
379 response = main.Mininet1.getSwController( "s" + str( i ) )
380 try:
381 main.log.info( str( response ) )
382 except Exception:
383 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700384 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700385 if re.search( "tcp:" + node.ip_address, response ):
386 mastershipCheck = mastershipCheck and main.TRUE
387 else:
388 main.log.error( "Error, node " + node.ip_address + " is " +
389 "not in the list of controllers s" +
390 str( i ) + " is connecting to." )
391 mastershipCheck = main.FALSE
392 utilities.assert_equals(
393 expect=main.TRUE,
394 actual=mastershipCheck,
395 onpass="Switch mastership assigned correctly",
396 onfail="Switches not assigned correctly to controllers" )
397
398 def CASE21( self, main ):
399 """
400 Assign mastership to controllers
401 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700402 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700403 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700404 assert main, "main not defined"
405 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700406 assert main.CLIs, "main.CLIs not defined"
407 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700408 assert ONOS1Port, "ONOS1Port not defined"
409 assert ONOS2Port, "ONOS2Port not defined"
410 assert ONOS3Port, "ONOS3Port not defined"
411 assert ONOS4Port, "ONOS4Port not defined"
412 assert ONOS5Port, "ONOS5Port not defined"
413 assert ONOS6Port, "ONOS6Port not defined"
414 assert ONOS7Port, "ONOS7Port not defined"
415
416 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700417 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700418 "device. Then manually assign" +\
419 " mastership to specific ONOS nodes using" +\
420 " 'device-role'"
421 main.step( "Assign mastership of switches to specific controllers" )
422 # Manually assign mastership to the controller we want
423 roleCall = main.TRUE
424
425 ipList = [ ]
426 deviceList = []
Jon Halla440e872016-03-31 15:15:50 -0700427 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700428 try:
429 # Assign mastership to specific controllers. This assignment was
430 # determined for a 7 node cluser, but will work with any sized
431 # cluster
432 for i in range( 1, 29 ): # switches 1 through 28
433 # set up correct variables:
434 if i == 1:
435 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700436 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700437 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700438 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700439 c = 1 % main.numCtrls
440 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700441 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700442 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700443 c = 1 % main.numCtrls
444 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700445 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700446 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700447 c = 3 % main.numCtrls
448 ip = main.nodes[ c ].ip_address # ONOS4
Jon Halla440e872016-03-31 15:15:50 -0700449 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700450 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700451 c = 2 % main.numCtrls
452 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700453 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700454 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700455 c = 2 % main.numCtrls
456 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700457 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700458 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700459 c = 5 % main.numCtrls
460 ip = main.nodes[ c ].ip_address # ONOS6
Jon Halla440e872016-03-31 15:15:50 -0700461 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700462 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700463 c = 4 % main.numCtrls
464 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700465 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700466 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700467 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700468 c = 6 % main.numCtrls
469 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700470 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700471 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700472 elif i == 28:
473 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700474 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700475 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700476 else:
477 main.log.error( "You didn't write an else statement for " +
478 "switch s" + str( i ) )
479 roleCall = main.FALSE
480 # Assign switch
481 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
482 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700483 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700484 ipList.append( ip )
485 deviceList.append( deviceId )
486 except ( AttributeError, AssertionError ):
487 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700488 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700489 utilities.assert_equals(
490 expect=main.TRUE,
491 actual=roleCall,
492 onpass="Re-assigned switch mastership to designated controller",
493 onfail="Something wrong with deviceRole calls" )
494
495 main.step( "Check mastership was correctly assigned" )
496 roleCheck = main.TRUE
497 # NOTE: This is due to the fact that device mastership change is not
498 # atomic and is actually a multi step process
499 time.sleep( 5 )
500 for i in range( len( ipList ) ):
501 ip = ipList[i]
502 deviceId = deviceList[i]
503 # Check assignment
Jon Halla440e872016-03-31 15:15:50 -0700504 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700505 if ip in master:
506 roleCheck = roleCheck and main.TRUE
507 else:
508 roleCheck = roleCheck and main.FALSE
509 main.log.error( "Error, controller " + ip + " is not" +
510 " master " + "of device " +
511 str( deviceId ) + ". Master is " +
512 repr( master ) + "." )
513 utilities.assert_equals(
514 expect=main.TRUE,
515 actual=roleCheck,
516 onpass="Switches were successfully reassigned to designated " +
517 "controller",
518 onfail="Switches were not successfully reassigned" )
519
520 def CASE3( self, main ):
521 """
522 Assign intents
523 """
524 import time
525 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700526 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700527 assert main, "main not defined"
528 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700529 assert main.CLIs, "main.CLIs not defined"
530 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700531 try:
532 labels
533 except NameError:
534 main.log.error( "labels not defined, setting to []" )
535 labels = []
536 try:
537 data
538 except NameError:
539 main.log.error( "data not defined, setting to []" )
540 data = []
541 # NOTE: we must reinstall intents until we have a persistant intent
542 # datastore!
543 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700544 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700545 "assign predetermined host-to-host intents." +\
546 " After installation, check that the intent" +\
547 " is distributed to all nodes and the state" +\
548 " is INSTALLED"
549
550 # install onos-app-fwd
551 main.step( "Install reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700552 onosCli = main.CLIs[ main.activeNodes[0] ]
553 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700554 utilities.assert_equals( expect=main.TRUE, actual=installResults,
555 onpass="Install fwd successful",
556 onfail="Install fwd failed" )
557
558 main.step( "Check app ids" )
559 appCheck = main.TRUE
560 threads = []
Jon Halla440e872016-03-31 15:15:50 -0700561 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700562 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700563 name="appToIDCheck-" + str( i ),
564 args=[] )
565 threads.append( t )
566 t.start()
567
568 for t in threads:
569 t.join()
570 appCheck = appCheck and t.result
571 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700572 main.log.warn( onosCli.apps() )
573 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700574 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
575 onpass="App Ids seem to be correct",
576 onfail="Something is wrong with app Ids" )
577
578 main.step( "Discovering Hosts( Via pingall for now )" )
579 # FIXME: Once we have a host discovery mechanism, use that instead
580 # REACTIVE FWD test
581 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700582 passMsg = "Reactive Pingall test passed"
583 time1 = time.time()
584 pingResult = main.Mininet1.pingall()
585 time2 = time.time()
586 if not pingResult:
587 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700588 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700589 passMsg += " on the second try"
590 utilities.assert_equals(
591 expect=main.TRUE,
592 actual=pingResult,
593 onpass= passMsg,
594 onfail="Reactive Pingall failed, " +
595 "one or more ping pairs failed" )
596 main.log.info( "Time for pingall: %2f seconds" %
597 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700598 # timeout for fwd flows
599 time.sleep( 11 )
600 # uninstall onos-app-fwd
601 main.step( "Uninstall reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700602 node = main.activeNodes[0]
603 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700604 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
605 onpass="Uninstall fwd successful",
606 onfail="Uninstall fwd failed" )
607
608 main.step( "Check app ids" )
609 threads = []
610 appCheck2 = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700611 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700612 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700613 name="appToIDCheck-" + str( i ),
614 args=[] )
615 threads.append( t )
616 t.start()
617
618 for t in threads:
619 t.join()
620 appCheck2 = appCheck2 and t.result
621 if appCheck2 != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700622 node = main.activeNodes[0]
623 main.log.warn( main.CLIs[node].apps() )
624 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700625 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
626 onpass="App Ids seem to be correct",
627 onfail="Something is wrong with app Ids" )
628
629 main.step( "Add host intents via cli" )
630 intentIds = []
Jon Hall6e709752016-02-01 13:38:46 -0800631 # TODO: move the host numbers to params
632 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700633 intentAddResult = True
634 hostResult = main.TRUE
635 for i in range( 8, 18 ):
636 main.log.info( "Adding host intent between h" + str( i ) +
637 " and h" + str( i + 10 ) )
638 host1 = "00:00:00:00:00:" + \
639 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
640 host2 = "00:00:00:00:00:" + \
641 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
642 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700643 host1Dict = onosCli.getHost( host1 )
644 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700645 host1Id = None
646 host2Id = None
647 if host1Dict and host2Dict:
648 host1Id = host1Dict.get( 'id', None )
649 host2Id = host2Dict.get( 'id', None )
650 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700651 nodeNum = ( i % len( main.activeNodes ) )
652 node = main.activeNodes[nodeNum]
653 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700654 if tmpId:
655 main.log.info( "Added intent with id: " + tmpId )
656 intentIds.append( tmpId )
657 else:
658 main.log.error( "addHostIntent returned: " +
659 repr( tmpId ) )
660 else:
661 main.log.error( "Error, getHost() failed for h" + str( i ) +
662 " and/or h" + str( i + 10 ) )
Jon Halla440e872016-03-31 15:15:50 -0700663 node = main.activeNodes[0]
664 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700665 main.log.warn( "Hosts output: " )
666 try:
667 main.log.warn( json.dumps( json.loads( hosts ),
668 sort_keys=True,
669 indent=4,
670 separators=( ',', ': ' ) ) )
671 except ( ValueError, TypeError ):
672 main.log.warn( repr( hosts ) )
673 hostResult = main.FALSE
674 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
675 onpass="Found a host id for each host",
676 onfail="Error looking up host ids" )
677
678 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700679 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700680 main.log.info( "Submitted intents: " + str( intentIds ) )
681 main.log.info( "Intents in ONOS: " + str( onosIds ) )
682 for intent in intentIds:
683 if intent in onosIds:
684 pass # intent submitted is in onos
685 else:
686 intentAddResult = False
687 if intentAddResult:
688 intentStop = time.time()
689 else:
690 intentStop = None
691 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700692 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700693 intentStates = []
694 installedCheck = True
695 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
696 count = 0
697 try:
698 for intent in json.loads( intents ):
699 state = intent.get( 'state', None )
700 if "INSTALLED" not in state:
701 installedCheck = False
702 intentId = intent.get( 'id', None )
703 intentStates.append( ( intentId, state ) )
704 except ( ValueError, TypeError ):
705 main.log.exception( "Error parsing intents" )
706 # add submitted intents not in the store
707 tmplist = [ i for i, s in intentStates ]
708 missingIntents = False
709 for i in intentIds:
710 if i not in tmplist:
711 intentStates.append( ( i, " - " ) )
712 missingIntents = True
713 intentStates.sort()
714 for i, s in intentStates:
715 count += 1
716 main.log.info( "%-6s%-15s%-15s" %
717 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700718 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700719 try:
720 missing = False
721 if leaders:
722 parsedLeaders = json.loads( leaders )
723 main.log.warn( json.dumps( parsedLeaders,
724 sort_keys=True,
725 indent=4,
726 separators=( ',', ': ' ) ) )
727 # check for all intent partitions
728 topics = []
729 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700730 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700731 main.log.debug( topics )
732 ONOStopics = [ j['topic'] for j in parsedLeaders ]
733 for topic in topics:
734 if topic not in ONOStopics:
735 main.log.error( "Error: " + topic +
736 " not in leaders" )
737 missing = True
738 else:
739 main.log.error( "leaders() returned None" )
740 except ( ValueError, TypeError ):
741 main.log.exception( "Error parsing leaders" )
742 main.log.error( repr( leaders ) )
743 # Check all nodes
744 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700745 for i in main.activeNodes:
746 response = main.CLIs[i].leaders( jsonFormat=False)
747 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700748 str( response ) )
749
Jon Halla440e872016-03-31 15:15:50 -0700750 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700751 try:
752 if partitions :
753 parsedPartitions = json.loads( partitions )
754 main.log.warn( json.dumps( parsedPartitions,
755 sort_keys=True,
756 indent=4,
757 separators=( ',', ': ' ) ) )
758 # TODO check for a leader in all paritions
759 # TODO check for consistency among nodes
760 else:
761 main.log.error( "partitions() returned None" )
762 except ( ValueError, TypeError ):
763 main.log.exception( "Error parsing partitions" )
764 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700765 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700766 try:
767 if pendingMap :
768 parsedPending = json.loads( pendingMap )
769 main.log.warn( json.dumps( parsedPending,
770 sort_keys=True,
771 indent=4,
772 separators=( ',', ': ' ) ) )
773 # TODO check something here?
774 else:
775 main.log.error( "pendingMap() returned None" )
776 except ( ValueError, TypeError ):
777 main.log.exception( "Error parsing pending map" )
778 main.log.error( repr( pendingMap ) )
779
780 intentAddResult = bool( intentAddResult and not missingIntents and
781 installedCheck )
782 if not intentAddResult:
783 main.log.error( "Error in pushing host intents to ONOS" )
784
785 main.step( "Intent Anti-Entropy dispersion" )
Jon Halla440e872016-03-31 15:15:50 -0700786 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700787 correct = True
788 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700789 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700790 onosIds = []
Jon Halla440e872016-03-31 15:15:50 -0700791 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700792 onosIds.append( ids )
Jon Halla440e872016-03-31 15:15:50 -0700793 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700794 str( sorted( onosIds ) ) )
795 if sorted( ids ) != sorted( intentIds ):
796 main.log.warn( "Set of intent IDs doesn't match" )
797 correct = False
798 break
799 else:
Jon Halla440e872016-03-31 15:15:50 -0700800 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700801 for intent in intents:
802 if intent[ 'state' ] != "INSTALLED":
803 main.log.warn( "Intent " + intent[ 'id' ] +
804 " is " + intent[ 'state' ] )
805 correct = False
806 break
807 if correct:
808 break
809 else:
810 time.sleep(1)
811 if not intentStop:
812 intentStop = time.time()
813 global gossipTime
814 gossipTime = intentStop - intentStart
815 main.log.info( "It took about " + str( gossipTime ) +
816 " seconds for all intents to appear in each node" )
817 append = False
818 title = "Gossip Intents"
819 count = 1
820 while append is False:
821 curTitle = title + str( count )
822 if curTitle not in labels:
823 labels.append( curTitle )
824 data.append( str( gossipTime ) )
825 append = True
826 else:
827 count += 1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700828 gossipPeriod = int( main.params['timers']['gossip'] )
Jon Halla440e872016-03-31 15:15:50 -0700829 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700830 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700831 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700832 onpass="ECM anti-entropy for intents worked within " +
833 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700834 onfail="Intent ECM anti-entropy took too long. " +
835 "Expected time:{}, Actual time:{}".format( maxGossipTime,
836 gossipTime ) )
837 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700838 intentAddResult = True
839
840 if not intentAddResult or "key" in pendingMap:
841 import time
842 installedCheck = True
843 main.log.info( "Sleeping 60 seconds to see if intents are found" )
844 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700845 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700846 main.log.info( "Submitted intents: " + str( intentIds ) )
847 main.log.info( "Intents in ONOS: " + str( onosIds ) )
848 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700849 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700850 intentStates = []
851 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
852 count = 0
853 try:
854 for intent in json.loads( intents ):
855 # Iter through intents of a node
856 state = intent.get( 'state', None )
857 if "INSTALLED" not in state:
858 installedCheck = False
859 intentId = intent.get( 'id', None )
860 intentStates.append( ( intentId, state ) )
861 except ( ValueError, TypeError ):
862 main.log.exception( "Error parsing intents" )
863 # add submitted intents not in the store
864 tmplist = [ i for i, s in intentStates ]
865 for i in intentIds:
866 if i not in tmplist:
867 intentStates.append( ( i, " - " ) )
868 intentStates.sort()
869 for i, s in intentStates:
870 count += 1
871 main.log.info( "%-6s%-15s%-15s" %
872 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700873 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700874 try:
875 missing = False
876 if leaders:
877 parsedLeaders = json.loads( leaders )
878 main.log.warn( json.dumps( parsedLeaders,
879 sort_keys=True,
880 indent=4,
881 separators=( ',', ': ' ) ) )
882 # check for all intent partitions
883 # check for election
884 topics = []
885 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700886 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700887 # FIXME: this should only be after we start the app
888 topics.append( "org.onosproject.election" )
889 main.log.debug( topics )
890 ONOStopics = [ j['topic'] for j in parsedLeaders ]
891 for topic in topics:
892 if topic not in ONOStopics:
893 main.log.error( "Error: " + topic +
894 " not in leaders" )
895 missing = True
896 else:
897 main.log.error( "leaders() returned None" )
898 except ( ValueError, TypeError ):
899 main.log.exception( "Error parsing leaders" )
900 main.log.error( repr( leaders ) )
901 # Check all nodes
902 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700903 for i in main.activeNodes:
904 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700905 response = node.leaders( jsonFormat=False)
906 main.log.warn( str( node.name ) + " leaders output: \n" +
907 str( response ) )
908
Jon Halla440e872016-03-31 15:15:50 -0700909 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700910 try:
911 if partitions :
912 parsedPartitions = json.loads( partitions )
913 main.log.warn( json.dumps( parsedPartitions,
914 sort_keys=True,
915 indent=4,
916 separators=( ',', ': ' ) ) )
917 # TODO check for a leader in all paritions
918 # TODO check for consistency among nodes
919 else:
920 main.log.error( "partitions() returned None" )
921 except ( ValueError, TypeError ):
922 main.log.exception( "Error parsing partitions" )
923 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700924 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700925 try:
926 if pendingMap :
927 parsedPending = json.loads( pendingMap )
928 main.log.warn( json.dumps( parsedPending,
929 sort_keys=True,
930 indent=4,
931 separators=( ',', ': ' ) ) )
932 # TODO check something here?
933 else:
934 main.log.error( "pendingMap() returned None" )
935 except ( ValueError, TypeError ):
936 main.log.exception( "Error parsing pending map" )
937 main.log.error( repr( pendingMap ) )
938
939 def CASE4( self, main ):
940 """
941 Ping across added host intents
942 """
943 import json
944 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700945 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700946 assert main, "main not defined"
947 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700948 assert main.CLIs, "main.CLIs not defined"
949 assert main.nodes, "main.nodes not defined"
Jon Hall6e709752016-02-01 13:38:46 -0800950 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700951 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700952 "functionality and check the state of " +\
953 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700954
Jon Hall41d39f12016-04-11 22:54:35 -0700955 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700956 main.step( "Check Intent state" )
957 installedCheck = False
958 loopCount = 0
959 while not installedCheck and loopCount < 40:
960 installedCheck = True
961 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700962 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700963 intentStates = []
964 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700965 count = 0
Jon Hall5cf14d52015-07-16 12:15:19 -0700966 # Iter through intents of a node
967 try:
968 for intent in json.loads( intents ):
969 state = intent.get( 'state', None )
970 if "INSTALLED" not in state:
971 installedCheck = False
972 intentId = intent.get( 'id', None )
973 intentStates.append( ( intentId, state ) )
974 except ( ValueError, TypeError ):
975 main.log.exception( "Error parsing intents." )
976 # Print states
977 intentStates.sort()
978 for i, s in intentStates:
979 count += 1
980 main.log.info( "%-6s%-15s%-15s" %
981 ( str( count ), str( i ), str( s ) ) )
982 if not installedCheck:
983 time.sleep( 1 )
984 loopCount += 1
985 utilities.assert_equals( expect=True, actual=installedCheck,
986 onpass="Intents are all INSTALLED",
987 onfail="Intents are not all in " +
988 "INSTALLED state" )
989
Jon Hall9d2dcad2016-04-08 10:15:20 -0700990 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700991 PingResult = main.TRUE
992 for i in range( 8, 18 ):
993 ping = main.Mininet1.pingHost( src="h" + str( i ),
994 target="h" + str( i + 10 ) )
995 PingResult = PingResult and ping
996 if ping == main.FALSE:
997 main.log.warn( "Ping failed between h" + str( i ) +
998 " and h" + str( i + 10 ) )
999 elif ping == main.TRUE:
1000 main.log.info( "Ping test passed!" )
1001 # Don't set PingResult or you'd override failures
1002 if PingResult == main.FALSE:
1003 main.log.error(
1004 "Intents have not been installed correctly, pings failed." )
1005 # TODO: pretty print
1006 main.log.warn( "ONOS1 intents: " )
1007 try:
1008 tmpIntents = onosCli.intents()
1009 main.log.warn( json.dumps( json.loads( tmpIntents ),
1010 sort_keys=True,
1011 indent=4,
1012 separators=( ',', ': ' ) ) )
1013 except ( ValueError, TypeError ):
1014 main.log.warn( repr( tmpIntents ) )
1015 utilities.assert_equals(
1016 expect=main.TRUE,
1017 actual=PingResult,
1018 onpass="Intents have been installed correctly and pings work",
1019 onfail="Intents have not been installed correctly, pings failed." )
1020
Jon Hall5cf14d52015-07-16 12:15:19 -07001021 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -07001022 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001023 topicCheck = main.TRUE
1024 try:
1025 if leaders:
1026 parsedLeaders = json.loads( leaders )
1027 main.log.warn( json.dumps( parsedLeaders,
1028 sort_keys=True,
1029 indent=4,
1030 separators=( ',', ': ' ) ) )
1031 # check for all intent partitions
1032 # check for election
1033 # TODO: Look at Devices as topics now that it uses this system
1034 topics = []
1035 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001036 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001037 # FIXME: this should only be after we start the app
1038 # FIXME: topics.append( "org.onosproject.election" )
1039 # Print leaders output
1040 main.log.debug( topics )
1041 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1042 for topic in topics:
1043 if topic not in ONOStopics:
1044 main.log.error( "Error: " + topic +
1045 " not in leaders" )
1046 topicCheck = main.FALSE
1047 else:
1048 main.log.error( "leaders() returned None" )
1049 topicCheck = main.FALSE
1050 except ( ValueError, TypeError ):
1051 topicCheck = main.FALSE
1052 main.log.exception( "Error parsing leaders" )
1053 main.log.error( repr( leaders ) )
1054 # TODO: Check for a leader of these topics
1055 # Check all nodes
1056 if topicCheck:
Jon Halla440e872016-03-31 15:15:50 -07001057 for i in main.activeNodes:
1058 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001059 response = node.leaders( jsonFormat=False)
1060 main.log.warn( str( node.name ) + " leaders output: \n" +
1061 str( response ) )
1062
1063 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1064 onpass="intent Partitions is in leaders",
1065 onfail="Some topics were lost " )
1066 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -07001067 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001068 try:
1069 if partitions :
1070 parsedPartitions = json.loads( partitions )
1071 main.log.warn( json.dumps( parsedPartitions,
1072 sort_keys=True,
1073 indent=4,
1074 separators=( ',', ': ' ) ) )
1075 # TODO check for a leader in all paritions
1076 # TODO check for consistency among nodes
1077 else:
1078 main.log.error( "partitions() returned None" )
1079 except ( ValueError, TypeError ):
1080 main.log.exception( "Error parsing partitions" )
1081 main.log.error( repr( partitions ) )
1082 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001083 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001084 try:
1085 if pendingMap :
1086 parsedPending = json.loads( pendingMap )
1087 main.log.warn( json.dumps( parsedPending,
1088 sort_keys=True,
1089 indent=4,
1090 separators=( ',', ': ' ) ) )
1091 # TODO check something here?
1092 else:
1093 main.log.error( "pendingMap() returned None" )
1094 except ( ValueError, TypeError ):
1095 main.log.exception( "Error parsing pending map" )
1096 main.log.error( repr( pendingMap ) )
1097
1098 if not installedCheck:
1099 main.log.info( "Waiting 60 seconds to see if the state of " +
1100 "intents change" )
1101 time.sleep( 60 )
1102 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001103 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001104 intentStates = []
1105 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1106 count = 0
1107 # Iter through intents of a node
1108 try:
1109 for intent in json.loads( intents ):
1110 state = intent.get( 'state', None )
1111 if "INSTALLED" not in state:
1112 installedCheck = False
1113 intentId = intent.get( 'id', None )
1114 intentStates.append( ( intentId, state ) )
1115 except ( ValueError, TypeError ):
1116 main.log.exception( "Error parsing intents." )
1117 intentStates.sort()
1118 for i, s in intentStates:
1119 count += 1
1120 main.log.info( "%-6s%-15s%-15s" %
1121 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001122 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001123 try:
1124 missing = False
1125 if leaders:
1126 parsedLeaders = json.loads( leaders )
1127 main.log.warn( json.dumps( parsedLeaders,
1128 sort_keys=True,
1129 indent=4,
1130 separators=( ',', ': ' ) ) )
1131 # check for all intent partitions
1132 # check for election
1133 topics = []
1134 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001135 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001136 # FIXME: this should only be after we start the app
1137 topics.append( "org.onosproject.election" )
1138 main.log.debug( topics )
1139 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1140 for topic in topics:
1141 if topic not in ONOStopics:
1142 main.log.error( "Error: " + topic +
1143 " not in leaders" )
1144 missing = True
1145 else:
1146 main.log.error( "leaders() returned None" )
1147 except ( ValueError, TypeError ):
1148 main.log.exception( "Error parsing leaders" )
1149 main.log.error( repr( leaders ) )
1150 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001151 for i in main.activeNodes:
1152 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001153 response = node.leaders( jsonFormat=False)
1154 main.log.warn( str( node.name ) + " leaders output: \n" +
1155 str( response ) )
1156
Jon Halla440e872016-03-31 15:15:50 -07001157 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001158 try:
1159 if partitions :
1160 parsedPartitions = json.loads( partitions )
1161 main.log.warn( json.dumps( parsedPartitions,
1162 sort_keys=True,
1163 indent=4,
1164 separators=( ',', ': ' ) ) )
1165 # TODO check for a leader in all paritions
1166 # TODO check for consistency among nodes
1167 else:
1168 main.log.error( "partitions() returned None" )
1169 except ( ValueError, TypeError ):
1170 main.log.exception( "Error parsing partitions" )
1171 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001172 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001173 try:
1174 if pendingMap :
1175 parsedPending = json.loads( pendingMap )
1176 main.log.warn( json.dumps( parsedPending,
1177 sort_keys=True,
1178 indent=4,
1179 separators=( ',', ': ' ) ) )
1180 # TODO check something here?
1181 else:
1182 main.log.error( "pendingMap() returned None" )
1183 except ( ValueError, TypeError ):
1184 main.log.exception( "Error parsing pending map" )
1185 main.log.error( repr( pendingMap ) )
1186 # Print flowrules
Jon Halla440e872016-03-31 15:15:50 -07001187 node = main.activeNodes[0]
1188 main.log.debug( main.CLIs[node].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001189 main.step( "Wait a minute then ping again" )
1190 # the wait is above
1191 PingResult = main.TRUE
1192 for i in range( 8, 18 ):
1193 ping = main.Mininet1.pingHost( src="h" + str( i ),
1194 target="h" + str( i + 10 ) )
1195 PingResult = PingResult and ping
1196 if ping == main.FALSE:
1197 main.log.warn( "Ping failed between h" + str( i ) +
1198 " and h" + str( i + 10 ) )
1199 elif ping == main.TRUE:
1200 main.log.info( "Ping test passed!" )
1201 # Don't set PingResult or you'd override failures
1202 if PingResult == main.FALSE:
1203 main.log.error(
1204 "Intents have not been installed correctly, pings failed." )
1205 # TODO: pretty print
1206 main.log.warn( "ONOS1 intents: " )
1207 try:
Jon Halla440e872016-03-31 15:15:50 -07001208 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001209 main.log.warn( json.dumps( json.loads( tmpIntents ),
1210 sort_keys=True,
1211 indent=4,
1212 separators=( ',', ': ' ) ) )
1213 except ( ValueError, TypeError ):
1214 main.log.warn( repr( tmpIntents ) )
1215 utilities.assert_equals(
1216 expect=main.TRUE,
1217 actual=PingResult,
1218 onpass="Intents have been installed correctly and pings work",
1219 onfail="Intents have not been installed correctly, pings failed." )
1220
1221 def CASE5( self, main ):
1222 """
1223 Reading state of ONOS
1224 """
1225 import json
1226 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001227 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001228 assert main, "main not defined"
1229 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001230 assert main.CLIs, "main.CLIs not defined"
1231 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001232
1233 main.case( "Setting up and gathering data for current state" )
1234 # The general idea for this test case is to pull the state of
1235 # ( intents,flows, topology,... ) from each ONOS node
1236 # We can then compare them with each other and also with past states
1237
1238 main.step( "Check that each switch has a master" )
1239 global mastershipState
1240 mastershipState = '[]'
1241
1242 # Assert that each device has a master
1243 rolesNotNull = main.TRUE
1244 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001245 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001246 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001247 name="rolesNotNull-" + str( i ),
1248 args=[] )
1249 threads.append( t )
1250 t.start()
1251
1252 for t in threads:
1253 t.join()
1254 rolesNotNull = rolesNotNull and t.result
1255 utilities.assert_equals(
1256 expect=main.TRUE,
1257 actual=rolesNotNull,
1258 onpass="Each device has a master",
1259 onfail="Some devices don't have a master assigned" )
1260
1261 main.step( "Get the Mastership of each switch from each controller" )
1262 ONOSMastership = []
1263 mastershipCheck = main.FALSE
1264 consistentMastership = True
1265 rolesResults = True
1266 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001267 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001268 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001269 name="roles-" + str( i ),
1270 args=[] )
1271 threads.append( t )
1272 t.start()
1273
1274 for t in threads:
1275 t.join()
1276 ONOSMastership.append( t.result )
1277
Jon Halla440e872016-03-31 15:15:50 -07001278 for i in range( len( ONOSMastership ) ):
1279 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001280 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001281 main.log.error( "Error in getting ONOS" + node + " roles" )
1282 main.log.warn( "ONOS" + node + " mastership response: " +
1283 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001284 rolesResults = False
1285 utilities.assert_equals(
1286 expect=True,
1287 actual=rolesResults,
1288 onpass="No error in reading roles output",
1289 onfail="Error in reading roles from ONOS" )
1290
1291 main.step( "Check for consistency in roles from each controller" )
1292 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1293 main.log.info(
1294 "Switch roles are consistent across all ONOS nodes" )
1295 else:
1296 consistentMastership = False
1297 utilities.assert_equals(
1298 expect=True,
1299 actual=consistentMastership,
1300 onpass="Switch roles are consistent across all ONOS nodes",
1301 onfail="ONOS nodes have different views of switch roles" )
1302
1303 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001304 for i in range( len( main.activeNodes ) ):
1305 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001306 try:
1307 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001308 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001309 json.dumps(
1310 json.loads( ONOSMastership[ i ] ),
1311 sort_keys=True,
1312 indent=4,
1313 separators=( ',', ': ' ) ) )
1314 except ( ValueError, TypeError ):
1315 main.log.warn( repr( ONOSMastership[ i ] ) )
1316 elif rolesResults and consistentMastership:
1317 mastershipCheck = main.TRUE
1318 mastershipState = ONOSMastership[ 0 ]
1319
1320 main.step( "Get the intents from each controller" )
1321 global intentState
1322 intentState = []
1323 ONOSIntents = []
1324 intentCheck = main.FALSE
1325 consistentIntents = True
1326 intentsResults = True
1327 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001328 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001329 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001330 name="intents-" + str( i ),
1331 args=[],
1332 kwargs={ 'jsonFormat': True } )
1333 threads.append( t )
1334 t.start()
1335
1336 for t in threads:
1337 t.join()
1338 ONOSIntents.append( t.result )
1339
Jon Halla440e872016-03-31 15:15:50 -07001340 for i in range( len( ONOSIntents ) ):
1341 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001342 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hall6e709752016-02-01 13:38:46 -08001343 main.log.error( "Error in getting ONOS" + node + " intents" )
1344 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001345 repr( ONOSIntents[ i ] ) )
1346 intentsResults = False
1347 utilities.assert_equals(
1348 expect=True,
1349 actual=intentsResults,
1350 onpass="No error in reading intents output",
1351 onfail="Error in reading intents from ONOS" )
1352
1353 main.step( "Check for consistency in Intents from each controller" )
1354 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1355 main.log.info( "Intents are consistent across all ONOS " +
1356 "nodes" )
1357 else:
1358 consistentIntents = False
1359 main.log.error( "Intents not consistent" )
1360 utilities.assert_equals(
1361 expect=True,
1362 actual=consistentIntents,
1363 onpass="Intents are consistent across all ONOS nodes",
1364 onfail="ONOS nodes have different views of intents" )
1365
1366 if intentsResults:
1367 # Try to make it easy to figure out what is happening
1368 #
1369 # Intent ONOS1 ONOS2 ...
1370 # 0x01 INSTALLED INSTALLING
1371 # ... ... ...
1372 # ... ... ...
1373 title = " Id"
Jon Halla440e872016-03-31 15:15:50 -07001374 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001375 title += " " * 10 + "ONOS" + str( n + 1 )
1376 main.log.warn( title )
1377 # get all intent keys in the cluster
1378 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001379 try:
1380 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001381 for nodeStr in ONOSIntents:
1382 node = json.loads( nodeStr )
1383 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001384 keys.append( intent.get( 'id' ) )
1385 keys = set( keys )
1386 # For each intent key, print the state on each node
1387 for key in keys:
1388 row = "%-13s" % key
1389 for nodeStr in ONOSIntents:
1390 node = json.loads( nodeStr )
1391 for intent in node:
1392 if intent.get( 'id', "Error" ) == key:
1393 row += "%-15s" % intent.get( 'state' )
1394 main.log.warn( row )
1395 # End of intent state table
1396 except ValueError as e:
1397 main.log.exception( e )
1398 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001399
1400 if intentsResults and not consistentIntents:
1401 # print the json objects
Jon Halla440e872016-03-31 15:15:50 -07001402 n = str( main.activeNodes[-1] + 1 )
1403 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001404 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1405 sort_keys=True,
1406 indent=4,
1407 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001408 for i in range( len( ONOSIntents ) ):
1409 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001410 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07001411 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001412 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1413 sort_keys=True,
1414 indent=4,
1415 separators=( ',', ': ' ) ) )
1416 else:
Jon Halla440e872016-03-31 15:15:50 -07001417 main.log.debug( "ONOS" + node + " intents match ONOS" +
1418 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001419 elif intentsResults and consistentIntents:
1420 intentCheck = main.TRUE
1421 intentState = ONOSIntents[ 0 ]
1422
1423 main.step( "Get the flows from each controller" )
1424 global flowState
1425 flowState = []
1426 ONOSFlows = []
1427 ONOSFlowsJson = []
1428 flowCheck = main.FALSE
1429 consistentFlows = True
1430 flowsResults = True
1431 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001432 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001433 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001434 name="flows-" + str( i ),
1435 args=[],
1436 kwargs={ 'jsonFormat': True } )
1437 threads.append( t )
1438 t.start()
1439
1440 # NOTE: Flows command can take some time to run
1441 time.sleep(30)
1442 for t in threads:
1443 t.join()
1444 result = t.result
1445 ONOSFlows.append( result )
1446
Jon Halla440e872016-03-31 15:15:50 -07001447 for i in range( len( ONOSFlows ) ):
1448 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001449 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1450 main.log.error( "Error in getting ONOS" + num + " flows" )
1451 main.log.warn( "ONOS" + num + " flows response: " +
1452 repr( ONOSFlows[ i ] ) )
1453 flowsResults = False
1454 ONOSFlowsJson.append( None )
1455 else:
1456 try:
1457 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1458 except ( ValueError, TypeError ):
1459 # FIXME: change this to log.error?
1460 main.log.exception( "Error in parsing ONOS" + num +
1461 " response as json." )
1462 main.log.error( repr( ONOSFlows[ i ] ) )
1463 ONOSFlowsJson.append( None )
1464 flowsResults = False
1465 utilities.assert_equals(
1466 expect=True,
1467 actual=flowsResults,
1468 onpass="No error in reading flows output",
1469 onfail="Error in reading flows from ONOS" )
1470
1471 main.step( "Check for consistency in Flows from each controller" )
1472 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1473 if all( tmp ):
1474 main.log.info( "Flow count is consistent across all ONOS nodes" )
1475 else:
1476 consistentFlows = False
1477 utilities.assert_equals(
1478 expect=True,
1479 actual=consistentFlows,
1480 onpass="The flow count is consistent across all ONOS nodes",
1481 onfail="ONOS nodes have different flow counts" )
1482
1483 if flowsResults and not consistentFlows:
Jon Halla440e872016-03-31 15:15:50 -07001484 for i in range( len( ONOSFlows ) ):
1485 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001486 try:
1487 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001488 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001489 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1490 indent=4, separators=( ',', ': ' ) ) )
1491 except ( ValueError, TypeError ):
Jon Halla440e872016-03-31 15:15:50 -07001492 main.log.warn( "ONOS" + node + " flows: " +
1493 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001494 elif flowsResults and consistentFlows:
1495 flowCheck = main.TRUE
1496 flowState = ONOSFlows[ 0 ]
1497
1498 main.step( "Get the OF Table entries" )
1499 global flows
1500 flows = []
1501 for i in range( 1, 29 ):
Jon Halla440e872016-03-31 15:15:50 -07001502 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001503 if flowCheck == main.FALSE:
1504 for table in flows:
1505 main.log.warn( table )
1506 # TODO: Compare switch flow tables with ONOS flow tables
1507
1508 main.step( "Start continuous pings" )
1509 main.Mininet2.pingLong(
1510 src=main.params[ 'PING' ][ 'source1' ],
1511 target=main.params[ 'PING' ][ 'target1' ],
1512 pingTime=500 )
1513 main.Mininet2.pingLong(
1514 src=main.params[ 'PING' ][ 'source2' ],
1515 target=main.params[ 'PING' ][ 'target2' ],
1516 pingTime=500 )
1517 main.Mininet2.pingLong(
1518 src=main.params[ 'PING' ][ 'source3' ],
1519 target=main.params[ 'PING' ][ 'target3' ],
1520 pingTime=500 )
1521 main.Mininet2.pingLong(
1522 src=main.params[ 'PING' ][ 'source4' ],
1523 target=main.params[ 'PING' ][ 'target4' ],
1524 pingTime=500 )
1525 main.Mininet2.pingLong(
1526 src=main.params[ 'PING' ][ 'source5' ],
1527 target=main.params[ 'PING' ][ 'target5' ],
1528 pingTime=500 )
1529 main.Mininet2.pingLong(
1530 src=main.params[ 'PING' ][ 'source6' ],
1531 target=main.params[ 'PING' ][ 'target6' ],
1532 pingTime=500 )
1533 main.Mininet2.pingLong(
1534 src=main.params[ 'PING' ][ 'source7' ],
1535 target=main.params[ 'PING' ][ 'target7' ],
1536 pingTime=500 )
1537 main.Mininet2.pingLong(
1538 src=main.params[ 'PING' ][ 'source8' ],
1539 target=main.params[ 'PING' ][ 'target8' ],
1540 pingTime=500 )
1541 main.Mininet2.pingLong(
1542 src=main.params[ 'PING' ][ 'source9' ],
1543 target=main.params[ 'PING' ][ 'target9' ],
1544 pingTime=500 )
1545 main.Mininet2.pingLong(
1546 src=main.params[ 'PING' ][ 'source10' ],
1547 target=main.params[ 'PING' ][ 'target10' ],
1548 pingTime=500 )
1549
1550 main.step( "Collecting topology information from ONOS" )
1551 devices = []
1552 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001553 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001554 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001555 name="devices-" + str( i ),
1556 args=[ ] )
1557 threads.append( t )
1558 t.start()
1559
1560 for t in threads:
1561 t.join()
1562 devices.append( t.result )
1563 hosts = []
1564 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001565 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001566 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001567 name="hosts-" + str( i ),
1568 args=[ ] )
1569 threads.append( t )
1570 t.start()
1571
1572 for t in threads:
1573 t.join()
1574 try:
1575 hosts.append( json.loads( t.result ) )
1576 except ( ValueError, TypeError ):
1577 # FIXME: better handling of this, print which node
1578 # Maybe use thread name?
1579 main.log.exception( "Error parsing json output of hosts" )
Jon Hall3afe4c92015-12-14 19:30:38 -08001580 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001581 hosts.append( None )
1582
1583 ports = []
1584 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001585 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001586 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001587 name="ports-" + str( i ),
1588 args=[ ] )
1589 threads.append( t )
1590 t.start()
1591
1592 for t in threads:
1593 t.join()
1594 ports.append( t.result )
1595 links = []
1596 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001597 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001598 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001599 name="links-" + str( i ),
1600 args=[ ] )
1601 threads.append( t )
1602 t.start()
1603
1604 for t in threads:
1605 t.join()
1606 links.append( t.result )
1607 clusters = []
1608 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001609 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001610 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001611 name="clusters-" + str( i ),
1612 args=[ ] )
1613 threads.append( t )
1614 t.start()
1615
1616 for t in threads:
1617 t.join()
1618 clusters.append( t.result )
1619 # Compare json objects for hosts and dataplane clusters
1620
1621 # hosts
1622 main.step( "Host view is consistent across ONOS nodes" )
1623 consistentHostsResult = main.TRUE
1624 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001625 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001626 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001627 if hosts[ controller ] == hosts[ 0 ]:
1628 continue
1629 else: # hosts not consistent
1630 main.log.error( "hosts from ONOS" +
1631 controllerStr +
1632 " is inconsistent with ONOS1" )
1633 main.log.warn( repr( hosts[ controller ] ) )
1634 consistentHostsResult = main.FALSE
1635
1636 else:
1637 main.log.error( "Error in getting ONOS hosts from ONOS" +
1638 controllerStr )
1639 consistentHostsResult = main.FALSE
1640 main.log.warn( "ONOS" + controllerStr +
1641 " hosts response: " +
1642 repr( hosts[ controller ] ) )
1643 utilities.assert_equals(
1644 expect=main.TRUE,
1645 actual=consistentHostsResult,
1646 onpass="Hosts view is consistent across all ONOS nodes",
1647 onfail="ONOS nodes have different views of hosts" )
1648
1649 main.step( "Each host has an IP address" )
1650 ipResult = main.TRUE
1651 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001652 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001653 if hosts[ controller ]:
1654 for host in hosts[ controller ]:
1655 if not host.get( 'ipAddresses', [ ] ):
Jon Hallf3d16e72015-12-16 17:45:08 -08001656 main.log.error( "Error with host ips on controller" +
Jon Hall3afe4c92015-12-14 19:30:38 -08001657 controllerStr + ": " + str( host ) )
1658 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001659 utilities.assert_equals(
1660 expect=main.TRUE,
1661 actual=ipResult,
1662 onpass="The ips of the hosts aren't empty",
1663 onfail="The ip of at least one host is missing" )
1664
1665 # Strongly connected clusters of devices
1666 main.step( "Cluster view is consistent across ONOS nodes" )
1667 consistentClustersResult = main.TRUE
1668 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07001669 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001670 if "Error" not in clusters[ controller ]:
1671 if clusters[ controller ] == clusters[ 0 ]:
1672 continue
1673 else: # clusters not consistent
1674 main.log.error( "clusters from ONOS" + controllerStr +
1675 " is inconsistent with ONOS1" )
1676 consistentClustersResult = main.FALSE
1677
1678 else:
1679 main.log.error( "Error in getting dataplane clusters " +
1680 "from ONOS" + controllerStr )
1681 consistentClustersResult = main.FALSE
1682 main.log.warn( "ONOS" + controllerStr +
1683 " clusters response: " +
1684 repr( clusters[ controller ] ) )
1685 utilities.assert_equals(
1686 expect=main.TRUE,
1687 actual=consistentClustersResult,
1688 onpass="Clusters view is consistent across all ONOS nodes",
1689 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001690 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001691 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001692
Jon Hall5cf14d52015-07-16 12:15:19 -07001693 # there should always only be one cluster
1694 main.step( "Cluster view correct across ONOS nodes" )
1695 try:
1696 numClusters = len( json.loads( clusters[ 0 ] ) )
1697 except ( ValueError, TypeError ):
1698 main.log.exception( "Error parsing clusters[0]: " +
1699 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001700 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001701 clusterResults = main.FALSE
1702 if numClusters == 1:
1703 clusterResults = main.TRUE
1704 utilities.assert_equals(
1705 expect=1,
1706 actual=numClusters,
1707 onpass="ONOS shows 1 SCC",
1708 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1709
1710 main.step( "Comparing ONOS topology to MN" )
1711 devicesResults = main.TRUE
1712 linksResults = main.TRUE
1713 hostsResults = main.TRUE
1714 mnSwitches = main.Mininet1.getSwitches()
1715 mnLinks = main.Mininet1.getLinks()
1716 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001717 for controller in main.activeNodes:
1718 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001719 if devices[ controller ] and ports[ controller ] and\
1720 "Error" not in devices[ controller ] and\
1721 "Error" not in ports[ controller ]:
Jon Halle1a3b752015-07-22 13:02:46 -07001722 currentDevicesResult = main.Mininet1.compareSwitches(
1723 mnSwitches,
1724 json.loads( devices[ controller ] ),
1725 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001726 else:
1727 currentDevicesResult = main.FALSE
1728 utilities.assert_equals( expect=main.TRUE,
1729 actual=currentDevicesResult,
1730 onpass="ONOS" + controllerStr +
1731 " Switches view is correct",
1732 onfail="ONOS" + controllerStr +
1733 " Switches view is incorrect" )
1734 if links[ controller ] and "Error" not in links[ controller ]:
1735 currentLinksResult = main.Mininet1.compareLinks(
1736 mnSwitches, mnLinks,
1737 json.loads( links[ controller ] ) )
1738 else:
1739 currentLinksResult = main.FALSE
1740 utilities.assert_equals( expect=main.TRUE,
1741 actual=currentLinksResult,
1742 onpass="ONOS" + controllerStr +
1743 " links view is correct",
1744 onfail="ONOS" + controllerStr +
1745 " links view is incorrect" )
1746
Jon Hall657cdf62015-12-17 14:40:51 -08001747 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001748 currentHostsResult = main.Mininet1.compareHosts(
1749 mnHosts,
1750 hosts[ controller ] )
1751 else:
1752 currentHostsResult = main.FALSE
1753 utilities.assert_equals( expect=main.TRUE,
1754 actual=currentHostsResult,
1755 onpass="ONOS" + controllerStr +
1756 " hosts exist in Mininet",
1757 onfail="ONOS" + controllerStr +
1758 " hosts don't match Mininet" )
1759
1760 devicesResults = devicesResults and currentDevicesResult
1761 linksResults = linksResults and currentLinksResult
1762 hostsResults = hostsResults and currentHostsResult
1763
1764 main.step( "Device information is correct" )
1765 utilities.assert_equals(
1766 expect=main.TRUE,
1767 actual=devicesResults,
1768 onpass="Device information is correct",
1769 onfail="Device information is incorrect" )
1770
1771 main.step( "Links are correct" )
1772 utilities.assert_equals(
1773 expect=main.TRUE,
1774 actual=linksResults,
1775 onpass="Link are correct",
1776 onfail="Links are incorrect" )
1777
1778 main.step( "Hosts are correct" )
1779 utilities.assert_equals(
1780 expect=main.TRUE,
1781 actual=hostsResults,
1782 onpass="Hosts are correct",
1783 onfail="Hosts are incorrect" )
1784
1785 def CASE6( self, main ):
1786 """
1787 The Failure case.
1788 """
1789 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001790 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001791 assert main, "main not defined"
1792 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001793 assert main.CLIs, "main.CLIs not defined"
1794 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001795 try:
1796 labels
1797 except NameError:
1798 main.log.error( "labels not defined, setting to []" )
1799 global labels
1800 labels = []
1801 try:
1802 data
1803 except NameError:
1804 main.log.error( "data not defined, setting to []" )
1805 global data
1806 data = []
1807 # Reset non-persistent variables
1808 try:
1809 iCounterValue = 0
1810 except NameError:
1811 main.log.error( "iCounterValue not defined, setting to 0" )
1812 iCounterValue = 0
1813
1814 main.case( "Restart entire ONOS cluster" )
1815
Jon Hall5ec6b1b2015-09-17 18:20:14 -07001816 main.step( "Checking ONOS Logs for errors" )
1817 for node in main.nodes:
1818 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1819 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1820
Jon Hall5cf14d52015-07-16 12:15:19 -07001821 main.step( "Killing ONOS nodes" )
1822 killResults = main.TRUE
1823 killTime = time.time()
Jon Halle1a3b752015-07-22 13:02:46 -07001824 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001825 killed = main.ONOSbench.onosKill( node.ip_address )
1826 killResults = killResults and killed
1827 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1828 onpass="ONOS nodes killed",
1829 onfail="ONOS kill unsuccessful" )
1830
1831 main.step( "Checking if ONOS is up yet" )
1832 for i in range( 2 ):
1833 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001834 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001835 started = main.ONOSbench.isup( node.ip_address )
1836 if not started:
1837 main.log.error( node.name + " didn't start!" )
1838 onosIsupResult = onosIsupResult and started
1839 if onosIsupResult == main.TRUE:
1840 break
1841 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1842 onpass="ONOS restarted",
1843 onfail="ONOS restart NOT successful" )
1844
Jon Hall6509dbf2016-06-21 17:01:17 -07001845 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001846 cliResults = main.TRUE
1847 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001848 for i in range( main.numCtrls ):
1849 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -07001850 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -07001851 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001852 threads.append( t )
1853 t.start()
1854
1855 for t in threads:
1856 t.join()
1857 cliResults = cliResults and t.result
1858 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1859 onpass="ONOS cli started",
1860 onfail="ONOS clis did not restart" )
1861
Jon Hall6e709752016-02-01 13:38:46 -08001862 for i in range( 10 ):
1863 ready = True
Jon Hall7ac7bc32016-05-05 10:57:02 -07001864 for i in main.activeNodes:
1865 cli = main.CLIs[i]
Jon Hall6e709752016-02-01 13:38:46 -08001866 output = cli.summary()
1867 if not output:
1868 ready = False
Jon Halld2871c22016-07-26 11:01:14 -07001869 if ready:
1870 break
Jon Hall6e709752016-02-01 13:38:46 -08001871 time.sleep( 30 )
1872 utilities.assert_equals( expect=True, actual=ready,
1873 onpass="ONOS summary command succeded",
1874 onfail="ONOS summary command failed" )
1875 if not ready:
1876 main.cleanup()
1877 main.exit()
1878
Jon Hall5cf14d52015-07-16 12:15:19 -07001879 # Grab the time of restart so we chan check how long the gossip
1880 # protocol has had time to work
1881 main.restartTime = time.time() - killTime
1882 main.log.debug( "Restart time: " + str( main.restartTime ) )
1883 labels.append( "Restart" )
1884 data.append( str( main.restartTime ) )
1885
Jon Hall5cf14d52015-07-16 12:15:19 -07001886 # Rerun for election on restarted nodes
1887 runResults = main.TRUE
Jon Hall7ac7bc32016-05-05 10:57:02 -07001888 for i in main.activeNodes:
1889 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -07001890 run = cli.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001891 if run != main.TRUE:
1892 main.log.error( "Error running for election on " + cli.name )
1893 runResults = runResults and run
1894 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1895 onpass="Reran for election",
1896 onfail="Failed to rerun for election" )
1897
1898 # TODO: Make this configurable
1899 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -07001900 node = main.activeNodes[0]
1901 main.log.debug( main.CLIs[node].nodes( jsonFormat=False ) )
1902 main.log.debug( main.CLIs[node].leaders( jsonFormat=False ) )
1903 main.log.debug( main.CLIs[node].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001904
1905 def CASE7( self, main ):
1906 """
1907 Check state after ONOS failure
1908 """
1909 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001910 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001911 assert main, "main not defined"
1912 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001913 assert main.CLIs, "main.CLIs not defined"
1914 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001915 main.case( "Running ONOS Constant State Tests" )
1916
1917 main.step( "Check that each switch has a master" )
1918 # Assert that each device has a master
1919 rolesNotNull = main.TRUE
1920 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001921 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001922 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001923 name="rolesNotNull-" + str( i ),
1924 args=[ ] )
1925 threads.append( t )
1926 t.start()
1927
1928 for t in threads:
1929 t.join()
1930 rolesNotNull = rolesNotNull and t.result
1931 utilities.assert_equals(
1932 expect=main.TRUE,
1933 actual=rolesNotNull,
1934 onpass="Each device has a master",
1935 onfail="Some devices don't have a master assigned" )
1936
1937 main.step( "Read device roles from ONOS" )
1938 ONOSMastership = []
1939 mastershipCheck = main.FALSE
1940 consistentMastership = True
1941 rolesResults = True
1942 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001943 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001944 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001945 name="roles-" + str( i ),
1946 args=[] )
1947 threads.append( t )
1948 t.start()
1949
1950 for t in threads:
1951 t.join()
1952 ONOSMastership.append( t.result )
1953
Jon Halla440e872016-03-31 15:15:50 -07001954 for i in range( len( ONOSMastership ) ):
1955 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001956 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001957 main.log.error( "Error in getting ONOS" + node + " roles" )
1958 main.log.warn( "ONOS" + node + " mastership response: " +
1959 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001960 rolesResults = False
1961 utilities.assert_equals(
1962 expect=True,
1963 actual=rolesResults,
1964 onpass="No error in reading roles output",
1965 onfail="Error in reading roles from ONOS" )
1966
1967 main.step( "Check for consistency in roles from each controller" )
1968 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1969 main.log.info(
1970 "Switch roles are consistent across all ONOS nodes" )
1971 else:
1972 consistentMastership = False
1973 utilities.assert_equals(
1974 expect=True,
1975 actual=consistentMastership,
1976 onpass="Switch roles are consistent across all ONOS nodes",
1977 onfail="ONOS nodes have different views of switch roles" )
1978
1979 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001980 for i in range( len( ONOSMastership ) ):
1981 node = str( main.activeNodes[i] + 1 )
1982 main.log.warn( "ONOS" + node + " roles: ",
Jon Hall6e709752016-02-01 13:38:46 -08001983 json.dumps( json.loads( ONOSMastership[ i ] ),
1984 sort_keys=True,
1985 indent=4,
1986 separators=( ',', ': ' ) ) )
1987 elif rolesResults and consistentMastership:
Jon Hall5cf14d52015-07-16 12:15:19 -07001988 mastershipCheck = main.TRUE
1989
Jon Hall5cf14d52015-07-16 12:15:19 -07001990 # NOTE: we expect mastership to change on controller failure
1991
1992 main.step( "Get the intents and compare across all nodes" )
1993 ONOSIntents = []
1994 intentCheck = main.FALSE
1995 consistentIntents = True
1996 intentsResults = True
1997 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001998 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001999 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07002000 name="intents-" + str( i ),
2001 args=[],
2002 kwargs={ 'jsonFormat': True } )
2003 threads.append( t )
2004 t.start()
2005
2006 for t in threads:
2007 t.join()
2008 ONOSIntents.append( t.result )
2009
Jon Halla440e872016-03-31 15:15:50 -07002010 for i in range( len( ONOSIntents) ):
2011 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002012 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07002013 main.log.error( "Error in getting ONOS" + node + " intents" )
2014 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002015 repr( ONOSIntents[ i ] ) )
2016 intentsResults = False
2017 utilities.assert_equals(
2018 expect=True,
2019 actual=intentsResults,
2020 onpass="No error in reading intents output",
2021 onfail="Error in reading intents from ONOS" )
2022
2023 main.step( "Check for consistency in Intents from each controller" )
2024 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
2025 main.log.info( "Intents are consistent across all ONOS " +
2026 "nodes" )
2027 else:
2028 consistentIntents = False
2029
2030 # Try to make it easy to figure out what is happening
2031 #
2032 # Intent ONOS1 ONOS2 ...
2033 # 0x01 INSTALLED INSTALLING
2034 # ... ... ...
2035 # ... ... ...
2036 title = " ID"
Jon Halla440e872016-03-31 15:15:50 -07002037 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002038 title += " " * 10 + "ONOS" + str( n + 1 )
2039 main.log.warn( title )
2040 # get all intent keys in the cluster
2041 keys = []
2042 for nodeStr in ONOSIntents:
2043 node = json.loads( nodeStr )
2044 for intent in node:
2045 keys.append( intent.get( 'id' ) )
2046 keys = set( keys )
2047 for key in keys:
2048 row = "%-13s" % key
2049 for nodeStr in ONOSIntents:
2050 node = json.loads( nodeStr )
2051 for intent in node:
2052 if intent.get( 'id' ) == key:
2053 row += "%-15s" % intent.get( 'state' )
2054 main.log.warn( row )
2055 # End table view
2056
2057 utilities.assert_equals(
2058 expect=True,
2059 actual=consistentIntents,
2060 onpass="Intents are consistent across all ONOS nodes",
2061 onfail="ONOS nodes have different views of intents" )
2062 intentStates = []
2063 for node in ONOSIntents: # Iter through ONOS nodes
2064 nodeStates = []
2065 # Iter through intents of a node
2066 try:
2067 for intent in json.loads( node ):
2068 nodeStates.append( intent[ 'state' ] )
2069 except ( ValueError, TypeError ):
2070 main.log.exception( "Error in parsing intents" )
2071 main.log.error( repr( node ) )
2072 intentStates.append( nodeStates )
2073 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
2074 main.log.info( dict( out ) )
2075
2076 if intentsResults and not consistentIntents:
Jon Halla440e872016-03-31 15:15:50 -07002077 for i in range( len( main.activeNodes ) ):
2078 node = str( main.activeNodes[i] + 1 )
2079 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002080 main.log.warn( json.dumps(
2081 json.loads( ONOSIntents[ i ] ),
2082 sort_keys=True,
2083 indent=4,
2084 separators=( ',', ': ' ) ) )
2085 elif intentsResults and consistentIntents:
2086 intentCheck = main.TRUE
2087
2088 # NOTE: Store has no durability, so intents are lost across system
2089 # restarts
2090 """
2091 main.step( "Compare current intents with intents before the failure" )
2092 # NOTE: this requires case 5 to pass for intentState to be set.
2093 # maybe we should stop the test if that fails?
2094 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002095 try:
2096 intentState
2097 except NameError:
2098 main.log.warn( "No previous intent state was saved" )
2099 else:
2100 if intentState and intentState == ONOSIntents[ 0 ]:
2101 sameIntents = main.TRUE
2102 main.log.info( "Intents are consistent with before failure" )
2103 # TODO: possibly the states have changed? we may need to figure out
2104 # what the acceptable states are
2105 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2106 sameIntents = main.TRUE
2107 try:
2108 before = json.loads( intentState )
2109 after = json.loads( ONOSIntents[ 0 ] )
2110 for intent in before:
2111 if intent not in after:
2112 sameIntents = main.FALSE
2113 main.log.debug( "Intent is not currently in ONOS " +
2114 "(at least in the same form):" )
2115 main.log.debug( json.dumps( intent ) )
2116 except ( ValueError, TypeError ):
2117 main.log.exception( "Exception printing intents" )
2118 main.log.debug( repr( ONOSIntents[0] ) )
2119 main.log.debug( repr( intentState ) )
2120 if sameIntents == main.FALSE:
2121 try:
2122 main.log.debug( "ONOS intents before: " )
2123 main.log.debug( json.dumps( json.loads( intentState ),
2124 sort_keys=True, indent=4,
2125 separators=( ',', ': ' ) ) )
2126 main.log.debug( "Current ONOS intents: " )
2127 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2128 sort_keys=True, indent=4,
2129 separators=( ',', ': ' ) ) )
2130 except ( ValueError, TypeError ):
2131 main.log.exception( "Exception printing intents" )
2132 main.log.debug( repr( ONOSIntents[0] ) )
2133 main.log.debug( repr( intentState ) )
2134 utilities.assert_equals(
2135 expect=main.TRUE,
2136 actual=sameIntents,
2137 onpass="Intents are consistent with before failure",
2138 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002139 intentCheck = intentCheck and sameIntents
2140 """
2141 main.step( "Get the OF Table entries and compare to before " +
2142 "component failure" )
2143 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002144 for i in range( 28 ):
2145 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002146 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002147 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2148 FlowTables = FlowTables and curSwitch
2149 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002150 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002151 utilities.assert_equals(
2152 expect=main.TRUE,
2153 actual=FlowTables,
2154 onpass="No changes were found in the flow tables",
2155 onfail="Changes were found in the flow tables" )
2156
2157 main.Mininet2.pingLongKill()
2158 '''
2159 # main.step( "Check the continuous pings to ensure that no packets " +
2160 # "were dropped during component failure" )
2161 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2162 main.params[ 'TESTONIP' ] )
2163 LossInPings = main.FALSE
2164 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2165 for i in range( 8, 18 ):
2166 main.log.info(
2167 "Checking for a loss in pings along flow from s" +
2168 str( i ) )
2169 LossInPings = main.Mininet2.checkForLoss(
2170 "/tmp/ping.h" +
2171 str( i ) ) or LossInPings
2172 if LossInPings == main.TRUE:
2173 main.log.info( "Loss in ping detected" )
2174 elif LossInPings == main.ERROR:
2175 main.log.info( "There are multiple mininet process running" )
2176 elif LossInPings == main.FALSE:
2177 main.log.info( "No Loss in the pings" )
2178 main.log.info( "No loss of dataplane connectivity" )
2179 # utilities.assert_equals(
2180 # expect=main.FALSE,
2181 # actual=LossInPings,
2182 # onpass="No Loss of connectivity",
2183 # onfail="Loss of dataplane connectivity detected" )
2184
2185 # NOTE: Since intents are not persisted with IntnentStore,
2186 # we expect loss in dataplane connectivity
2187 LossInPings = main.FALSE
2188 '''
2189
2190 main.step( "Leadership Election is still functional" )
2191 # Test of LeadershipElection
2192 leaderList = []
2193 leaderResult = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002194
2195 for i in main.activeNodes:
2196 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002197 leaderN = cli.electionTestLeader()
2198 leaderList.append( leaderN )
2199 if leaderN == main.FALSE:
2200 # error in response
2201 main.log.error( "Something is wrong with " +
2202 "electionTestLeader function, check the" +
2203 " error logs" )
2204 leaderResult = main.FALSE
2205 elif leaderN is None:
2206 main.log.error( cli.name +
2207 " shows no leader for the election-app." )
2208 leaderResult = main.FALSE
2209 if len( set( leaderList ) ) != 1:
2210 leaderResult = main.FALSE
2211 main.log.error(
2212 "Inconsistent view of leader for the election test app" )
2213 # TODO: print the list
2214 utilities.assert_equals(
2215 expect=main.TRUE,
2216 actual=leaderResult,
2217 onpass="Leadership election passed",
2218 onfail="Something went wrong with Leadership election" )
2219
2220 def CASE8( self, main ):
2221 """
2222 Compare topo
2223 """
2224 import json
2225 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002226 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002227 assert main, "main not defined"
2228 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002229 assert main.CLIs, "main.CLIs not defined"
2230 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002231
2232 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002233 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002234 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002235 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002236 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002237 elapsed = 0
2238 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002239 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002240 startTime = time.time()
2241 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002242 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hallba609822015-09-18 12:00:21 -07002243 devicesResults = main.TRUE
2244 linksResults = main.TRUE
2245 hostsResults = main.TRUE
2246 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002247 count += 1
2248 cliStart = time.time()
2249 devices = []
2250 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002251 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002252 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002253 name="devices-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002254 args=[ main.CLIs[i].devices, [ None ] ],
2255 kwargs= { 'sleep': 5, 'attempts': 5,
2256 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002257 threads.append( t )
2258 t.start()
2259
2260 for t in threads:
2261 t.join()
2262 devices.append( t.result )
2263 hosts = []
2264 ipResult = main.TRUE
2265 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002266 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002267 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002268 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002269 args=[ main.CLIs[i].hosts, [ None ] ],
2270 kwargs= { 'sleep': 5, 'attempts': 5,
2271 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002272 threads.append( t )
2273 t.start()
2274
2275 for t in threads:
2276 t.join()
2277 try:
2278 hosts.append( json.loads( t.result ) )
2279 except ( ValueError, TypeError ):
2280 main.log.exception( "Error parsing hosts results" )
2281 main.log.error( repr( t.result ) )
Jon Hall3afe4c92015-12-14 19:30:38 -08002282 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002283 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002284 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002285 if hosts[ controller ]:
2286 for host in hosts[ controller ]:
2287 if host is None or host.get( 'ipAddresses', [] ) == []:
2288 main.log.error(
2289 "Error with host ipAddresses on controller" +
2290 controllerStr + ": " + str( host ) )
2291 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002292 ports = []
2293 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002294 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002295 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002296 name="ports-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002297 args=[ main.CLIs[i].ports, [ None ] ],
2298 kwargs= { 'sleep': 5, 'attempts': 5,
2299 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002300 threads.append( t )
2301 t.start()
2302
2303 for t in threads:
2304 t.join()
2305 ports.append( t.result )
2306 links = []
2307 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002308 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002309 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002310 name="links-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002311 args=[ main.CLIs[i].links, [ None ] ],
2312 kwargs= { 'sleep': 5, 'attempts': 5,
2313 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002314 threads.append( t )
2315 t.start()
2316
2317 for t in threads:
2318 t.join()
2319 links.append( t.result )
2320 clusters = []
2321 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002322 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002323 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002324 name="clusters-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08002325 args=[ main.CLIs[i].clusters, [ None ] ],
2326 kwargs= { 'sleep': 5, 'attempts': 5,
2327 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002328 threads.append( t )
2329 t.start()
2330
2331 for t in threads:
2332 t.join()
2333 clusters.append( t.result )
2334
2335 elapsed = time.time() - startTime
2336 cliTime = time.time() - cliStart
2337 print "Elapsed time: " + str( elapsed )
2338 print "CLI time: " + str( cliTime )
2339
Jon Hall6e709752016-02-01 13:38:46 -08002340 if all( e is None for e in devices ) and\
2341 all( e is None for e in hosts ) and\
2342 all( e is None for e in ports ) and\
2343 all( e is None for e in links ) and\
2344 all( e is None for e in clusters ):
2345 topoFailMsg = "Could not get topology from ONOS"
2346 main.log.error( topoFailMsg )
2347 continue # Try again, No use trying to compare
2348
Jon Hall5cf14d52015-07-16 12:15:19 -07002349 mnSwitches = main.Mininet1.getSwitches()
2350 mnLinks = main.Mininet1.getLinks()
2351 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07002352 for controller in range( len( main.activeNodes ) ):
2353 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002354 if devices[ controller ] and ports[ controller ] and\
2355 "Error" not in devices[ controller ] and\
2356 "Error" not in ports[ controller ]:
2357
Jon Hallc6793552016-01-19 14:18:37 -08002358 try:
2359 currentDevicesResult = main.Mininet1.compareSwitches(
2360 mnSwitches,
2361 json.loads( devices[ controller ] ),
2362 json.loads( ports[ controller ] ) )
2363 except ( TypeError, ValueError ) as e:
2364 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2365 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002366 else:
2367 currentDevicesResult = main.FALSE
2368 utilities.assert_equals( expect=main.TRUE,
2369 actual=currentDevicesResult,
2370 onpass="ONOS" + controllerStr +
2371 " Switches view is correct",
2372 onfail="ONOS" + controllerStr +
2373 " Switches view is incorrect" )
2374
2375 if links[ controller ] and "Error" not in links[ controller ]:
2376 currentLinksResult = main.Mininet1.compareLinks(
2377 mnSwitches, mnLinks,
2378 json.loads( links[ controller ] ) )
2379 else:
2380 currentLinksResult = main.FALSE
2381 utilities.assert_equals( expect=main.TRUE,
2382 actual=currentLinksResult,
2383 onpass="ONOS" + controllerStr +
2384 " links view is correct",
2385 onfail="ONOS" + controllerStr +
2386 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002387 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002388 currentHostsResult = main.Mininet1.compareHosts(
2389 mnHosts,
2390 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002391 elif hosts[ controller ] == []:
2392 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002393 else:
2394 currentHostsResult = main.FALSE
2395 utilities.assert_equals( expect=main.TRUE,
2396 actual=currentHostsResult,
2397 onpass="ONOS" + controllerStr +
2398 " hosts exist in Mininet",
2399 onfail="ONOS" + controllerStr +
2400 " hosts don't match Mininet" )
2401 # CHECKING HOST ATTACHMENT POINTS
2402 hostAttachment = True
Jon Halla440e872016-03-31 15:15:50 -07002403 zeroHosts = False
Jon Hall5cf14d52015-07-16 12:15:19 -07002404 # FIXME: topo-HA/obelisk specific mappings:
2405 # key is mac and value is dpid
2406 mappings = {}
2407 for i in range( 1, 29 ): # hosts 1 through 28
2408 # set up correct variables:
2409 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2410 if i == 1:
2411 deviceId = "1000".zfill(16)
2412 elif i == 2:
2413 deviceId = "2000".zfill(16)
2414 elif i == 3:
2415 deviceId = "3000".zfill(16)
2416 elif i == 4:
2417 deviceId = "3004".zfill(16)
2418 elif i == 5:
2419 deviceId = "5000".zfill(16)
2420 elif i == 6:
2421 deviceId = "6000".zfill(16)
2422 elif i == 7:
2423 deviceId = "6007".zfill(16)
2424 elif i >= 8 and i <= 17:
2425 dpid = '3' + str( i ).zfill( 3 )
2426 deviceId = dpid.zfill(16)
2427 elif i >= 18 and i <= 27:
2428 dpid = '6' + str( i ).zfill( 3 )
2429 deviceId = dpid.zfill(16)
2430 elif i == 28:
2431 deviceId = "2800".zfill(16)
2432 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002433 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002434 if hosts[ controller ] == []:
2435 main.log.warn( "There are no hosts discovered" )
Jon Halla440e872016-03-31 15:15:50 -07002436 zeroHosts = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002437 else:
2438 for host in hosts[ controller ]:
2439 mac = None
2440 location = None
2441 device = None
2442 port = None
2443 try:
2444 mac = host.get( 'mac' )
2445 assert mac, "mac field could not be found for this host object"
2446
2447 location = host.get( 'location' )
2448 assert location, "location field could not be found for this host object"
2449
2450 # Trim the protocol identifier off deviceId
2451 device = str( location.get( 'elementId' ) ).split(':')[1]
2452 assert device, "elementId field could not be found for this host location object"
2453
2454 port = location.get( 'port' )
2455 assert port, "port field could not be found for this host location object"
2456
2457 # Now check if this matches where they should be
2458 if mac and device and port:
2459 if str( port ) != "1":
2460 main.log.error( "The attachment port is incorrect for " +
2461 "host " + str( mac ) +
2462 ". Expected: 1 Actual: " + str( port) )
2463 hostAttachment = False
2464 if device != mappings[ str( mac ) ]:
2465 main.log.error( "The attachment device is incorrect for " +
2466 "host " + str( mac ) +
2467 ". Expected: " + mappings[ str( mac ) ] +
2468 " Actual: " + device )
2469 hostAttachment = False
2470 else:
2471 hostAttachment = False
2472 except AssertionError:
2473 main.log.exception( "Json object not as expected" )
2474 main.log.error( repr( host ) )
2475 hostAttachment = False
2476 else:
2477 main.log.error( "No hosts json output or \"Error\"" +
2478 " in output. hosts = " +
2479 repr( hosts[ controller ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002480 if zeroHosts is False:
Jon Hall5cf14d52015-07-16 12:15:19 -07002481 # TODO: Find a way to know if there should be hosts in a
2482 # given point of the test
2483 hostAttachment = True
2484
2485 # END CHECKING HOST ATTACHMENT POINTS
2486 devicesResults = devicesResults and currentDevicesResult
2487 linksResults = linksResults and currentLinksResult
2488 hostsResults = hostsResults and currentHostsResult
2489 hostAttachmentResults = hostAttachmentResults and\
2490 hostAttachment
2491 topoResult = ( devicesResults and linksResults
2492 and hostsResults and ipResult and
2493 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002494 utilities.assert_equals( expect=True,
2495 actual=topoResult,
2496 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002497 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002498 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002499
2500 # Compare json objects for hosts and dataplane clusters
2501
2502 # hosts
2503 main.step( "Hosts view is consistent across all ONOS nodes" )
2504 consistentHostsResult = main.TRUE
2505 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002506 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002507 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002508 if hosts[ controller ] == hosts[ 0 ]:
2509 continue
2510 else: # hosts not consistent
2511 main.log.error( "hosts from ONOS" + controllerStr +
2512 " is inconsistent with ONOS1" )
2513 main.log.warn( repr( hosts[ controller ] ) )
2514 consistentHostsResult = main.FALSE
2515
2516 else:
2517 main.log.error( "Error in getting ONOS hosts from ONOS" +
2518 controllerStr )
2519 consistentHostsResult = main.FALSE
2520 main.log.warn( "ONOS" + controllerStr +
2521 " hosts response: " +
2522 repr( hosts[ controller ] ) )
2523 utilities.assert_equals(
2524 expect=main.TRUE,
2525 actual=consistentHostsResult,
2526 onpass="Hosts view is consistent across all ONOS nodes",
2527 onfail="ONOS nodes have different views of hosts" )
2528
2529 main.step( "Hosts information is correct" )
2530 hostsResults = hostsResults and ipResult
2531 utilities.assert_equals(
2532 expect=main.TRUE,
2533 actual=hostsResults,
2534 onpass="Host information is correct",
2535 onfail="Host information is incorrect" )
2536
2537 main.step( "Host attachment points to the network" )
2538 utilities.assert_equals(
2539 expect=True,
2540 actual=hostAttachmentResults,
2541 onpass="Hosts are correctly attached to the network",
2542 onfail="ONOS did not correctly attach hosts to the network" )
2543
2544 # Strongly connected clusters of devices
2545 main.step( "Clusters view is consistent across all ONOS nodes" )
2546 consistentClustersResult = main.TRUE
2547 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07002548 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002549 if "Error" not in clusters[ controller ]:
2550 if clusters[ controller ] == clusters[ 0 ]:
2551 continue
2552 else: # clusters not consistent
2553 main.log.error( "clusters from ONOS" +
2554 controllerStr +
2555 " is inconsistent with ONOS1" )
2556 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002557 else:
2558 main.log.error( "Error in getting dataplane clusters " +
2559 "from ONOS" + controllerStr )
2560 consistentClustersResult = main.FALSE
2561 main.log.warn( "ONOS" + controllerStr +
2562 " clusters response: " +
2563 repr( clusters[ controller ] ) )
2564 utilities.assert_equals(
2565 expect=main.TRUE,
2566 actual=consistentClustersResult,
2567 onpass="Clusters view is consistent across all ONOS nodes",
2568 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002569 if not consistentClustersResult:
2570 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002571
2572 main.step( "There is only one SCC" )
2573 # there should always only be one cluster
2574 try:
2575 numClusters = len( json.loads( clusters[ 0 ] ) )
2576 except ( ValueError, TypeError ):
2577 main.log.exception( "Error parsing clusters[0]: " +
2578 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002579 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002580 clusterResults = main.FALSE
2581 if numClusters == 1:
2582 clusterResults = main.TRUE
2583 utilities.assert_equals(
2584 expect=1,
2585 actual=numClusters,
2586 onpass="ONOS shows 1 SCC",
2587 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2588
2589 topoResult = ( devicesResults and linksResults
2590 and hostsResults and consistentHostsResult
2591 and consistentClustersResult and clusterResults
2592 and ipResult and hostAttachmentResults )
2593
2594 topoResult = topoResult and int( count <= 2 )
2595 note = "note it takes about " + str( int( cliTime ) ) + \
2596 " seconds for the test to make all the cli calls to fetch " +\
2597 "the topology from each ONOS instance"
2598 main.log.info(
2599 "Very crass estimate for topology discovery/convergence( " +
2600 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2601 str( count ) + " tries" )
2602
2603 main.step( "Device information is correct" )
2604 utilities.assert_equals(
2605 expect=main.TRUE,
2606 actual=devicesResults,
2607 onpass="Device information is correct",
2608 onfail="Device information is incorrect" )
2609
2610 main.step( "Links are correct" )
2611 utilities.assert_equals(
2612 expect=main.TRUE,
2613 actual=linksResults,
2614 onpass="Link are correct",
2615 onfail="Links are incorrect" )
2616
Jon Halla440e872016-03-31 15:15:50 -07002617 main.step( "Hosts are correct" )
2618 utilities.assert_equals(
2619 expect=main.TRUE,
2620 actual=hostsResults,
2621 onpass="Hosts are correct",
2622 onfail="Hosts are incorrect" )
2623
Jon Hall5cf14d52015-07-16 12:15:19 -07002624 # FIXME: move this to an ONOS state case
2625 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002626 nodeResults = utilities.retry( main.HA.nodesCheck,
2627 False,
2628 args=[main.activeNodes],
2629 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002630
Jon Hall41d39f12016-04-11 22:54:35 -07002631 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002632 onpass="Nodes check successful",
2633 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002634 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002635 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002636 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002637 main.CLIs[i].name,
2638 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002639
Jon Halld2871c22016-07-26 11:01:14 -07002640 if not topoResult:
2641 main.cleanup()
2642 main.exit()
2643
Jon Hall5cf14d52015-07-16 12:15:19 -07002644 def CASE9( self, main ):
2645 """
2646 Link s3-s28 down
2647 """
2648 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002649 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002650 assert main, "main not defined"
2651 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002652 assert main.CLIs, "main.CLIs not defined"
2653 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002654 # NOTE: You should probably run a topology check after this
2655
2656 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2657
2658 description = "Turn off a link to ensure that Link Discovery " +\
2659 "is working properly"
2660 main.case( description )
2661
2662 main.step( "Kill Link between s3 and s28" )
2663 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2664 main.log.info( "Waiting " + str( linkSleep ) +
2665 " seconds for link down to be discovered" )
2666 time.sleep( linkSleep )
2667 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2668 onpass="Link down successful",
2669 onfail="Failed to bring link down" )
2670 # TODO do some sort of check here
2671
2672 def CASE10( self, main ):
2673 """
2674 Link s3-s28 up
2675 """
2676 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002677 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002678 assert main, "main not defined"
2679 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002680 assert main.CLIs, "main.CLIs not defined"
2681 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002682 # NOTE: You should probably run a topology check after this
2683
2684 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2685
2686 description = "Restore a link to ensure that Link Discovery is " + \
2687 "working properly"
2688 main.case( description )
2689
2690 main.step( "Bring link between s3 and s28 back up" )
2691 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2692 main.log.info( "Waiting " + str( linkSleep ) +
2693 " seconds for link up to be discovered" )
2694 time.sleep( linkSleep )
2695 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2696 onpass="Link up successful",
2697 onfail="Failed to bring link up" )
2698 # TODO do some sort of check here
2699
2700 def CASE11( self, main ):
2701 """
2702 Switch Down
2703 """
2704 # NOTE: You should probably run a topology check after this
2705 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002706 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002707 assert main, "main not defined"
2708 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002709 assert main.CLIs, "main.CLIs not defined"
2710 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002711
2712 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2713
2714 description = "Killing a switch to ensure it is discovered correctly"
Jon Halla440e872016-03-31 15:15:50 -07002715 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002716 main.case( description )
2717 switch = main.params[ 'kill' ][ 'switch' ]
2718 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2719
2720 # TODO: Make this switch parameterizable
2721 main.step( "Kill " + switch )
2722 main.log.info( "Deleting " + switch )
2723 main.Mininet1.delSwitch( switch )
2724 main.log.info( "Waiting " + str( switchSleep ) +
2725 " seconds for switch down to be discovered" )
2726 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002727 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002728 # Peek at the deleted switch
2729 main.log.warn( str( device ) )
2730 result = main.FALSE
2731 if device and device[ 'available' ] is False:
2732 result = main.TRUE
2733 utilities.assert_equals( expect=main.TRUE, actual=result,
2734 onpass="Kill switch successful",
2735 onfail="Failed to kill switch?" )
2736
2737 def CASE12( self, main ):
2738 """
2739 Switch Up
2740 """
2741 # NOTE: You should probably run a topology check after this
2742 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002743 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002744 assert main, "main not defined"
2745 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002746 assert main.CLIs, "main.CLIs not defined"
2747 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002748 assert ONOS1Port, "ONOS1Port not defined"
2749 assert ONOS2Port, "ONOS2Port not defined"
2750 assert ONOS3Port, "ONOS3Port not defined"
2751 assert ONOS4Port, "ONOS4Port not defined"
2752 assert ONOS5Port, "ONOS5Port not defined"
2753 assert ONOS6Port, "ONOS6Port not defined"
2754 assert ONOS7Port, "ONOS7Port not defined"
2755
2756 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2757 switch = main.params[ 'kill' ][ 'switch' ]
2758 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2759 links = main.params[ 'kill' ][ 'links' ].split()
Jon Halla440e872016-03-31 15:15:50 -07002760 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002761 description = "Adding a switch to ensure it is discovered correctly"
2762 main.case( description )
2763
2764 main.step( "Add back " + switch )
2765 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2766 for peer in links:
2767 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07002768 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002769 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2770 main.log.info( "Waiting " + str( switchSleep ) +
2771 " seconds for switch up to be discovered" )
2772 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002773 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002774 # Peek at the deleted switch
2775 main.log.warn( str( device ) )
2776 result = main.FALSE
2777 if device and device[ 'available' ]:
2778 result = main.TRUE
2779 utilities.assert_equals( expect=main.TRUE, actual=result,
2780 onpass="add switch successful",
2781 onfail="Failed to add switch?" )
2782
2783 def CASE13( self, main ):
2784 """
2785 Clean up
2786 """
2787 import os
2788 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002789 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002790 assert main, "main not defined"
2791 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002792 assert main.CLIs, "main.CLIs not defined"
2793 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002794
2795 # printing colors to terminal
2796 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2797 'blue': '\033[94m', 'green': '\033[92m',
2798 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2799 main.case( "Test Cleanup" )
2800 main.step( "Killing tcpdumps" )
2801 main.Mininet2.stopTcpdump()
2802
2803 testname = main.TEST
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002804 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002805 main.step( "Copying MN pcap and ONOS log files to test station" )
2806 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2807 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002808 # NOTE: MN Pcap file is being saved to logdir.
2809 # We scp this file as MN and TestON aren't necessarily the same vm
2810
2811 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002812 # TODO: Load these from params
2813 # NOTE: must end in /
2814 logFolder = "/opt/onos/log/"
2815 logFiles = [ "karaf.log", "karaf.log.1" ]
2816 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002817 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002818 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002819 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002820 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2821 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002822 # std*.log's
2823 # NOTE: must end in /
2824 logFolder = "/opt/onos/var/"
2825 logFiles = [ "stderr.log", "stdout.log" ]
2826 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002827 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002828 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002829 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002830 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2831 logFolder + f, dstName )
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002832 else:
2833 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002834
2835 main.step( "Stopping Mininet" )
2836 mnResult = main.Mininet1.stopNet()
2837 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2838 onpass="Mininet stopped",
2839 onfail="MN cleanup NOT successful" )
2840
2841 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002842 for node in main.nodes:
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002843 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2844 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002845
2846 try:
2847 timerLog = open( main.logdir + "/Timers.csv", 'w')
2848 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2849 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2850 timerLog.close()
2851 except NameError, e:
2852 main.log.exception(e)
2853
2854 def CASE14( self, main ):
2855 """
2856 start election app on all onos nodes
2857 """
Jon Halle1a3b752015-07-22 13:02:46 -07002858 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002859 assert main, "main not defined"
2860 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002861 assert main.CLIs, "main.CLIs not defined"
2862 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002863
2864 main.case("Start Leadership Election app")
2865 main.step( "Install leadership election app" )
Jon Halla440e872016-03-31 15:15:50 -07002866 onosCli = main.CLIs[ main.activeNodes[0] ]
2867 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002868 utilities.assert_equals(
2869 expect=main.TRUE,
2870 actual=appResult,
2871 onpass="Election app installed",
2872 onfail="Something went wrong with installing Leadership election" )
2873
2874 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002875 for i in main.activeNodes:
2876 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002877 time.sleep(5)
2878 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2879 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002880 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002881 expect=True,
2882 actual=sameResult,
2883 onpass="All nodes see the same leaderboards",
2884 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002885
Jon Hall25463a82016-04-13 14:03:52 -07002886 if sameResult:
2887 leader = leaders[ 0 ][ 0 ]
2888 if main.nodes[main.activeNodes[0]].ip_address in leader:
2889 correctLeader = True
2890 else:
2891 correctLeader = False
2892 main.step( "First node was elected leader" )
2893 utilities.assert_equals(
2894 expect=True,
2895 actual=correctLeader,
2896 onpass="Correct leader was elected",
2897 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002898
2899 def CASE15( self, main ):
2900 """
2901 Check that Leadership Election is still functional
acsmars9475b1c2015-08-28 18:02:08 -07002902 15.1 Run election on each node
2903 15.2 Check that each node has the same leaders and candidates
2904 15.3 Find current leader and withdraw
2905 15.4 Check that a new node was elected leader
2906 15.5 Check that that new leader was the candidate of old leader
2907 15.6 Run for election on old leader
2908 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2909 15.8 Make sure that the old leader was added to the candidate list
2910
2911 old and new variable prefixes refer to data from before vs after
2912 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002913 """
2914 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002915 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002916 assert main, "main not defined"
2917 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002918 assert main.CLIs, "main.CLIs not defined"
2919 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002920
Jon Hall5cf14d52015-07-16 12:15:19 -07002921 description = "Check that Leadership Election is still functional"
2922 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002923 # NOTE: Need to re-run after restarts since being a canidate is not persistant
acsmars9475b1c2015-08-28 18:02:08 -07002924
Jon Halla440e872016-03-31 15:15:50 -07002925 oldLeaders = [] # list of lists of each nodes' candidates before
2926 newLeaders = [] # list of lists of each nodes' candidates after
acsmars9475b1c2015-08-28 18:02:08 -07002927 oldLeader = '' # the old leader from oldLeaders, None if not same
2928 newLeader = '' # the new leaders fron newLoeaders, None if not same
2929 oldLeaderCLI = None # the CLI of the old leader used for re-electing
acsmars71adceb2015-08-31 15:09:26 -07002930 expectNoLeader = False # True when there is only one leader
2931 if main.numCtrls == 1:
2932 expectNoLeader = True
acsmars9475b1c2015-08-28 18:02:08 -07002933
Jon Hall5cf14d52015-07-16 12:15:19 -07002934 main.step( "Run for election on each node" )
acsmars9475b1c2015-08-28 18:02:08 -07002935 electionResult = main.TRUE
2936
Jon Halla440e872016-03-31 15:15:50 -07002937 for i in main.activeNodes: # run test election on each node
2938 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars9475b1c2015-08-28 18:02:08 -07002939 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002940 utilities.assert_equals(
2941 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002942 actual=electionResult,
2943 onpass="All nodes successfully ran for leadership",
2944 onfail="At least one node failed to run for leadership" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002945
acsmars3a72bde2015-09-02 14:16:22 -07002946 if electionResult == main.FALSE:
2947 main.log.error(
2948 "Skipping Test Case because Election Test App isn't loaded" )
2949 main.skipCase()
2950
acsmars9475b1c2015-08-28 18:02:08 -07002951 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002952 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002953 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002954 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002955 if sameResult:
2956 oldLeader = oldLeaders[ 0 ][ 0 ]
2957 main.log.warn( oldLeader )
acsmars9475b1c2015-08-28 18:02:08 -07002958 else:
Jon Halla440e872016-03-31 15:15:50 -07002959 oldLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002960 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002961 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002962 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002963 onpass="Leaderboards are consistent for the election topic",
acsmars9475b1c2015-08-28 18:02:08 -07002964 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002965
2966 main.step( "Find current leader and withdraw" )
acsmars9475b1c2015-08-28 18:02:08 -07002967 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002968 # do some sanity checking on leader before using it
acsmars9475b1c2015-08-28 18:02:08 -07002969 if oldLeader is None:
2970 main.log.error( "Leadership isn't consistent." )
2971 withdrawResult = main.FALSE
2972 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002973 for i in main.activeNodes:
acsmars9475b1c2015-08-28 18:02:08 -07002974 if oldLeader == main.nodes[ i ].ip_address:
2975 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002976 break
2977 else: # FOR/ELSE statement
2978 main.log.error( "Leader election, could not find current leader" )
2979 if oldLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002980 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002981 utilities.assert_equals(
2982 expect=main.TRUE,
2983 actual=withdrawResult,
2984 onpass="Node was withdrawn from election",
2985 onfail="Node was not withdrawn from election" )
2986
acsmars9475b1c2015-08-28 18:02:08 -07002987 main.step( "Check that a new node was elected leader" )
acsmars9475b1c2015-08-28 18:02:08 -07002988 failMessage = "Nodes have different leaders"
acsmars9475b1c2015-08-28 18:02:08 -07002989 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002990 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002991 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002992 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002993 if newLeaders[ 0 ][ 0 ] == 'none':
2994 main.log.error( "No leader was elected on at least 1 node" )
2995 if not expectNoLeader:
2996 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002997 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002998
acsmars9475b1c2015-08-28 18:02:08 -07002999 # Check that the new leader is not the older leader, which was withdrawn
3000 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07003001 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08003002 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars9475b1c2015-08-28 18:02:08 -07003003 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003004 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003005 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07003006 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003007 onpass="Leadership election passed",
3008 onfail="Something went wrong with Leadership election" )
3009
Jon Halla440e872016-03-31 15:15:50 -07003010 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08003011 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars9475b1c2015-08-28 18:02:08 -07003012 correctCandidateResult = main.TRUE
acsmars71adceb2015-08-31 15:09:26 -07003013 if expectNoLeader:
3014 if newLeader == 'none':
3015 main.log.info( "No leader expected. None found. Pass" )
3016 correctCandidateResult = main.TRUE
3017 else:
3018 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3019 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003020 elif len( oldLeaders[0] ) >= 3:
3021 if newLeader == oldLeaders[ 0 ][ 2 ]:
3022 # correct leader was elected
3023 correctCandidateResult = main.TRUE
3024 else:
3025 correctCandidateResult = main.FALSE
3026 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3027 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003028 else:
3029 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003030 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003031 correctCandidateResult = main.FALSE
acsmars9475b1c2015-08-28 18:02:08 -07003032 utilities.assert_equals(
3033 expect=main.TRUE,
3034 actual=correctCandidateResult,
3035 onpass="Correct Candidate Elected",
3036 onfail="Incorrect Candidate Elected" )
3037
Jon Hall5cf14d52015-07-16 12:15:19 -07003038 main.step( "Run for election on old leader( just so everyone " +
3039 "is in the hat )" )
acsmars9475b1c2015-08-28 18:02:08 -07003040 if oldLeaderCLI is not None:
3041 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003042 else:
acsmars9475b1c2015-08-28 18:02:08 -07003043 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003044 runResult = main.FALSE
3045 utilities.assert_equals(
3046 expect=main.TRUE,
3047 actual=runResult,
3048 onpass="App re-ran for election",
3049 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003050
acsmars9475b1c2015-08-28 18:02:08 -07003051 main.step(
3052 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003053 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003054 # Get new leaders and candidates
3055 reRunLeaders = []
3056 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003057 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003058
acsmars9475b1c2015-08-28 18:02:08 -07003059 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07003060 if not reRunLeaders[0]:
3061 positionResult = main.FALSE
3062 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07003063 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
3064 str( reRunLeaders[ 0 ] ) ) )
acsmars9475b1c2015-08-28 18:02:08 -07003065 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003066 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003067 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07003068 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003069 onpass="Old leader successfully re-ran for election",
3070 onfail="Something went wrong with Leadership election after " +
3071 "the old leader re-ran for election" )
3072
3073 def CASE16( self, main ):
3074 """
3075 Install Distributed Primitives app
3076 """
3077 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003078 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003079 assert main, "main not defined"
3080 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003081 assert main.CLIs, "main.CLIs not defined"
3082 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003083
3084 # Variables for the distributed primitives tests
3085 global pCounterName
Jon Hall5cf14d52015-07-16 12:15:19 -07003086 global pCounterValue
Jon Hall5cf14d52015-07-16 12:15:19 -07003087 global onosSet
3088 global onosSetName
3089 pCounterName = "TestON-Partitions"
Jon Hall5cf14d52015-07-16 12:15:19 -07003090 pCounterValue = 0
Jon Hall5cf14d52015-07-16 12:15:19 -07003091 onosSet = set([])
3092 onosSetName = "TestON-set"
3093
3094 description = "Install Primitives app"
3095 main.case( description )
3096 main.step( "Install Primitives app" )
3097 appName = "org.onosproject.distributedprimitives"
Jon Halla440e872016-03-31 15:15:50 -07003098 node = main.activeNodes[0]
3099 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003100 utilities.assert_equals( expect=main.TRUE,
3101 actual=appResults,
3102 onpass="Primitives app activated",
3103 onfail="Primitives app not activated" )
3104 time.sleep( 5 ) # To allow all nodes to activate
3105
3106 def CASE17( self, main ):
3107 """
3108 Check for basic functionality with distributed primitives
3109 """
Jon Hall5cf14d52015-07-16 12:15:19 -07003110 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07003111 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003112 assert main, "main not defined"
3113 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003114 assert main.CLIs, "main.CLIs not defined"
3115 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003116 assert pCounterName, "pCounterName not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003117 assert onosSetName, "onosSetName not defined"
3118 # NOTE: assert fails if value is 0/None/Empty/False
3119 try:
3120 pCounterValue
3121 except NameError:
3122 main.log.error( "pCounterValue not defined, setting to 0" )
3123 pCounterValue = 0
3124 try:
Jon Hall5cf14d52015-07-16 12:15:19 -07003125 onosSet
3126 except NameError:
3127 main.log.error( "onosSet not defined, setting to empty Set" )
3128 onosSet = set([])
3129 # Variables for the distributed primitives tests. These are local only
3130 addValue = "a"
3131 addAllValue = "a b c d e f"
3132 retainValue = "c d e f"
3133
3134 description = "Check for basic functionality with distributed " +\
3135 "primitives"
3136 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07003137 main.caseExplanation = "Test the methods of the distributed " +\
3138 "primitives (counters and sets) throught the cli"
Jon Hall5cf14d52015-07-16 12:15:19 -07003139 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07003140 # Partitioned counters
3141 main.step( "Increment then get a default counter on each node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003142 pCounters = []
3143 threads = []
3144 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003145 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003146 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3147 name="counterAddAndGet-" + str( i ),
Jon Hall5cf14d52015-07-16 12:15:19 -07003148 args=[ pCounterName ] )
3149 pCounterValue += 1
3150 addedPValues.append( pCounterValue )
3151 threads.append( t )
3152 t.start()
3153
3154 for t in threads:
3155 t.join()
3156 pCounters.append( t.result )
3157 # Check that counter incremented numController times
3158 pCounterResults = True
3159 for i in addedPValues:
3160 tmpResult = i in pCounters
3161 pCounterResults = pCounterResults and tmpResult
3162 if not tmpResult:
3163 main.log.error( str( i ) + " is not in partitioned "
3164 "counter incremented results" )
3165 utilities.assert_equals( expect=True,
3166 actual=pCounterResults,
3167 onpass="Default counter incremented",
3168 onfail="Error incrementing default" +
3169 " counter" )
3170
Jon Halle1a3b752015-07-22 13:02:46 -07003171 main.step( "Get then Increment a default counter on each node" )
3172 pCounters = []
3173 threads = []
3174 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003175 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003176 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3177 name="counterGetAndAdd-" + str( i ),
3178 args=[ pCounterName ] )
3179 addedPValues.append( pCounterValue )
3180 pCounterValue += 1
3181 threads.append( t )
3182 t.start()
3183
3184 for t in threads:
3185 t.join()
3186 pCounters.append( t.result )
3187 # Check that counter incremented numController times
3188 pCounterResults = True
3189 for i in addedPValues:
3190 tmpResult = i in pCounters
3191 pCounterResults = pCounterResults and tmpResult
3192 if not tmpResult:
3193 main.log.error( str( i ) + " is not in partitioned "
3194 "counter incremented results" )
3195 utilities.assert_equals( expect=True,
3196 actual=pCounterResults,
3197 onpass="Default counter incremented",
3198 onfail="Error incrementing default" +
3199 " counter" )
3200
3201 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003202 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003203 utilities.assert_equals( expect=main.TRUE,
3204 actual=incrementCheck,
3205 onpass="Added counters are correct",
3206 onfail="Added counters are incorrect" )
3207
3208 main.step( "Add -8 to then get a default counter on each node" )
3209 pCounters = []
3210 threads = []
3211 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003212 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003213 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3214 name="counterIncrement-" + str( i ),
3215 args=[ pCounterName ],
3216 kwargs={ "delta": -8 } )
3217 pCounterValue += -8
3218 addedPValues.append( pCounterValue )
3219 threads.append( t )
3220 t.start()
3221
3222 for t in threads:
3223 t.join()
3224 pCounters.append( t.result )
3225 # Check that counter incremented numController times
3226 pCounterResults = True
3227 for i in addedPValues:
3228 tmpResult = i in pCounters
3229 pCounterResults = pCounterResults and tmpResult
3230 if not tmpResult:
3231 main.log.error( str( i ) + " is not in partitioned "
3232 "counter incremented results" )
3233 utilities.assert_equals( expect=True,
3234 actual=pCounterResults,
3235 onpass="Default counter incremented",
3236 onfail="Error incrementing default" +
3237 " counter" )
3238
3239 main.step( "Add 5 to then get a default counter on each node" )
3240 pCounters = []
3241 threads = []
3242 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003243 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003244 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
3245 name="counterIncrement-" + str( i ),
3246 args=[ pCounterName ],
3247 kwargs={ "delta": 5 } )
3248 pCounterValue += 5
3249 addedPValues.append( pCounterValue )
3250 threads.append( t )
3251 t.start()
3252
3253 for t in threads:
3254 t.join()
3255 pCounters.append( t.result )
3256 # Check that counter incremented numController times
3257 pCounterResults = True
3258 for i in addedPValues:
3259 tmpResult = i in pCounters
3260 pCounterResults = pCounterResults and tmpResult
3261 if not tmpResult:
3262 main.log.error( str( i ) + " is not in partitioned "
3263 "counter incremented results" )
3264 utilities.assert_equals( expect=True,
3265 actual=pCounterResults,
3266 onpass="Default counter incremented",
3267 onfail="Error incrementing default" +
3268 " counter" )
3269
3270 main.step( "Get then add 5 to a default counter on each node" )
3271 pCounters = []
3272 threads = []
3273 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07003274 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003275 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
3276 name="counterIncrement-" + str( i ),
3277 args=[ pCounterName ],
3278 kwargs={ "delta": 5 } )
3279 addedPValues.append( pCounterValue )
3280 pCounterValue += 5
3281 threads.append( t )
3282 t.start()
3283
3284 for t in threads:
3285 t.join()
3286 pCounters.append( t.result )
3287 # Check that counter incremented numController times
3288 pCounterResults = True
3289 for i in addedPValues:
3290 tmpResult = i in pCounters
3291 pCounterResults = pCounterResults and tmpResult
3292 if not tmpResult:
3293 main.log.error( str( i ) + " is not in partitioned "
3294 "counter incremented results" )
3295 utilities.assert_equals( expect=True,
3296 actual=pCounterResults,
3297 onpass="Default counter incremented",
3298 onfail="Error incrementing default" +
3299 " counter" )
3300
3301 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07003302 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07003303 utilities.assert_equals( expect=main.TRUE,
3304 actual=incrementCheck,
3305 onpass="Added counters are correct",
3306 onfail="Added counters are incorrect" )
3307
Jon Hall5cf14d52015-07-16 12:15:19 -07003308 # DISTRIBUTED SETS
3309 main.step( "Distributed Set get" )
3310 size = len( onosSet )
3311 getResponses = []
3312 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003313 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003314 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003315 name="setTestGet-" + str( i ),
3316 args=[ onosSetName ] )
3317 threads.append( t )
3318 t.start()
3319 for t in threads:
3320 t.join()
3321 getResponses.append( t.result )
3322
3323 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003324 for i in range( len( main.activeNodes ) ):
3325 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003326 if isinstance( getResponses[ i ], list):
3327 current = set( getResponses[ i ] )
3328 if len( current ) == len( getResponses[ i ] ):
3329 # no repeats
3330 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003331 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003332 " has incorrect view" +
3333 " of set " + onosSetName + ":\n" +
3334 str( getResponses[ i ] ) )
3335 main.log.debug( "Expected: " + str( onosSet ) )
3336 main.log.debug( "Actual: " + str( current ) )
3337 getResults = main.FALSE
3338 else:
3339 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003340 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003341 " has repeat elements in" +
3342 " set " + onosSetName + ":\n" +
3343 str( getResponses[ i ] ) )
3344 getResults = main.FALSE
3345 elif getResponses[ i ] == main.ERROR:
3346 getResults = main.FALSE
3347 utilities.assert_equals( expect=main.TRUE,
3348 actual=getResults,
3349 onpass="Set elements are correct",
3350 onfail="Set elements are incorrect" )
3351
3352 main.step( "Distributed Set size" )
3353 sizeResponses = []
3354 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003355 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003356 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003357 name="setTestSize-" + str( i ),
3358 args=[ onosSetName ] )
3359 threads.append( t )
3360 t.start()
3361 for t in threads:
3362 t.join()
3363 sizeResponses.append( t.result )
3364
3365 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003366 for i in range( len( main.activeNodes ) ):
3367 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003368 if size != sizeResponses[ i ]:
3369 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003370 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003371 " expected a size of " + str( size ) +
3372 " for set " + onosSetName +
3373 " but got " + str( sizeResponses[ i ] ) )
3374 utilities.assert_equals( expect=main.TRUE,
3375 actual=sizeResults,
3376 onpass="Set sizes are correct",
3377 onfail="Set sizes are incorrect" )
3378
3379 main.step( "Distributed Set add()" )
3380 onosSet.add( addValue )
3381 addResponses = []
3382 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003383 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003384 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003385 name="setTestAdd-" + str( i ),
3386 args=[ onosSetName, addValue ] )
3387 threads.append( t )
3388 t.start()
3389 for t in threads:
3390 t.join()
3391 addResponses.append( t.result )
3392
3393 # main.TRUE = successfully changed the set
3394 # main.FALSE = action resulted in no change in set
3395 # main.ERROR - Some error in executing the function
3396 addResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003397 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003398 if addResponses[ i ] == main.TRUE:
3399 # All is well
3400 pass
3401 elif addResponses[ i ] == main.FALSE:
3402 # Already in set, probably fine
3403 pass
3404 elif addResponses[ i ] == main.ERROR:
3405 # Error in execution
3406 addResults = main.FALSE
3407 else:
3408 # unexpected result
3409 addResults = main.FALSE
3410 if addResults != main.TRUE:
3411 main.log.error( "Error executing set add" )
3412
3413 # Check if set is still correct
3414 size = len( onosSet )
3415 getResponses = []
3416 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003417 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003418 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003419 name="setTestGet-" + str( i ),
3420 args=[ onosSetName ] )
3421 threads.append( t )
3422 t.start()
3423 for t in threads:
3424 t.join()
3425 getResponses.append( t.result )
3426 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003427 for i in range( len( main.activeNodes ) ):
3428 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003429 if isinstance( getResponses[ i ], list):
3430 current = set( getResponses[ i ] )
3431 if len( current ) == len( getResponses[ i ] ):
3432 # no repeats
3433 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003434 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003435 " of set " + onosSetName + ":\n" +
3436 str( getResponses[ i ] ) )
3437 main.log.debug( "Expected: " + str( onosSet ) )
3438 main.log.debug( "Actual: " + str( current ) )
3439 getResults = main.FALSE
3440 else:
3441 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003442 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall5cf14d52015-07-16 12:15:19 -07003443 " set " + onosSetName + ":\n" +
3444 str( getResponses[ i ] ) )
3445 getResults = main.FALSE
3446 elif getResponses[ i ] == main.ERROR:
3447 getResults = main.FALSE
3448 sizeResponses = []
3449 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003450 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003451 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003452 name="setTestSize-" + str( i ),
3453 args=[ onosSetName ] )
3454 threads.append( t )
3455 t.start()
3456 for t in threads:
3457 t.join()
3458 sizeResponses.append( t.result )
3459 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003460 for i in range( len( main.activeNodes ) ):
3461 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003462 if size != sizeResponses[ i ]:
3463 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003464 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003465 " expected a size of " + str( size ) +
3466 " for set " + onosSetName +
3467 " but got " + str( sizeResponses[ i ] ) )
3468 addResults = addResults and getResults and sizeResults
3469 utilities.assert_equals( expect=main.TRUE,
3470 actual=addResults,
3471 onpass="Set add correct",
3472 onfail="Set add was incorrect" )
3473
3474 main.step( "Distributed Set addAll()" )
3475 onosSet.update( addAllValue.split() )
3476 addResponses = []
3477 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003478 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003479 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003480 name="setTestAddAll-" + str( i ),
3481 args=[ onosSetName, addAllValue ] )
3482 threads.append( t )
3483 t.start()
3484 for t in threads:
3485 t.join()
3486 addResponses.append( t.result )
3487
3488 # main.TRUE = successfully changed the set
3489 # main.FALSE = action resulted in no change in set
3490 # main.ERROR - Some error in executing the function
3491 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003492 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003493 if addResponses[ i ] == main.TRUE:
3494 # All is well
3495 pass
3496 elif addResponses[ i ] == main.FALSE:
3497 # Already in set, probably fine
3498 pass
3499 elif addResponses[ i ] == main.ERROR:
3500 # Error in execution
3501 addAllResults = main.FALSE
3502 else:
3503 # unexpected result
3504 addAllResults = main.FALSE
3505 if addAllResults != main.TRUE:
3506 main.log.error( "Error executing set addAll" )
3507
3508 # Check if set is still correct
3509 size = len( onosSet )
3510 getResponses = []
3511 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003512 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003513 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003514 name="setTestGet-" + str( i ),
3515 args=[ onosSetName ] )
3516 threads.append( t )
3517 t.start()
3518 for t in threads:
3519 t.join()
3520 getResponses.append( t.result )
3521 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003522 for i in range( len( main.activeNodes ) ):
3523 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003524 if isinstance( getResponses[ i ], list):
3525 current = set( getResponses[ i ] )
3526 if len( current ) == len( getResponses[ i ] ):
3527 # no repeats
3528 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003529 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003530 " has incorrect view" +
3531 " of set " + onosSetName + ":\n" +
3532 str( getResponses[ i ] ) )
3533 main.log.debug( "Expected: " + str( onosSet ) )
3534 main.log.debug( "Actual: " + str( current ) )
3535 getResults = main.FALSE
3536 else:
3537 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003538 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003539 " has repeat elements in" +
3540 " set " + onosSetName + ":\n" +
3541 str( getResponses[ i ] ) )
3542 getResults = main.FALSE
3543 elif getResponses[ i ] == main.ERROR:
3544 getResults = main.FALSE
3545 sizeResponses = []
3546 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003547 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003548 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003549 name="setTestSize-" + str( i ),
3550 args=[ onosSetName ] )
3551 threads.append( t )
3552 t.start()
3553 for t in threads:
3554 t.join()
3555 sizeResponses.append( t.result )
3556 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003557 for i in range( len( main.activeNodes ) ):
3558 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003559 if size != sizeResponses[ i ]:
3560 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003561 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003562 " expected a size of " + str( size ) +
3563 " for set " + onosSetName +
3564 " but got " + str( sizeResponses[ i ] ) )
3565 addAllResults = addAllResults and getResults and sizeResults
3566 utilities.assert_equals( expect=main.TRUE,
3567 actual=addAllResults,
3568 onpass="Set addAll correct",
3569 onfail="Set addAll was incorrect" )
3570
3571 main.step( "Distributed Set contains()" )
3572 containsResponses = []
3573 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003574 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003575 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003576 name="setContains-" + str( i ),
3577 args=[ onosSetName ],
3578 kwargs={ "values": addValue } )
3579 threads.append( t )
3580 t.start()
3581 for t in threads:
3582 t.join()
3583 # NOTE: This is the tuple
3584 containsResponses.append( t.result )
3585
3586 containsResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003587 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003588 if containsResponses[ i ] == main.ERROR:
3589 containsResults = main.FALSE
3590 else:
3591 containsResults = containsResults and\
3592 containsResponses[ i ][ 1 ]
3593 utilities.assert_equals( expect=main.TRUE,
3594 actual=containsResults,
3595 onpass="Set contains is functional",
3596 onfail="Set contains failed" )
3597
3598 main.step( "Distributed Set containsAll()" )
3599 containsAllResponses = []
3600 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003601 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003602 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003603 name="setContainsAll-" + str( i ),
3604 args=[ onosSetName ],
3605 kwargs={ "values": addAllValue } )
3606 threads.append( t )
3607 t.start()
3608 for t in threads:
3609 t.join()
3610 # NOTE: This is the tuple
3611 containsAllResponses.append( t.result )
3612
3613 containsAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003614 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003615 if containsResponses[ i ] == main.ERROR:
3616 containsResults = main.FALSE
3617 else:
3618 containsResults = containsResults and\
3619 containsResponses[ i ][ 1 ]
3620 utilities.assert_equals( expect=main.TRUE,
3621 actual=containsAllResults,
3622 onpass="Set containsAll is functional",
3623 onfail="Set containsAll failed" )
3624
3625 main.step( "Distributed Set remove()" )
3626 onosSet.remove( addValue )
3627 removeResponses = []
3628 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003629 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003630 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003631 name="setTestRemove-" + str( i ),
3632 args=[ onosSetName, addValue ] )
3633 threads.append( t )
3634 t.start()
3635 for t in threads:
3636 t.join()
3637 removeResponses.append( t.result )
3638
3639 # main.TRUE = successfully changed the set
3640 # main.FALSE = action resulted in no change in set
3641 # main.ERROR - Some error in executing the function
3642 removeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003643 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003644 if removeResponses[ i ] == main.TRUE:
3645 # All is well
3646 pass
3647 elif removeResponses[ i ] == main.FALSE:
3648 # not in set, probably fine
3649 pass
3650 elif removeResponses[ i ] == main.ERROR:
3651 # Error in execution
3652 removeResults = main.FALSE
3653 else:
3654 # unexpected result
3655 removeResults = main.FALSE
3656 if removeResults != main.TRUE:
3657 main.log.error( "Error executing set remove" )
3658
3659 # Check if set is still correct
3660 size = len( onosSet )
3661 getResponses = []
3662 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003663 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003664 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003665 name="setTestGet-" + str( i ),
3666 args=[ onosSetName ] )
3667 threads.append( t )
3668 t.start()
3669 for t in threads:
3670 t.join()
3671 getResponses.append( t.result )
3672 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003673 for i in range( len( main.activeNodes ) ):
3674 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003675 if isinstance( getResponses[ i ], list):
3676 current = set( getResponses[ i ] )
3677 if len( current ) == len( getResponses[ i ] ):
3678 # no repeats
3679 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003680 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003681 " has incorrect view" +
3682 " of set " + onosSetName + ":\n" +
3683 str( getResponses[ i ] ) )
3684 main.log.debug( "Expected: " + str( onosSet ) )
3685 main.log.debug( "Actual: " + str( current ) )
3686 getResults = main.FALSE
3687 else:
3688 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003689 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003690 " has repeat elements in" +
3691 " set " + onosSetName + ":\n" +
3692 str( getResponses[ i ] ) )
3693 getResults = main.FALSE
3694 elif getResponses[ i ] == main.ERROR:
3695 getResults = main.FALSE
3696 sizeResponses = []
3697 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003698 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003699 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003700 name="setTestSize-" + str( i ),
3701 args=[ onosSetName ] )
3702 threads.append( t )
3703 t.start()
3704 for t in threads:
3705 t.join()
3706 sizeResponses.append( t.result )
3707 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003708 for i in range( len( main.activeNodes ) ):
3709 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003710 if size != sizeResponses[ i ]:
3711 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003712 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003713 " expected a size of " + str( size ) +
3714 " for set " + onosSetName +
3715 " but got " + str( sizeResponses[ i ] ) )
3716 removeResults = removeResults and getResults and sizeResults
3717 utilities.assert_equals( expect=main.TRUE,
3718 actual=removeResults,
3719 onpass="Set remove correct",
3720 onfail="Set remove was incorrect" )
3721
3722 main.step( "Distributed Set removeAll()" )
3723 onosSet.difference_update( addAllValue.split() )
3724 removeAllResponses = []
3725 threads = []
3726 try:
Jon Halla440e872016-03-31 15:15:50 -07003727 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003728 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003729 name="setTestRemoveAll-" + str( i ),
3730 args=[ onosSetName, addAllValue ] )
3731 threads.append( t )
3732 t.start()
3733 for t in threads:
3734 t.join()
3735 removeAllResponses.append( t.result )
3736 except Exception, e:
3737 main.log.exception(e)
3738
3739 # main.TRUE = successfully changed the set
3740 # main.FALSE = action resulted in no change in set
3741 # main.ERROR - Some error in executing the function
3742 removeAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003743 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003744 if removeAllResponses[ i ] == main.TRUE:
3745 # All is well
3746 pass
3747 elif removeAllResponses[ i ] == main.FALSE:
3748 # not in set, probably fine
3749 pass
3750 elif removeAllResponses[ i ] == main.ERROR:
3751 # Error in execution
3752 removeAllResults = main.FALSE
3753 else:
3754 # unexpected result
3755 removeAllResults = main.FALSE
3756 if removeAllResults != main.TRUE:
3757 main.log.error( "Error executing set removeAll" )
3758
3759 # Check if set is still correct
3760 size = len( onosSet )
3761 getResponses = []
3762 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003763 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003764 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003765 name="setTestGet-" + str( i ),
3766 args=[ onosSetName ] )
3767 threads.append( t )
3768 t.start()
3769 for t in threads:
3770 t.join()
3771 getResponses.append( t.result )
3772 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003773 for i in range( len( main.activeNodes ) ):
3774 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003775 if isinstance( getResponses[ i ], list):
3776 current = set( getResponses[ i ] )
3777 if len( current ) == len( getResponses[ i ] ):
3778 # no repeats
3779 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003780 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003781 " has incorrect view" +
3782 " of set " + onosSetName + ":\n" +
3783 str( getResponses[ i ] ) )
3784 main.log.debug( "Expected: " + str( onosSet ) )
3785 main.log.debug( "Actual: " + str( current ) )
3786 getResults = main.FALSE
3787 else:
3788 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003789 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003790 " has repeat elements in" +
3791 " set " + onosSetName + ":\n" +
3792 str( getResponses[ i ] ) )
3793 getResults = main.FALSE
3794 elif getResponses[ i ] == main.ERROR:
3795 getResults = main.FALSE
3796 sizeResponses = []
3797 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003798 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003799 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003800 name="setTestSize-" + str( i ),
3801 args=[ onosSetName ] )
3802 threads.append( t )
3803 t.start()
3804 for t in threads:
3805 t.join()
3806 sizeResponses.append( t.result )
3807 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003808 for i in range( len( main.activeNodes ) ):
3809 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003810 if size != sizeResponses[ i ]:
3811 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003812 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003813 " expected a size of " + str( size ) +
3814 " for set " + onosSetName +
3815 " but got " + str( sizeResponses[ i ] ) )
3816 removeAllResults = removeAllResults and getResults and sizeResults
3817 utilities.assert_equals( expect=main.TRUE,
3818 actual=removeAllResults,
3819 onpass="Set removeAll correct",
3820 onfail="Set removeAll was incorrect" )
3821
3822 main.step( "Distributed Set addAll()" )
3823 onosSet.update( addAllValue.split() )
3824 addResponses = []
3825 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003826 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003827 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07003828 name="setTestAddAll-" + str( i ),
3829 args=[ onosSetName, addAllValue ] )
3830 threads.append( t )
3831 t.start()
3832 for t in threads:
3833 t.join()
3834 addResponses.append( t.result )
3835
3836 # main.TRUE = successfully changed the set
3837 # main.FALSE = action resulted in no change in set
3838 # main.ERROR - Some error in executing the function
3839 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003840 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003841 if addResponses[ i ] == main.TRUE:
3842 # All is well
3843 pass
3844 elif addResponses[ i ] == main.FALSE:
3845 # Already in set, probably fine
3846 pass
3847 elif addResponses[ i ] == main.ERROR:
3848 # Error in execution
3849 addAllResults = main.FALSE
3850 else:
3851 # unexpected result
3852 addAllResults = main.FALSE
3853 if addAllResults != main.TRUE:
3854 main.log.error( "Error executing set addAll" )
3855
3856 # Check if set is still correct
3857 size = len( onosSet )
3858 getResponses = []
3859 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003860 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003861 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003862 name="setTestGet-" + str( i ),
3863 args=[ onosSetName ] )
3864 threads.append( t )
3865 t.start()
3866 for t in threads:
3867 t.join()
3868 getResponses.append( t.result )
3869 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003870 for i in range( len( main.activeNodes ) ):
3871 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003872 if isinstance( getResponses[ i ], list):
3873 current = set( getResponses[ i ] )
3874 if len( current ) == len( getResponses[ i ] ):
3875 # no repeats
3876 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003877 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003878 " has incorrect view" +
3879 " of set " + onosSetName + ":\n" +
3880 str( getResponses[ i ] ) )
3881 main.log.debug( "Expected: " + str( onosSet ) )
3882 main.log.debug( "Actual: " + str( current ) )
3883 getResults = main.FALSE
3884 else:
3885 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003886 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003887 " has repeat elements in" +
3888 " set " + onosSetName + ":\n" +
3889 str( getResponses[ i ] ) )
3890 getResults = main.FALSE
3891 elif getResponses[ i ] == main.ERROR:
3892 getResults = main.FALSE
3893 sizeResponses = []
3894 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003895 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003896 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003897 name="setTestSize-" + str( i ),
3898 args=[ onosSetName ] )
3899 threads.append( t )
3900 t.start()
3901 for t in threads:
3902 t.join()
3903 sizeResponses.append( t.result )
3904 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003905 for i in range( len( main.activeNodes ) ):
3906 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003907 if size != sizeResponses[ i ]:
3908 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08003909 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003910 " expected a size of " + str( size ) +
3911 " for set " + onosSetName +
3912 " but got " + str( sizeResponses[ i ] ) )
3913 addAllResults = addAllResults and getResults and sizeResults
3914 utilities.assert_equals( expect=main.TRUE,
3915 actual=addAllResults,
3916 onpass="Set addAll correct",
3917 onfail="Set addAll was incorrect" )
3918
3919 main.step( "Distributed Set clear()" )
3920 onosSet.clear()
3921 clearResponses = []
3922 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003923 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003924 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07003925 name="setTestClear-" + str( i ),
3926 args=[ onosSetName, " "], # Values doesn't matter
3927 kwargs={ "clear": True } )
3928 threads.append( t )
3929 t.start()
3930 for t in threads:
3931 t.join()
3932 clearResponses.append( t.result )
3933
3934 # main.TRUE = successfully changed the set
3935 # main.FALSE = action resulted in no change in set
3936 # main.ERROR - Some error in executing the function
3937 clearResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003938 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07003939 if clearResponses[ i ] == main.TRUE:
3940 # All is well
3941 pass
3942 elif clearResponses[ i ] == main.FALSE:
3943 # Nothing set, probably fine
3944 pass
3945 elif clearResponses[ i ] == main.ERROR:
3946 # Error in execution
3947 clearResults = main.FALSE
3948 else:
3949 # unexpected result
3950 clearResults = main.FALSE
3951 if clearResults != main.TRUE:
3952 main.log.error( "Error executing set clear" )
3953
3954 # Check if set is still correct
3955 size = len( onosSet )
3956 getResponses = []
3957 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003958 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003959 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07003960 name="setTestGet-" + str( i ),
3961 args=[ onosSetName ] )
3962 threads.append( t )
3963 t.start()
3964 for t in threads:
3965 t.join()
3966 getResponses.append( t.result )
3967 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003968 for i in range( len( main.activeNodes ) ):
3969 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07003970 if isinstance( getResponses[ i ], list):
3971 current = set( getResponses[ i ] )
3972 if len( current ) == len( getResponses[ i ] ):
3973 # no repeats
3974 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08003975 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003976 " has incorrect view" +
3977 " of set " + onosSetName + ":\n" +
3978 str( getResponses[ i ] ) )
3979 main.log.debug( "Expected: " + str( onosSet ) )
3980 main.log.debug( "Actual: " + str( current ) )
3981 getResults = main.FALSE
3982 else:
3983 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08003984 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07003985 " has repeat elements in" +
3986 " set " + onosSetName + ":\n" +
3987 str( getResponses[ i ] ) )
3988 getResults = main.FALSE
3989 elif getResponses[ i ] == main.ERROR:
3990 getResults = main.FALSE
3991 sizeResponses = []
3992 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003993 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003994 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07003995 name="setTestSize-" + str( i ),
3996 args=[ onosSetName ] )
3997 threads.append( t )
3998 t.start()
3999 for t in threads:
4000 t.join()
4001 sizeResponses.append( t.result )
4002 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004003 for i in range( len( main.activeNodes ) ):
4004 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004005 if size != sizeResponses[ i ]:
4006 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004007 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004008 " expected a size of " + str( size ) +
4009 " for set " + onosSetName +
4010 " but got " + str( sizeResponses[ i ] ) )
4011 clearResults = clearResults and getResults and sizeResults
4012 utilities.assert_equals( expect=main.TRUE,
4013 actual=clearResults,
4014 onpass="Set clear correct",
4015 onfail="Set clear was incorrect" )
4016
4017 main.step( "Distributed Set addAll()" )
4018 onosSet.update( addAllValue.split() )
4019 addResponses = []
4020 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004021 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004022 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall5cf14d52015-07-16 12:15:19 -07004023 name="setTestAddAll-" + str( i ),
4024 args=[ onosSetName, addAllValue ] )
4025 threads.append( t )
4026 t.start()
4027 for t in threads:
4028 t.join()
4029 addResponses.append( t.result )
4030
4031 # main.TRUE = successfully changed the set
4032 # main.FALSE = action resulted in no change in set
4033 # main.ERROR - Some error in executing the function
4034 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004035 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004036 if addResponses[ i ] == main.TRUE:
4037 # All is well
4038 pass
4039 elif addResponses[ i ] == main.FALSE:
4040 # Already in set, probably fine
4041 pass
4042 elif addResponses[ i ] == main.ERROR:
4043 # Error in execution
4044 addAllResults = main.FALSE
4045 else:
4046 # unexpected result
4047 addAllResults = main.FALSE
4048 if addAllResults != main.TRUE:
4049 main.log.error( "Error executing set addAll" )
4050
4051 # Check if set is still correct
4052 size = len( onosSet )
4053 getResponses = []
4054 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004055 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004056 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004057 name="setTestGet-" + str( i ),
4058 args=[ onosSetName ] )
4059 threads.append( t )
4060 t.start()
4061 for t in threads:
4062 t.join()
4063 getResponses.append( t.result )
4064 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004065 for i in range( len( main.activeNodes ) ):
4066 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004067 if isinstance( getResponses[ i ], list):
4068 current = set( getResponses[ i ] )
4069 if len( current ) == len( getResponses[ i ] ):
4070 # no repeats
4071 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004072 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004073 " has incorrect view" +
4074 " of set " + onosSetName + ":\n" +
4075 str( getResponses[ i ] ) )
4076 main.log.debug( "Expected: " + str( onosSet ) )
4077 main.log.debug( "Actual: " + str( current ) )
4078 getResults = main.FALSE
4079 else:
4080 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004081 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004082 " has repeat elements in" +
4083 " set " + onosSetName + ":\n" +
4084 str( getResponses[ i ] ) )
4085 getResults = main.FALSE
4086 elif getResponses[ i ] == main.ERROR:
4087 getResults = main.FALSE
4088 sizeResponses = []
4089 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004090 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004091 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004092 name="setTestSize-" + str( i ),
4093 args=[ onosSetName ] )
4094 threads.append( t )
4095 t.start()
4096 for t in threads:
4097 t.join()
4098 sizeResponses.append( t.result )
4099 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004100 for i in range( len( main.activeNodes ) ):
4101 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004102 if size != sizeResponses[ i ]:
4103 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004104 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004105 " expected a size of " + str( size ) +
4106 " for set " + onosSetName +
4107 " but got " + str( sizeResponses[ i ] ) )
4108 addAllResults = addAllResults and getResults and sizeResults
4109 utilities.assert_equals( expect=main.TRUE,
4110 actual=addAllResults,
4111 onpass="Set addAll correct",
4112 onfail="Set addAll was incorrect" )
4113
4114 main.step( "Distributed Set retain()" )
4115 onosSet.intersection_update( retainValue.split() )
4116 retainResponses = []
4117 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004118 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004119 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall5cf14d52015-07-16 12:15:19 -07004120 name="setTestRetain-" + str( i ),
4121 args=[ onosSetName, retainValue ],
4122 kwargs={ "retain": True } )
4123 threads.append( t )
4124 t.start()
4125 for t in threads:
4126 t.join()
4127 retainResponses.append( t.result )
4128
4129 # main.TRUE = successfully changed the set
4130 # main.FALSE = action resulted in no change in set
4131 # main.ERROR - Some error in executing the function
4132 retainResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004133 for i in range( len( main.activeNodes ) ):
Jon Hall5cf14d52015-07-16 12:15:19 -07004134 if retainResponses[ i ] == main.TRUE:
4135 # All is well
4136 pass
4137 elif retainResponses[ i ] == main.FALSE:
4138 # Already in set, probably fine
4139 pass
4140 elif retainResponses[ i ] == main.ERROR:
4141 # Error in execution
4142 retainResults = main.FALSE
4143 else:
4144 # unexpected result
4145 retainResults = main.FALSE
4146 if retainResults != main.TRUE:
4147 main.log.error( "Error executing set retain" )
4148
4149 # Check if set is still correct
4150 size = len( onosSet )
4151 getResponses = []
4152 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004153 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004154 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall5cf14d52015-07-16 12:15:19 -07004155 name="setTestGet-" + str( i ),
4156 args=[ onosSetName ] )
4157 threads.append( t )
4158 t.start()
4159 for t in threads:
4160 t.join()
4161 getResponses.append( t.result )
4162 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004163 for i in range( len( main.activeNodes ) ):
4164 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004165 if isinstance( getResponses[ i ], list):
4166 current = set( getResponses[ i ] )
4167 if len( current ) == len( getResponses[ i ] ):
4168 # no repeats
4169 if onosSet != current:
Jon Hall6e709752016-02-01 13:38:46 -08004170 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004171 " has incorrect view" +
4172 " of set " + onosSetName + ":\n" +
4173 str( getResponses[ i ] ) )
4174 main.log.debug( "Expected: " + str( onosSet ) )
4175 main.log.debug( "Actual: " + str( current ) )
4176 getResults = main.FALSE
4177 else:
4178 # error, set is not a set
Jon Hall6e709752016-02-01 13:38:46 -08004179 main.log.error( "ONOS" + node +
Jon Hall5cf14d52015-07-16 12:15:19 -07004180 " has repeat elements in" +
4181 " set " + onosSetName + ":\n" +
4182 str( getResponses[ i ] ) )
4183 getResults = main.FALSE
4184 elif getResponses[ i ] == main.ERROR:
4185 getResults = main.FALSE
4186 sizeResponses = []
4187 threads = []
Jon Halla440e872016-03-31 15:15:50 -07004188 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07004189 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall5cf14d52015-07-16 12:15:19 -07004190 name="setTestSize-" + str( i ),
4191 args=[ onosSetName ] )
4192 threads.append( t )
4193 t.start()
4194 for t in threads:
4195 t.join()
4196 sizeResponses.append( t.result )
4197 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07004198 for i in range( len( main.activeNodes ) ):
4199 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07004200 if size != sizeResponses[ i ]:
4201 sizeResults = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08004202 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall5cf14d52015-07-16 12:15:19 -07004203 str( size ) + " for set " + onosSetName +
4204 " but got " + str( sizeResponses[ i ] ) )
4205 retainResults = retainResults and getResults and sizeResults
4206 utilities.assert_equals( expect=main.TRUE,
4207 actual=retainResults,
4208 onpass="Set retain correct",
4209 onfail="Set retain was incorrect" )
4210
Jon Hall2a5002c2015-08-21 16:49:11 -07004211 # Transactional maps
4212 main.step( "Partitioned Transactional maps put" )
4213 tMapValue = "Testing"
4214 numKeys = 100
4215 putResult = True
Jon Halla440e872016-03-31 15:15:50 -07004216 node = main.activeNodes[0]
4217 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
Jon Hall6e709752016-02-01 13:38:46 -08004218 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07004219 for i in putResponses:
4220 if putResponses[ i ][ 'value' ] != tMapValue:
4221 putResult = False
4222 else:
4223 putResult = False
4224 if not putResult:
4225 main.log.debug( "Put response values: " + str( putResponses ) )
4226 utilities.assert_equals( expect=True,
4227 actual=putResult,
4228 onpass="Partitioned Transactional Map put successful",
4229 onfail="Partitioned Transactional Map put values are incorrect" )
4230
4231 main.step( "Partitioned Transactional maps get" )
Jon Hall9bfadd22016-05-11 14:48:07 -07004232 # FIXME: is this sleep needed?
4233 time.sleep( 5 )
4234
Jon Hall2a5002c2015-08-21 16:49:11 -07004235 getCheck = True
4236 for n in range( 1, numKeys + 1 ):
4237 getResponses = []
4238 threads = []
4239 valueCheck = True
Jon Halla440e872016-03-31 15:15:50 -07004240 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07004241 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
4242 name="TMap-get-" + str( i ),
Jon Hall6e709752016-02-01 13:38:46 -08004243 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07004244 threads.append( t )
4245 t.start()
4246 for t in threads:
4247 t.join()
4248 getResponses.append( t.result )
4249 for node in getResponses:
4250 if node != tMapValue:
4251 valueCheck = False
4252 if not valueCheck:
4253 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
4254 main.log.warn( getResponses )
4255 getCheck = getCheck and valueCheck
4256 utilities.assert_equals( expect=True,
4257 actual=getCheck,
4258 onpass="Partitioned Transactional Map get values were correct",
4259 onfail="Partitioned Transactional Map values incorrect" )