blob: 2b2ea084f2a863d127d4cab1c532f4441023986e [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
26
27
28class HAsanity:
29
30 def __init__( self ):
31 self.default = ''
32
33 def CASE1( self, main ):
34 """
35 CASE1 is to compile ONOS and push it to the test machines
36
37 Startup sequence:
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-uninstall
42 start mininet
43 git pull
44 mvn clean install
45 onos-package
46 onos-install -f
47 onos-wait-for-start
48 start cli sessions
49 start tcpdump
50 """
Jon Halle1a3b752015-07-22 13:02:46 -070051 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080052 import time
Jon Halla440e872016-03-31 15:15:50 -070053 import json
Jon Hall5cf14d52015-07-16 12:15:19 -070054 main.log.info( "ONOS HA Sanity test - initialization" )
55 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070056 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070057 "installing ONOS, starting Mininet and ONOS" +\
58 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070059
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070071 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070072 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070073 global ONOS1Port
74 global ONOS2Port
75 global ONOS3Port
76 global ONOS4Port
77 global ONOS5Port
78 global ONOS6Port
79 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070080 # These are for csv plotting in jenkins
81 global labels
82 global data
83 labels = []
84 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070085
86 # FIXME: just get controller port from params?
87 # TODO: do we really need all these?
88 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
89 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
90 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
91 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
92 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
93 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
94 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
95
Jon Halle1a3b752015-07-22 13:02:46 -070096 try:
Jon Hall53c5e662016-04-13 16:06:56 -070097 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070098 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070099 except Exception as e:
100 main.log.exception( e )
101 main.cleanup()
102 main.exit()
103
104 main.CLIs = []
105 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700106 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700107 for i in range( 1, main.numCtrls + 1 ):
108 try:
109 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
110 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
111 ipList.append( main.nodes[ -1 ].ip_address )
112 except AttributeError:
113 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700114
115 main.step( "Create cell file" )
116 cellAppString = main.params[ 'ENV' ][ 'appString' ]
117 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
118 main.Mininet1.ip_address,
119 cellAppString, ipList )
120 main.step( "Applying cell variable to environment" )
121 cellResult = main.ONOSbench.setCell( cellName )
122 verifyResult = main.ONOSbench.verifyCell()
123
124 # FIXME:this is short term fix
125 main.log.info( "Removing raft logs" )
126 main.ONOSbench.onosRemoveRaftLogs()
127
128 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700129 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700130 main.ONOSbench.onosUninstall( node.ip_address )
131
132 # Make sure ONOS is DEAD
133 main.log.info( "Killing any ONOS processes" )
134 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700135 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700136 killed = main.ONOSbench.onosKill( node.ip_address )
137 killResults = killResults and killed
138
139 cleanInstallResult = main.TRUE
140 gitPullResult = main.TRUE
141
142 main.step( "Starting Mininet" )
143 # scp topo file to mininet
144 # TODO: move to params?
145 topoName = "obelisk.py"
146 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700147 main.ONOSbench.scp( main.Mininet1,
148 filePath + topoName,
149 main.Mininet1.home,
150 direction="to" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700151 mnResult = main.Mininet1.startNet( )
152 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
153 onpass="Mininet Started",
154 onfail="Error starting Mininet" )
155
156 main.step( "Git checkout and pull " + gitBranch )
157 if PULLCODE:
158 main.ONOSbench.gitCheckout( gitBranch )
159 gitPullResult = main.ONOSbench.gitPull()
160 # values of 1 or 3 are good
161 utilities.assert_lesser( expect=0, actual=gitPullResult,
162 onpass="Git pull successful",
163 onfail="Git pull failed" )
164 main.ONOSbench.getVersion( report=True )
165
166 main.step( "Using mvn clean install" )
167 cleanInstallResult = main.TRUE
168 if PULLCODE and gitPullResult == main.TRUE:
169 cleanInstallResult = main.ONOSbench.cleanInstall()
170 else:
171 main.log.warn( "Did not pull new code so skipping mvn " +
172 "clean install" )
173 utilities.assert_equals( expect=main.TRUE,
174 actual=cleanInstallResult,
175 onpass="MCI successful",
176 onfail="MCI failed" )
177 # GRAPHS
178 # NOTE: important params here:
179 # job = name of Jenkins job
180 # Plot Name = Plot-HA, only can be used if multiple plots
181 # index = The number of the graph under plot name
182 job = "HAsanity"
183 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700184 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700185 graphs = '<ac:structured-macro ac:name="html">\n'
186 graphs += '<ac:plain-text-body><![CDATA[\n'
187 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800188 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700189 '&width=500&height=300"' +\
190 'noborder="0" width="500" height="300" scrolling="yes" ' +\
191 'seamless="seamless"></iframe>\n'
192 graphs += ']]></ac:plain-text-body>\n'
193 graphs += '</ac:structured-macro>\n'
194 main.log.wiki(graphs)
195
196 main.step( "Creating ONOS package" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700197 packageResult = main.ONOSbench.buckBuild()
Jon Hall5cf14d52015-07-16 12:15:19 -0700198 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
199 onpass="ONOS package successful",
200 onfail="ONOS package failed" )
201
202 main.step( "Installing ONOS package" )
203 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700204 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700205 tmpResult = main.ONOSbench.onosInstall( options="-f",
206 node=node.ip_address )
207 onosInstallResult = onosInstallResult and tmpResult
208 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
209 onpass="ONOS install successful",
210 onfail="ONOS install failed" )
211
You Wangf5de25b2017-01-06 15:13:01 -0800212 main.step( "Set up ONOS secure SSH" )
213 secureSshResult = main.TRUE
214 for node in main.nodes:
215 secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=node.ip_address )
216 utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
217 onpass="Test step PASS",
218 onfail="Test step FAIL" )
219
Jon Hall5cf14d52015-07-16 12:15:19 -0700220 main.step( "Checking if ONOS is up yet" )
221 for i in range( 2 ):
222 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700223 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700224 started = main.ONOSbench.isup( node.ip_address )
225 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800226 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700227 onosIsupResult = onosIsupResult and started
228 if onosIsupResult == main.TRUE:
229 break
230 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
231 onpass="ONOS startup successful",
232 onfail="ONOS startup failed" )
233
Jon Hall6509dbf2016-06-21 17:01:17 -0700234 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700235 cliResults = main.TRUE
236 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700237 for i in range( main.numCtrls ):
238 t = main.Thread( target=main.CLIs[i].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700239 name="startOnosCli-" + str( i ),
Jon Halle1a3b752015-07-22 13:02:46 -0700240 args=[main.nodes[i].ip_address] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700241 threads.append( t )
242 t.start()
243
244 for t in threads:
245 t.join()
246 cliResults = cliResults and t.result
247 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
248 onpass="ONOS cli startup successful",
249 onfail="ONOS cli startup failed" )
250
Jon Halla440e872016-03-31 15:15:50 -0700251 # Create a list of active nodes for use when some nodes are stopped
252 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
253
Jon Hall5cf14d52015-07-16 12:15:19 -0700254 if main.params[ 'tcpdump' ].lower() == "true":
255 main.step( "Start Packet Capture MN" )
256 main.Mininet2.startTcpdump(
257 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
258 + "-MN.pcap",
259 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
260 port=main.params[ 'MNtcpdump' ][ 'port' ] )
261
Jon Halla440e872016-03-31 15:15:50 -0700262 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700263 nodeResults = utilities.retry( main.HA.nodesCheck,
264 False,
265 args=[main.activeNodes],
266 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700267
Jon Hall41d39f12016-04-11 22:54:35 -0700268 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700269 onpass="Nodes check successful",
270 onfail="Nodes check NOT successful" )
271
272 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700273 for i in main.activeNodes:
274 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700275 main.log.debug( "{} components not ACTIVE: \n{}".format(
276 cli.name,
277 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700278 main.log.error( "Failed to start ONOS, stopping test" )
279 main.cleanup()
280 main.exit()
281
Jon Hall172b7ba2016-04-07 18:12:20 -0700282 main.step( "Activate apps defined in the params file" )
283 # get data from the params
284 apps = main.params.get( 'apps' )
285 if apps:
286 apps = apps.split(',')
287 main.log.warn( apps )
288 activateResult = True
289 for app in apps:
290 main.CLIs[ 0 ].app( app, "Activate" )
291 # TODO: check this worked
292 time.sleep( 10 ) # wait for apps to activate
293 for app in apps:
294 state = main.CLIs[ 0 ].appStatus( app )
295 if state == "ACTIVE":
Jon Hall937bc812017-01-31 16:44:10 -0800296 activateResult = activateResult and True
Jon Hall172b7ba2016-04-07 18:12:20 -0700297 else:
298 main.log.error( "{} is in {} state".format( app, state ) )
Jon Hall937bc812017-01-31 16:44:10 -0800299 activateResult = False
Jon Hall172b7ba2016-04-07 18:12:20 -0700300 utilities.assert_equals( expect=True,
301 actual=activateResult,
302 onpass="Successfully activated apps",
303 onfail="Failed to activate apps" )
304 else:
305 main.log.warn( "No apps were specified to be loaded after startup" )
306
307 main.step( "Set ONOS configurations" )
308 config = main.params.get( 'ONOS_Configuration' )
309 if config:
310 main.log.debug( config )
311 checkResult = main.TRUE
312 for component in config:
313 for setting in config[component]:
314 value = config[component][setting]
315 check = main.CLIs[ 0 ].setCfg( component, setting, value )
316 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
317 checkResult = check and checkResult
318 utilities.assert_equals( expect=main.TRUE,
319 actual=checkResult,
320 onpass="Successfully set config",
321 onfail="Failed to set config" )
322 else:
323 main.log.warn( "No configurations were specified to be changed after startup" )
324
Jon Hall9d2dcad2016-04-08 10:15:20 -0700325 main.step( "App Ids check" )
326 appCheck = main.TRUE
327 threads = []
328 for i in main.activeNodes:
329 t = main.Thread( target=main.CLIs[i].appToIDCheck,
330 name="appToIDCheck-" + str( i ),
331 args=[] )
332 threads.append( t )
333 t.start()
334
335 for t in threads:
336 t.join()
337 appCheck = appCheck and t.result
338 if appCheck != main.TRUE:
339 node = main.activeNodes[0]
340 main.log.warn( main.CLIs[node].apps() )
341 main.log.warn( main.CLIs[node].appIDs() )
342 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
343 onpass="App Ids seem to be correct",
344 onfail="Something is wrong with app Ids" )
345
Jon Hall5cf14d52015-07-16 12:15:19 -0700346 def CASE2( self, main ):
347 """
348 Assign devices to controllers
349 """
350 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700351 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700352 assert main, "main not defined"
353 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700354 assert main.CLIs, "main.CLIs not defined"
355 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700356 assert ONOS1Port, "ONOS1Port not defined"
357 assert ONOS2Port, "ONOS2Port not defined"
358 assert ONOS3Port, "ONOS3Port not defined"
359 assert ONOS4Port, "ONOS4Port not defined"
360 assert ONOS5Port, "ONOS5Port not defined"
361 assert ONOS6Port, "ONOS6Port not defined"
362 assert ONOS7Port, "ONOS7Port not defined"
363
364 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700365 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700366 "and check that an ONOS node becomes the " +\
367 "master of the device."
368 main.step( "Assign switches to controllers" )
369
370 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700371 for i in range( main.numCtrls ):
372 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700373 swList = []
374 for i in range( 1, 29 ):
375 swList.append( "s" + str( i ) )
376 main.Mininet1.assignSwController( sw=swList, ip=ipList )
377
378 mastershipCheck = main.TRUE
379 for i in range( 1, 29 ):
380 response = main.Mininet1.getSwController( "s" + str( i ) )
381 try:
382 main.log.info( str( response ) )
383 except Exception:
384 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700385 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700386 if re.search( "tcp:" + node.ip_address, response ):
387 mastershipCheck = mastershipCheck and main.TRUE
388 else:
389 main.log.error( "Error, node " + node.ip_address + " is " +
390 "not in the list of controllers s" +
391 str( i ) + " is connecting to." )
392 mastershipCheck = main.FALSE
393 utilities.assert_equals(
394 expect=main.TRUE,
395 actual=mastershipCheck,
396 onpass="Switch mastership assigned correctly",
397 onfail="Switches not assigned correctly to controllers" )
398
399 def CASE21( self, main ):
400 """
401 Assign mastership to controllers
402 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700403 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700404 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700405 assert main, "main not defined"
406 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700407 assert main.CLIs, "main.CLIs not defined"
408 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700409 assert ONOS1Port, "ONOS1Port not defined"
410 assert ONOS2Port, "ONOS2Port not defined"
411 assert ONOS3Port, "ONOS3Port not defined"
412 assert ONOS4Port, "ONOS4Port not defined"
413 assert ONOS5Port, "ONOS5Port not defined"
414 assert ONOS6Port, "ONOS6Port not defined"
415 assert ONOS7Port, "ONOS7Port not defined"
416
417 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700418 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700419 "device. Then manually assign" +\
420 " mastership to specific ONOS nodes using" +\
421 " 'device-role'"
422 main.step( "Assign mastership of switches to specific controllers" )
423 # Manually assign mastership to the controller we want
424 roleCall = main.TRUE
425
426 ipList = [ ]
427 deviceList = []
Jon Halla440e872016-03-31 15:15:50 -0700428 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700429 try:
430 # Assign mastership to specific controllers. This assignment was
431 # determined for a 7 node cluser, but will work with any sized
432 # cluster
433 for i in range( 1, 29 ): # switches 1 through 28
434 # set up correct variables:
435 if i == 1:
436 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700437 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700438 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700439 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700440 c = 1 % main.numCtrls
441 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700442 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700443 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700444 c = 1 % main.numCtrls
445 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700446 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700447 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700448 c = 3 % main.numCtrls
449 ip = main.nodes[ c ].ip_address # ONOS4
Jon Halla440e872016-03-31 15:15:50 -0700450 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700451 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700452 c = 2 % main.numCtrls
453 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700454 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700455 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700456 c = 2 % main.numCtrls
457 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700458 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700460 c = 5 % main.numCtrls
461 ip = main.nodes[ c ].ip_address # ONOS6
Jon Halla440e872016-03-31 15:15:50 -0700462 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700464 c = 4 % main.numCtrls
465 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700466 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700467 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700468 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700469 c = 6 % main.numCtrls
470 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700471 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700472 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700473 elif i == 28:
474 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700475 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700476 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700477 else:
478 main.log.error( "You didn't write an else statement for " +
479 "switch s" + str( i ) )
480 roleCall = main.FALSE
481 # Assign switch
482 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
483 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700484 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700485 ipList.append( ip )
486 deviceList.append( deviceId )
487 except ( AttributeError, AssertionError ):
488 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700489 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700490 utilities.assert_equals(
491 expect=main.TRUE,
492 actual=roleCall,
493 onpass="Re-assigned switch mastership to designated controller",
494 onfail="Something wrong with deviceRole calls" )
495
496 main.step( "Check mastership was correctly assigned" )
497 roleCheck = main.TRUE
498 # NOTE: This is due to the fact that device mastership change is not
499 # atomic and is actually a multi step process
500 time.sleep( 5 )
501 for i in range( len( ipList ) ):
502 ip = ipList[i]
503 deviceId = deviceList[i]
504 # Check assignment
Jon Halla440e872016-03-31 15:15:50 -0700505 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700506 if ip in master:
507 roleCheck = roleCheck and main.TRUE
508 else:
509 roleCheck = roleCheck and main.FALSE
510 main.log.error( "Error, controller " + ip + " is not" +
511 " master " + "of device " +
512 str( deviceId ) + ". Master is " +
513 repr( master ) + "." )
514 utilities.assert_equals(
515 expect=main.TRUE,
516 actual=roleCheck,
517 onpass="Switches were successfully reassigned to designated " +
518 "controller",
519 onfail="Switches were not successfully reassigned" )
520
521 def CASE3( self, main ):
522 """
523 Assign intents
524 """
525 import time
526 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700527 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700528 assert main, "main not defined"
529 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700530 assert main.CLIs, "main.CLIs not defined"
531 assert main.nodes, "main.nodes not defined"
Jon Halla440e872016-03-31 15:15:50 -0700532 try:
533 labels
534 except NameError:
535 main.log.error( "labels not defined, setting to []" )
536 labels = []
537 try:
538 data
539 except NameError:
540 main.log.error( "data not defined, setting to []" )
541 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700542 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700543 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700544 "assign predetermined host-to-host intents." +\
545 " After installation, check that the intent" +\
546 " is distributed to all nodes and the state" +\
547 " is INSTALLED"
548
549 # install onos-app-fwd
550 main.step( "Install reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700551 onosCli = main.CLIs[ main.activeNodes[0] ]
552 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700553 utilities.assert_equals( expect=main.TRUE, actual=installResults,
554 onpass="Install fwd successful",
555 onfail="Install fwd failed" )
556
557 main.step( "Check app ids" )
558 appCheck = main.TRUE
559 threads = []
Jon Halla440e872016-03-31 15:15:50 -0700560 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700561 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700562 name="appToIDCheck-" + str( i ),
563 args=[] )
564 threads.append( t )
565 t.start()
566
567 for t in threads:
568 t.join()
569 appCheck = appCheck and t.result
570 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700571 main.log.warn( onosCli.apps() )
572 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700573 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
574 onpass="App Ids seem to be correct",
575 onfail="Something is wrong with app Ids" )
576
577 main.step( "Discovering Hosts( Via pingall for now )" )
578 # FIXME: Once we have a host discovery mechanism, use that instead
579 # REACTIVE FWD test
580 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700581 passMsg = "Reactive Pingall test passed"
582 time1 = time.time()
583 pingResult = main.Mininet1.pingall()
584 time2 = time.time()
585 if not pingResult:
586 main.log.warn("First pingall failed. Trying again...")
Jon Hall5cf14d52015-07-16 12:15:19 -0700587 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700588 passMsg += " on the second try"
589 utilities.assert_equals(
590 expect=main.TRUE,
591 actual=pingResult,
592 onpass= passMsg,
593 onfail="Reactive Pingall failed, " +
594 "one or more ping pairs failed" )
595 main.log.info( "Time for pingall: %2f seconds" %
596 ( time2 - time1 ) )
Jon Halld2871c22016-07-26 11:01:14 -0700597 if not pingResult:
598 main.cleanup()
599 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700600 # timeout for fwd flows
601 time.sleep( 11 )
602 # uninstall onos-app-fwd
603 main.step( "Uninstall reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700604 node = main.activeNodes[0]
605 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700606 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
607 onpass="Uninstall fwd successful",
608 onfail="Uninstall fwd failed" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700609
610 main.step( "Check app ids" )
611 threads = []
612 appCheck2 = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700613 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -0700614 t = main.Thread( target=main.CLIs[i].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700615 name="appToIDCheck-" + str( i ),
616 args=[] )
617 threads.append( t )
618 t.start()
619
620 for t in threads:
621 t.join()
622 appCheck2 = appCheck2 and t.result
623 if appCheck2 != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700624 node = main.activeNodes[0]
625 main.log.warn( main.CLIs[node].apps() )
626 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700627 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
628 onpass="App Ids seem to be correct",
629 onfail="Something is wrong with app Ids" )
630
631 main.step( "Add host intents via cli" )
632 intentIds = []
Jon Hall6e709752016-02-01 13:38:46 -0800633 # TODO: move the host numbers to params
634 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700635 intentAddResult = True
636 hostResult = main.TRUE
637 for i in range( 8, 18 ):
638 main.log.info( "Adding host intent between h" + str( i ) +
639 " and h" + str( i + 10 ) )
640 host1 = "00:00:00:00:00:" + \
641 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
642 host2 = "00:00:00:00:00:" + \
643 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
644 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700645 host1Dict = onosCli.getHost( host1 )
646 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700647 host1Id = None
648 host2Id = None
649 if host1Dict and host2Dict:
650 host1Id = host1Dict.get( 'id', None )
651 host2Id = host2Dict.get( 'id', None )
652 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700653 nodeNum = ( i % len( main.activeNodes ) )
654 node = main.activeNodes[nodeNum]
655 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700656 if tmpId:
657 main.log.info( "Added intent with id: " + tmpId )
658 intentIds.append( tmpId )
659 else:
660 main.log.error( "addHostIntent returned: " +
661 repr( tmpId ) )
662 else:
663 main.log.error( "Error, getHost() failed for h" + str( i ) +
664 " and/or h" + str( i + 10 ) )
Jon Halla440e872016-03-31 15:15:50 -0700665 node = main.activeNodes[0]
666 hosts = main.CLIs[node].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700667 main.log.warn( "Hosts output: " )
668 try:
669 main.log.warn( json.dumps( json.loads( hosts ),
670 sort_keys=True,
671 indent=4,
672 separators=( ',', ': ' ) ) )
673 except ( ValueError, TypeError ):
674 main.log.warn( repr( hosts ) )
675 hostResult = main.FALSE
676 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
677 onpass="Found a host id for each host",
678 onfail="Error looking up host ids" )
679
680 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700681 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700682 main.log.info( "Submitted intents: " + str( intentIds ) )
683 main.log.info( "Intents in ONOS: " + str( onosIds ) )
684 for intent in intentIds:
685 if intent in onosIds:
686 pass # intent submitted is in onos
687 else:
688 intentAddResult = False
689 if intentAddResult:
690 intentStop = time.time()
691 else:
692 intentStop = None
693 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700694 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700695 intentStates = []
696 installedCheck = True
697 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
698 count = 0
699 try:
700 for intent in json.loads( intents ):
701 state = intent.get( 'state', None )
702 if "INSTALLED" not in state:
703 installedCheck = False
704 intentId = intent.get( 'id', None )
705 intentStates.append( ( intentId, state ) )
706 except ( ValueError, TypeError ):
707 main.log.exception( "Error parsing intents" )
708 # add submitted intents not in the store
709 tmplist = [ i for i, s in intentStates ]
710 missingIntents = False
711 for i in intentIds:
712 if i not in tmplist:
713 intentStates.append( ( i, " - " ) )
714 missingIntents = True
715 intentStates.sort()
716 for i, s in intentStates:
717 count += 1
718 main.log.info( "%-6s%-15s%-15s" %
719 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700720 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700721 try:
722 missing = False
723 if leaders:
724 parsedLeaders = json.loads( leaders )
725 main.log.warn( json.dumps( parsedLeaders,
726 sort_keys=True,
727 indent=4,
728 separators=( ',', ': ' ) ) )
729 # check for all intent partitions
730 topics = []
731 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700732 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700733 main.log.debug( topics )
734 ONOStopics = [ j['topic'] for j in parsedLeaders ]
735 for topic in topics:
736 if topic not in ONOStopics:
737 main.log.error( "Error: " + topic +
738 " not in leaders" )
739 missing = True
740 else:
741 main.log.error( "leaders() returned None" )
742 except ( ValueError, TypeError ):
743 main.log.exception( "Error parsing leaders" )
744 main.log.error( repr( leaders ) )
745 # Check all nodes
746 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700747 for i in main.activeNodes:
748 response = main.CLIs[i].leaders( jsonFormat=False)
749 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700750 str( response ) )
751
Jon Halla440e872016-03-31 15:15:50 -0700752 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700753 try:
754 if partitions :
755 parsedPartitions = json.loads( partitions )
756 main.log.warn( json.dumps( parsedPartitions,
757 sort_keys=True,
758 indent=4,
759 separators=( ',', ': ' ) ) )
760 # TODO check for a leader in all paritions
761 # TODO check for consistency among nodes
762 else:
763 main.log.error( "partitions() returned None" )
764 except ( ValueError, TypeError ):
765 main.log.exception( "Error parsing partitions" )
766 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700767 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700768 try:
769 if pendingMap :
770 parsedPending = json.loads( pendingMap )
771 main.log.warn( json.dumps( parsedPending,
772 sort_keys=True,
773 indent=4,
774 separators=( ',', ': ' ) ) )
775 # TODO check something here?
776 else:
777 main.log.error( "pendingMap() returned None" )
778 except ( ValueError, TypeError ):
779 main.log.exception( "Error parsing pending map" )
780 main.log.error( repr( pendingMap ) )
781
782 intentAddResult = bool( intentAddResult and not missingIntents and
783 installedCheck )
784 if not intentAddResult:
785 main.log.error( "Error in pushing host intents to ONOS" )
786
787 main.step( "Intent Anti-Entropy dispersion" )
Jon Halla440e872016-03-31 15:15:50 -0700788 for j in range(100):
Jon Hall5cf14d52015-07-16 12:15:19 -0700789 correct = True
790 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700791 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700792 onosIds = []
Jon Halla440e872016-03-31 15:15:50 -0700793 ids = main.CLIs[i].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700794 onosIds.append( ids )
Jon Halla440e872016-03-31 15:15:50 -0700795 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700796 str( sorted( onosIds ) ) )
797 if sorted( ids ) != sorted( intentIds ):
798 main.log.warn( "Set of intent IDs doesn't match" )
799 correct = False
800 break
801 else:
Jon Halla440e872016-03-31 15:15:50 -0700802 intents = json.loads( main.CLIs[i].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700803 for intent in intents:
804 if intent[ 'state' ] != "INSTALLED":
805 main.log.warn( "Intent " + intent[ 'id' ] +
806 " is " + intent[ 'state' ] )
807 correct = False
808 break
809 if correct:
810 break
811 else:
812 time.sleep(1)
813 if not intentStop:
814 intentStop = time.time()
815 global gossipTime
816 gossipTime = intentStop - intentStart
817 main.log.info( "It took about " + str( gossipTime ) +
818 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700819 gossipPeriod = int( main.params['timers']['gossip'] )
Jon Halla440e872016-03-31 15:15:50 -0700820 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700821 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700822 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700823 onpass="ECM anti-entropy for intents worked within " +
824 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700825 onfail="Intent ECM anti-entropy took too long. " +
826 "Expected time:{}, Actual time:{}".format( maxGossipTime,
827 gossipTime ) )
828 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700829 intentAddResult = True
830
831 if not intentAddResult or "key" in pendingMap:
832 import time
833 installedCheck = True
834 main.log.info( "Sleeping 60 seconds to see if intents are found" )
835 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700836 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700837 main.log.info( "Submitted intents: " + str( intentIds ) )
838 main.log.info( "Intents in ONOS: " + str( onosIds ) )
839 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700840 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700841 intentStates = []
842 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
843 count = 0
844 try:
845 for intent in json.loads( intents ):
846 # Iter through intents of a node
847 state = intent.get( 'state', None )
848 if "INSTALLED" not in state:
849 installedCheck = False
850 intentId = intent.get( 'id', None )
851 intentStates.append( ( intentId, state ) )
852 except ( ValueError, TypeError ):
853 main.log.exception( "Error parsing intents" )
854 # add submitted intents not in the store
855 tmplist = [ i for i, s in intentStates ]
856 for i in intentIds:
857 if i not in tmplist:
858 intentStates.append( ( i, " - " ) )
859 intentStates.sort()
860 for i, s in intentStates:
861 count += 1
862 main.log.info( "%-6s%-15s%-15s" %
863 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700864 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700865 try:
866 missing = False
867 if leaders:
868 parsedLeaders = json.loads( leaders )
869 main.log.warn( json.dumps( parsedLeaders,
870 sort_keys=True,
871 indent=4,
872 separators=( ',', ': ' ) ) )
873 # check for all intent partitions
874 # check for election
875 topics = []
876 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700877 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700878 # FIXME: this should only be after we start the app
879 topics.append( "org.onosproject.election" )
880 main.log.debug( topics )
881 ONOStopics = [ j['topic'] for j in parsedLeaders ]
882 for topic in topics:
883 if topic not in ONOStopics:
884 main.log.error( "Error: " + topic +
885 " not in leaders" )
886 missing = True
887 else:
888 main.log.error( "leaders() returned None" )
889 except ( ValueError, TypeError ):
890 main.log.exception( "Error parsing leaders" )
891 main.log.error( repr( leaders ) )
892 # Check all nodes
893 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700894 for i in main.activeNodes:
895 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -0700896 response = node.leaders( jsonFormat=False)
897 main.log.warn( str( node.name ) + " leaders output: \n" +
898 str( response ) )
899
Jon Halla440e872016-03-31 15:15:50 -0700900 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700901 try:
902 if partitions :
903 parsedPartitions = json.loads( partitions )
904 main.log.warn( json.dumps( parsedPartitions,
905 sort_keys=True,
906 indent=4,
907 separators=( ',', ': ' ) ) )
908 # TODO check for a leader in all paritions
909 # TODO check for consistency among nodes
910 else:
911 main.log.error( "partitions() returned None" )
912 except ( ValueError, TypeError ):
913 main.log.exception( "Error parsing partitions" )
914 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700915 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700916 try:
917 if pendingMap :
918 parsedPending = json.loads( pendingMap )
919 main.log.warn( json.dumps( parsedPending,
920 sort_keys=True,
921 indent=4,
922 separators=( ',', ': ' ) ) )
923 # TODO check something here?
924 else:
925 main.log.error( "pendingMap() returned None" )
926 except ( ValueError, TypeError ):
927 main.log.exception( "Error parsing pending map" )
928 main.log.error( repr( pendingMap ) )
929
930 def CASE4( self, main ):
931 """
932 Ping across added host intents
933 """
934 import json
935 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700936 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700937 assert main, "main not defined"
938 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700939 assert main.CLIs, "main.CLIs not defined"
940 assert main.nodes, "main.nodes not defined"
Jon Halla440e872016-03-31 15:15:50 -0700941 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700942 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700943 "functionality and check the state of " +\
944 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700945
Jon Hall41d39f12016-04-11 22:54:35 -0700946 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700947 main.step( "Check Intent state" )
948 installedCheck = False
949 loopCount = 0
950 while not installedCheck and loopCount < 40:
951 installedCheck = True
952 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700953 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700954 intentStates = []
955 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
956 count = 0
957 # Iter through intents of a node
958 try:
959 for intent in json.loads( intents ):
960 state = intent.get( 'state', None )
961 if "INSTALLED" not in state:
962 installedCheck = False
963 intentId = intent.get( 'id', None )
964 intentStates.append( ( intentId, state ) )
965 except ( ValueError, TypeError ):
966 main.log.exception( "Error parsing intents." )
967 # Print states
968 intentStates.sort()
969 for i, s in intentStates:
970 count += 1
971 main.log.info( "%-6s%-15s%-15s" %
972 ( str( count ), str( i ), str( s ) ) )
973 if not installedCheck:
974 time.sleep( 1 )
975 loopCount += 1
976 utilities.assert_equals( expect=True, actual=installedCheck,
977 onpass="Intents are all INSTALLED",
978 onfail="Intents are not all in " +
979 "INSTALLED state" )
980
Jon Hall9d2dcad2016-04-08 10:15:20 -0700981 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700982 PingResult = main.TRUE
983 for i in range( 8, 18 ):
984 ping = main.Mininet1.pingHost( src="h" + str( i ),
985 target="h" + str( i + 10 ) )
986 PingResult = PingResult and ping
987 if ping == main.FALSE:
988 main.log.warn( "Ping failed between h" + str( i ) +
989 " and h" + str( i + 10 ) )
990 elif ping == main.TRUE:
991 main.log.info( "Ping test passed!" )
992 # Don't set PingResult or you'd override failures
993 if PingResult == main.FALSE:
994 main.log.error(
995 "Intents have not been installed correctly, pings failed." )
996 # TODO: pretty print
997 main.log.warn( "ONOS1 intents: " )
998 try:
999 tmpIntents = onosCli.intents()
1000 main.log.warn( json.dumps( json.loads( tmpIntents ),
1001 sort_keys=True,
1002 indent=4,
1003 separators=( ',', ': ' ) ) )
1004 except ( ValueError, TypeError ):
1005 main.log.warn( repr( tmpIntents ) )
1006 utilities.assert_equals(
1007 expect=main.TRUE,
1008 actual=PingResult,
1009 onpass="Intents have been installed correctly and pings work",
1010 onfail="Intents have not been installed correctly, pings failed." )
1011
Jon Hall5cf14d52015-07-16 12:15:19 -07001012 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -07001013 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001014 topicCheck = main.TRUE
1015 try:
1016 if leaders:
1017 parsedLeaders = json.loads( leaders )
1018 main.log.warn( json.dumps( parsedLeaders,
1019 sort_keys=True,
1020 indent=4,
1021 separators=( ',', ': ' ) ) )
1022 # check for all intent partitions
1023 # check for election
1024 # TODO: Look at Devices as topics now that it uses this system
1025 topics = []
1026 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001027 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001028 # FIXME: this should only be after we start the app
1029 # FIXME: topics.append( "org.onosproject.election" )
1030 # Print leaders output
1031 main.log.debug( topics )
1032 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1033 for topic in topics:
1034 if topic not in ONOStopics:
1035 main.log.error( "Error: " + topic +
1036 " not in leaders" )
1037 topicCheck = main.FALSE
1038 else:
1039 main.log.error( "leaders() returned None" )
1040 topicCheck = main.FALSE
1041 except ( ValueError, TypeError ):
1042 topicCheck = main.FALSE
1043 main.log.exception( "Error parsing leaders" )
1044 main.log.error( repr( leaders ) )
1045 # TODO: Check for a leader of these topics
1046 # Check all nodes
1047 if topicCheck:
Jon Halla440e872016-03-31 15:15:50 -07001048 for i in main.activeNodes:
1049 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001050 response = node.leaders( jsonFormat=False)
1051 main.log.warn( str( node.name ) + " leaders output: \n" +
1052 str( response ) )
1053
1054 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1055 onpass="intent Partitions is in leaders",
1056 onfail="Some topics were lost " )
1057 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -07001058 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001059 try:
1060 if partitions :
1061 parsedPartitions = json.loads( partitions )
1062 main.log.warn( json.dumps( parsedPartitions,
1063 sort_keys=True,
1064 indent=4,
1065 separators=( ',', ': ' ) ) )
1066 # TODO check for a leader in all paritions
1067 # TODO check for consistency among nodes
1068 else:
1069 main.log.error( "partitions() returned None" )
1070 except ( ValueError, TypeError ):
1071 main.log.exception( "Error parsing partitions" )
1072 main.log.error( repr( partitions ) )
1073 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001074 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001075 try:
1076 if pendingMap :
1077 parsedPending = json.loads( pendingMap )
1078 main.log.warn( json.dumps( parsedPending,
1079 sort_keys=True,
1080 indent=4,
1081 separators=( ',', ': ' ) ) )
1082 # TODO check something here?
1083 else:
1084 main.log.error( "pendingMap() returned None" )
1085 except ( ValueError, TypeError ):
1086 main.log.exception( "Error parsing pending map" )
1087 main.log.error( repr( pendingMap ) )
1088
1089 if not installedCheck:
1090 main.log.info( "Waiting 60 seconds to see if the state of " +
1091 "intents change" )
1092 time.sleep( 60 )
1093 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001094 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001095 intentStates = []
1096 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1097 count = 0
1098 # Iter through intents of a node
1099 try:
1100 for intent in json.loads( intents ):
1101 state = intent.get( 'state', None )
1102 if "INSTALLED" not in state:
1103 installedCheck = False
1104 intentId = intent.get( 'id', None )
1105 intentStates.append( ( intentId, state ) )
1106 except ( ValueError, TypeError ):
1107 main.log.exception( "Error parsing intents." )
1108 intentStates.sort()
1109 for i, s in intentStates:
1110 count += 1
1111 main.log.info( "%-6s%-15s%-15s" %
1112 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001113 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001114 try:
1115 missing = False
1116 if leaders:
1117 parsedLeaders = json.loads( leaders )
1118 main.log.warn( json.dumps( parsedLeaders,
1119 sort_keys=True,
1120 indent=4,
1121 separators=( ',', ': ' ) ) )
1122 # check for all intent partitions
1123 # check for election
1124 topics = []
1125 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001126 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001127 # FIXME: this should only be after we start the app
1128 topics.append( "org.onosproject.election" )
1129 main.log.debug( topics )
1130 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1131 for topic in topics:
1132 if topic not in ONOStopics:
1133 main.log.error( "Error: " + topic +
1134 " not in leaders" )
1135 missing = True
1136 else:
1137 main.log.error( "leaders() returned None" )
1138 except ( ValueError, TypeError ):
1139 main.log.exception( "Error parsing leaders" )
1140 main.log.error( repr( leaders ) )
1141 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001142 for i in main.activeNodes:
1143 node = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07001144 response = node.leaders( jsonFormat=False)
1145 main.log.warn( str( node.name ) + " leaders output: \n" +
1146 str( response ) )
1147
Jon Halla440e872016-03-31 15:15:50 -07001148 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001149 try:
1150 if partitions :
1151 parsedPartitions = json.loads( partitions )
1152 main.log.warn( json.dumps( parsedPartitions,
1153 sort_keys=True,
1154 indent=4,
1155 separators=( ',', ': ' ) ) )
1156 # TODO check for a leader in all paritions
1157 # TODO check for consistency among nodes
1158 else:
1159 main.log.error( "partitions() returned None" )
1160 except ( ValueError, TypeError ):
1161 main.log.exception( "Error parsing partitions" )
1162 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001163 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001164 try:
1165 if pendingMap :
1166 parsedPending = json.loads( pendingMap )
1167 main.log.warn( json.dumps( parsedPending,
1168 sort_keys=True,
1169 indent=4,
1170 separators=( ',', ': ' ) ) )
1171 # TODO check something here?
1172 else:
1173 main.log.error( "pendingMap() returned None" )
1174 except ( ValueError, TypeError ):
1175 main.log.exception( "Error parsing pending map" )
1176 main.log.error( repr( pendingMap ) )
1177 # Print flowrules
Jon Halla440e872016-03-31 15:15:50 -07001178 node = main.activeNodes[0]
1179 main.log.debug( main.CLIs[node].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001180 main.step( "Wait a minute then ping again" )
1181 # the wait is above
1182 PingResult = main.TRUE
1183 for i in range( 8, 18 ):
1184 ping = main.Mininet1.pingHost( src="h" + str( i ),
1185 target="h" + str( i + 10 ) )
1186 PingResult = PingResult and ping
1187 if ping == main.FALSE:
1188 main.log.warn( "Ping failed between h" + str( i ) +
1189 " and h" + str( i + 10 ) )
1190 elif ping == main.TRUE:
1191 main.log.info( "Ping test passed!" )
1192 # Don't set PingResult or you'd override failures
1193 if PingResult == main.FALSE:
1194 main.log.error(
1195 "Intents have not been installed correctly, pings failed." )
1196 # TODO: pretty print
1197 main.log.warn( "ONOS1 intents: " )
1198 try:
Jon Halla440e872016-03-31 15:15:50 -07001199 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001200 main.log.warn( json.dumps( json.loads( tmpIntents ),
1201 sort_keys=True,
1202 indent=4,
1203 separators=( ',', ': ' ) ) )
1204 except ( ValueError, TypeError ):
1205 main.log.warn( repr( tmpIntents ) )
1206 utilities.assert_equals(
1207 expect=main.TRUE,
1208 actual=PingResult,
1209 onpass="Intents have been installed correctly and pings work",
1210 onfail="Intents have not been installed correctly, pings failed." )
1211
1212 def CASE5( self, main ):
1213 """
1214 Reading state of ONOS
1215 """
1216 import json
1217 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001218 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001219 assert main, "main not defined"
1220 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001221 assert main.CLIs, "main.CLIs not defined"
1222 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001223
1224 main.case( "Setting up and gathering data for current state" )
1225 # The general idea for this test case is to pull the state of
1226 # ( intents,flows, topology,... ) from each ONOS node
1227 # We can then compare them with each other and also with past states
1228
1229 main.step( "Check that each switch has a master" )
1230 global mastershipState
1231 mastershipState = '[]'
1232
1233 # Assert that each device has a master
1234 rolesNotNull = main.TRUE
1235 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001236 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001237 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001238 name="rolesNotNull-" + str( i ),
1239 args=[] )
1240 threads.append( t )
1241 t.start()
1242
1243 for t in threads:
1244 t.join()
1245 rolesNotNull = rolesNotNull and t.result
1246 utilities.assert_equals(
1247 expect=main.TRUE,
1248 actual=rolesNotNull,
1249 onpass="Each device has a master",
1250 onfail="Some devices don't have a master assigned" )
1251
1252 main.step( "Get the Mastership of each switch from each controller" )
1253 ONOSMastership = []
1254 mastershipCheck = main.FALSE
1255 consistentMastership = True
1256 rolesResults = True
1257 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001258 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001259 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001260 name="roles-" + str( i ),
1261 args=[] )
1262 threads.append( t )
1263 t.start()
1264
1265 for t in threads:
1266 t.join()
1267 ONOSMastership.append( t.result )
1268
Jon Halla440e872016-03-31 15:15:50 -07001269 for i in range( len( ONOSMastership ) ):
1270 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001271 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001272 main.log.error( "Error in getting ONOS" + node + " roles" )
1273 main.log.warn( "ONOS" + node + " mastership response: " +
1274 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001275 rolesResults = False
1276 utilities.assert_equals(
1277 expect=True,
1278 actual=rolesResults,
1279 onpass="No error in reading roles output",
1280 onfail="Error in reading roles from ONOS" )
1281
1282 main.step( "Check for consistency in roles from each controller" )
1283 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1284 main.log.info(
1285 "Switch roles are consistent across all ONOS nodes" )
1286 else:
1287 consistentMastership = False
1288 utilities.assert_equals(
1289 expect=True,
1290 actual=consistentMastership,
1291 onpass="Switch roles are consistent across all ONOS nodes",
1292 onfail="ONOS nodes have different views of switch roles" )
1293
1294 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001295 for i in range( len( main.activeNodes ) ):
1296 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001297 try:
1298 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001299 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001300 json.dumps(
1301 json.loads( ONOSMastership[ i ] ),
1302 sort_keys=True,
1303 indent=4,
1304 separators=( ',', ': ' ) ) )
1305 except ( ValueError, TypeError ):
1306 main.log.warn( repr( ONOSMastership[ i ] ) )
1307 elif rolesResults and consistentMastership:
1308 mastershipCheck = main.TRUE
1309 mastershipState = ONOSMastership[ 0 ]
1310
1311 main.step( "Get the intents from each controller" )
1312 global intentState
1313 intentState = []
1314 ONOSIntents = []
1315 intentCheck = main.FALSE
1316 consistentIntents = True
1317 intentsResults = True
1318 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001319 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001320 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001321 name="intents-" + str( i ),
1322 args=[],
1323 kwargs={ 'jsonFormat': True } )
1324 threads.append( t )
1325 t.start()
1326
1327 for t in threads:
1328 t.join()
1329 ONOSIntents.append( t.result )
1330
Jon Halla440e872016-03-31 15:15:50 -07001331 for i in range( len( ONOSIntents ) ):
1332 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001333 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07001334 main.log.error( "Error in getting ONOS" + node + " intents" )
1335 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001336 repr( ONOSIntents[ i ] ) )
1337 intentsResults = False
1338 utilities.assert_equals(
1339 expect=True,
1340 actual=intentsResults,
1341 onpass="No error in reading intents output",
1342 onfail="Error in reading intents from ONOS" )
1343
1344 main.step( "Check for consistency in Intents from each controller" )
1345 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1346 main.log.info( "Intents are consistent across all ONOS " +
1347 "nodes" )
1348 else:
1349 consistentIntents = False
1350 main.log.error( "Intents not consistent" )
1351 utilities.assert_equals(
1352 expect=True,
1353 actual=consistentIntents,
1354 onpass="Intents are consistent across all ONOS nodes",
1355 onfail="ONOS nodes have different views of intents" )
1356
1357 if intentsResults:
1358 # Try to make it easy to figure out what is happening
1359 #
1360 # Intent ONOS1 ONOS2 ...
1361 # 0x01 INSTALLED INSTALLING
1362 # ... ... ...
1363 # ... ... ...
1364 title = " Id"
Jon Halla440e872016-03-31 15:15:50 -07001365 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001366 title += " " * 10 + "ONOS" + str( n + 1 )
1367 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001368 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001369 keys = []
1370 try:
1371 # Get the set of all intent keys
1372 for nodeStr in ONOSIntents:
1373 node = json.loads( nodeStr )
1374 for intent in node:
1375 keys.append( intent.get( 'id' ) )
1376 keys = set( keys )
1377 # For each intent key, print the state on each node
1378 for key in keys:
1379 row = "%-13s" % key
1380 for nodeStr in ONOSIntents:
1381 node = json.loads( nodeStr )
1382 for intent in node:
1383 if intent.get( 'id', "Error" ) == key:
1384 row += "%-15s" % intent.get( 'state' )
1385 main.log.warn( row )
1386 # End of intent state table
1387 except ValueError as e:
1388 main.log.exception( e )
1389 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1390
1391 if intentsResults and not consistentIntents:
1392 # print the json objects
Jon Halla440e872016-03-31 15:15:50 -07001393 n = str( main.activeNodes[-1] + 1 )
1394 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001395 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1396 sort_keys=True,
1397 indent=4,
1398 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001399 for i in range( len( ONOSIntents ) ):
1400 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001401 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07001402 main.log.debug( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001403 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ),
1404 sort_keys=True,
1405 indent=4,
1406 separators=( ',', ': ' ) ) )
1407 else:
Jon Halla440e872016-03-31 15:15:50 -07001408 main.log.debug( "ONOS" + node + " intents match ONOS" +
1409 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001410 elif intentsResults and consistentIntents:
1411 intentCheck = main.TRUE
1412 intentState = ONOSIntents[ 0 ]
1413
1414 main.step( "Get the flows from each controller" )
1415 global flowState
1416 flowState = []
1417 ONOSFlows = []
1418 ONOSFlowsJson = []
1419 flowCheck = main.FALSE
1420 consistentFlows = True
1421 flowsResults = True
1422 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001423 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001424 t = main.Thread( target=main.CLIs[i].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001425 name="flows-" + str( i ),
1426 args=[],
1427 kwargs={ 'jsonFormat': True } )
1428 threads.append( t )
1429 t.start()
1430
1431 # NOTE: Flows command can take some time to run
1432 time.sleep(30)
1433 for t in threads:
1434 t.join()
1435 result = t.result
1436 ONOSFlows.append( result )
1437
Jon Halla440e872016-03-31 15:15:50 -07001438 for i in range( len( ONOSFlows ) ):
1439 num = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001440 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1441 main.log.error( "Error in getting ONOS" + num + " flows" )
1442 main.log.warn( "ONOS" + num + " flows response: " +
1443 repr( ONOSFlows[ i ] ) )
1444 flowsResults = False
1445 ONOSFlowsJson.append( None )
1446 else:
1447 try:
1448 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1449 except ( ValueError, TypeError ):
1450 # FIXME: change this to log.error?
1451 main.log.exception( "Error in parsing ONOS" + num +
1452 " response as json." )
1453 main.log.error( repr( ONOSFlows[ i ] ) )
1454 ONOSFlowsJson.append( None )
1455 flowsResults = False
1456 utilities.assert_equals(
1457 expect=True,
1458 actual=flowsResults,
1459 onpass="No error in reading flows output",
1460 onfail="Error in reading flows from ONOS" )
1461
1462 main.step( "Check for consistency in Flows from each controller" )
1463 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1464 if all( tmp ):
1465 main.log.info( "Flow count is consistent across all ONOS nodes" )
1466 else:
1467 consistentFlows = False
1468 utilities.assert_equals(
1469 expect=True,
1470 actual=consistentFlows,
1471 onpass="The flow count is consistent across all ONOS nodes",
1472 onfail="ONOS nodes have different flow counts" )
1473
1474 if flowsResults and not consistentFlows:
Jon Halla440e872016-03-31 15:15:50 -07001475 for i in range( len( ONOSFlows ) ):
1476 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001477 try:
1478 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001479 "ONOS" + node + " flows: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001480 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1481 indent=4, separators=( ',', ': ' ) ) )
1482 except ( ValueError, TypeError ):
Jon Halla440e872016-03-31 15:15:50 -07001483 main.log.warn( "ONOS" + node + " flows: " +
1484 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001485 elif flowsResults and consistentFlows:
1486 flowCheck = main.TRUE
1487 flowState = ONOSFlows[ 0 ]
1488
1489 main.step( "Get the OF Table entries" )
1490 global flows
1491 flows = []
1492 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001493 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001494 if flowCheck == main.FALSE:
1495 for table in flows:
1496 main.log.warn( table )
1497 # TODO: Compare switch flow tables with ONOS flow tables
1498
1499 main.step( "Start continuous pings" )
1500 main.Mininet2.pingLong(
1501 src=main.params[ 'PING' ][ 'source1' ],
1502 target=main.params[ 'PING' ][ 'target1' ],
1503 pingTime=500 )
1504 main.Mininet2.pingLong(
1505 src=main.params[ 'PING' ][ 'source2' ],
1506 target=main.params[ 'PING' ][ 'target2' ],
1507 pingTime=500 )
1508 main.Mininet2.pingLong(
1509 src=main.params[ 'PING' ][ 'source3' ],
1510 target=main.params[ 'PING' ][ 'target3' ],
1511 pingTime=500 )
1512 main.Mininet2.pingLong(
1513 src=main.params[ 'PING' ][ 'source4' ],
1514 target=main.params[ 'PING' ][ 'target4' ],
1515 pingTime=500 )
1516 main.Mininet2.pingLong(
1517 src=main.params[ 'PING' ][ 'source5' ],
1518 target=main.params[ 'PING' ][ 'target5' ],
1519 pingTime=500 )
1520 main.Mininet2.pingLong(
1521 src=main.params[ 'PING' ][ 'source6' ],
1522 target=main.params[ 'PING' ][ 'target6' ],
1523 pingTime=500 )
1524 main.Mininet2.pingLong(
1525 src=main.params[ 'PING' ][ 'source7' ],
1526 target=main.params[ 'PING' ][ 'target7' ],
1527 pingTime=500 )
1528 main.Mininet2.pingLong(
1529 src=main.params[ 'PING' ][ 'source8' ],
1530 target=main.params[ 'PING' ][ 'target8' ],
1531 pingTime=500 )
1532 main.Mininet2.pingLong(
1533 src=main.params[ 'PING' ][ 'source9' ],
1534 target=main.params[ 'PING' ][ 'target9' ],
1535 pingTime=500 )
1536 main.Mininet2.pingLong(
1537 src=main.params[ 'PING' ][ 'source10' ],
1538 target=main.params[ 'PING' ][ 'target10' ],
1539 pingTime=500 )
1540
1541 main.step( "Collecting topology information from ONOS" )
1542 devices = []
1543 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001544 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001545 t = main.Thread( target=main.CLIs[i].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001546 name="devices-" + str( i ),
1547 args=[ ] )
1548 threads.append( t )
1549 t.start()
1550
1551 for t in threads:
1552 t.join()
1553 devices.append( t.result )
1554 hosts = []
1555 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001556 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001557 t = main.Thread( target=main.CLIs[i].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001558 name="hosts-" + str( i ),
1559 args=[ ] )
1560 threads.append( t )
1561 t.start()
1562
1563 for t in threads:
1564 t.join()
1565 try:
1566 hosts.append( json.loads( t.result ) )
1567 except ( ValueError, TypeError ):
1568 # FIXME: better handling of this, print which node
1569 # Maybe use thread name?
1570 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001571 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001572 hosts.append( None )
1573
1574 ports = []
1575 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001576 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001577 t = main.Thread( target=main.CLIs[i].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001578 name="ports-" + str( i ),
1579 args=[ ] )
1580 threads.append( t )
1581 t.start()
1582
1583 for t in threads:
1584 t.join()
1585 ports.append( t.result )
1586 links = []
1587 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001588 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001589 t = main.Thread( target=main.CLIs[i].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001590 name="links-" + str( i ),
1591 args=[ ] )
1592 threads.append( t )
1593 t.start()
1594
1595 for t in threads:
1596 t.join()
1597 links.append( t.result )
1598 clusters = []
1599 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001600 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001601 t = main.Thread( target=main.CLIs[i].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001602 name="clusters-" + str( i ),
1603 args=[ ] )
1604 threads.append( t )
1605 t.start()
1606
1607 for t in threads:
1608 t.join()
1609 clusters.append( t.result )
1610 # Compare json objects for hosts and dataplane clusters
1611
1612 # hosts
1613 main.step( "Host view is consistent across ONOS nodes" )
1614 consistentHostsResult = main.TRUE
1615 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001616 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001617 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001618 if hosts[ controller ] == hosts[ 0 ]:
1619 continue
1620 else: # hosts not consistent
1621 main.log.error( "hosts from ONOS" +
1622 controllerStr +
1623 " is inconsistent with ONOS1" )
1624 main.log.warn( repr( hosts[ controller ] ) )
1625 consistentHostsResult = main.FALSE
1626
1627 else:
1628 main.log.error( "Error in getting ONOS hosts from ONOS" +
1629 controllerStr )
1630 consistentHostsResult = main.FALSE
1631 main.log.warn( "ONOS" + controllerStr +
1632 " hosts response: " +
1633 repr( hosts[ controller ] ) )
1634 utilities.assert_equals(
1635 expect=main.TRUE,
1636 actual=consistentHostsResult,
1637 onpass="Hosts view is consistent across all ONOS nodes",
1638 onfail="ONOS nodes have different views of hosts" )
1639
1640 main.step( "Each host has an IP address" )
1641 ipResult = main.TRUE
1642 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001643 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001644 if hosts[ controller ]:
1645 for host in hosts[ controller ]:
1646 if not host.get( 'ipAddresses', [ ] ):
1647 main.log.error( "Error with host ips on controller" +
1648 controllerStr + ": " + str( host ) )
1649 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001650 utilities.assert_equals(
1651 expect=main.TRUE,
1652 actual=ipResult,
1653 onpass="The ips of the hosts aren't empty",
1654 onfail="The ip of at least one host is missing" )
1655
1656 # Strongly connected clusters of devices
1657 main.step( "Cluster view is consistent across ONOS nodes" )
1658 consistentClustersResult = main.TRUE
1659 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07001660 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001661 if "Error" not in clusters[ controller ]:
1662 if clusters[ controller ] == clusters[ 0 ]:
1663 continue
1664 else: # clusters not consistent
1665 main.log.error( "clusters from ONOS" + controllerStr +
1666 " is inconsistent with ONOS1" )
1667 consistentClustersResult = main.FALSE
1668
1669 else:
1670 main.log.error( "Error in getting dataplane clusters " +
1671 "from ONOS" + controllerStr )
1672 consistentClustersResult = main.FALSE
1673 main.log.warn( "ONOS" + controllerStr +
1674 " clusters response: " +
1675 repr( clusters[ controller ] ) )
1676 utilities.assert_equals(
1677 expect=main.TRUE,
1678 actual=consistentClustersResult,
1679 onpass="Clusters view is consistent across all ONOS nodes",
1680 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001681 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001682 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001683
Jon Hall5cf14d52015-07-16 12:15:19 -07001684 # there should always only be one cluster
1685 main.step( "Cluster view correct across ONOS nodes" )
1686 try:
1687 numClusters = len( json.loads( clusters[ 0 ] ) )
1688 except ( ValueError, TypeError ):
1689 main.log.exception( "Error parsing clusters[0]: " +
1690 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001691 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001692 clusterResults = main.FALSE
1693 if numClusters == 1:
1694 clusterResults = main.TRUE
1695 utilities.assert_equals(
1696 expect=1,
1697 actual=numClusters,
1698 onpass="ONOS shows 1 SCC",
1699 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1700
1701 main.step( "Comparing ONOS topology to MN" )
1702 devicesResults = main.TRUE
1703 linksResults = main.TRUE
1704 hostsResults = main.TRUE
1705 mnSwitches = main.Mininet1.getSwitches()
1706 mnLinks = main.Mininet1.getLinks()
1707 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001708 for controller in main.activeNodes:
1709 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001710 if devices[ controller ] and ports[ controller ] and\
1711 "Error" not in devices[ controller ] and\
1712 "Error" not in ports[ controller ]:
Jon Halla440e872016-03-31 15:15:50 -07001713 currentDevicesResult = main.Mininet1.compareSwitches(
1714 mnSwitches,
1715 json.loads( devices[ controller ] ),
1716 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001717 else:
1718 currentDevicesResult = main.FALSE
1719 utilities.assert_equals( expect=main.TRUE,
1720 actual=currentDevicesResult,
1721 onpass="ONOS" + controllerStr +
1722 " Switches view is correct",
1723 onfail="ONOS" + controllerStr +
1724 " Switches view is incorrect" )
1725 if links[ controller ] and "Error" not in links[ controller ]:
1726 currentLinksResult = main.Mininet1.compareLinks(
1727 mnSwitches, mnLinks,
1728 json.loads( links[ controller ] ) )
1729 else:
1730 currentLinksResult = main.FALSE
1731 utilities.assert_equals( expect=main.TRUE,
1732 actual=currentLinksResult,
1733 onpass="ONOS" + controllerStr +
1734 " links view is correct",
1735 onfail="ONOS" + controllerStr +
1736 " links view is incorrect" )
1737
Jon Hall657cdf62015-12-17 14:40:51 -08001738 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001739 currentHostsResult = main.Mininet1.compareHosts(
1740 mnHosts,
1741 hosts[ controller ] )
1742 else:
1743 currentHostsResult = main.FALSE
1744 utilities.assert_equals( expect=main.TRUE,
1745 actual=currentHostsResult,
1746 onpass="ONOS" + controllerStr +
1747 " hosts exist in Mininet",
1748 onfail="ONOS" + controllerStr +
1749 " hosts don't match Mininet" )
1750
1751 devicesResults = devicesResults and currentDevicesResult
1752 linksResults = linksResults and currentLinksResult
1753 hostsResults = hostsResults and currentHostsResult
1754
1755 main.step( "Device information is correct" )
1756 utilities.assert_equals(
1757 expect=main.TRUE,
1758 actual=devicesResults,
1759 onpass="Device information is correct",
1760 onfail="Device information is incorrect" )
1761
1762 main.step( "Links are correct" )
1763 utilities.assert_equals(
1764 expect=main.TRUE,
1765 actual=linksResults,
1766 onpass="Link are correct",
1767 onfail="Links are incorrect" )
1768
1769 main.step( "Hosts are correct" )
1770 utilities.assert_equals(
1771 expect=main.TRUE,
1772 actual=hostsResults,
1773 onpass="Hosts are correct",
1774 onfail="Hosts are incorrect" )
1775
1776 def CASE6( self, main ):
1777 """
1778 The Failure case. Since this is the Sanity test, we do nothing.
1779 """
1780 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001781 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001782 assert main, "main not defined"
1783 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001784 assert main.CLIs, "main.CLIs not defined"
1785 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001786 main.case( "Wait 60 seconds instead of inducing a failure" )
1787 time.sleep( 60 )
1788 utilities.assert_equals(
1789 expect=main.TRUE,
1790 actual=main.TRUE,
1791 onpass="Sleeping 60 seconds",
1792 onfail="Something is terribly wrong with my math" )
1793
1794 def CASE7( self, main ):
1795 """
1796 Check state after ONOS failure
1797 """
1798 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001799 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001800 assert main, "main not defined"
1801 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001802 assert main.CLIs, "main.CLIs not defined"
1803 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001804 main.case( "Running ONOS Constant State Tests" )
1805
1806 main.step( "Check that each switch has a master" )
1807 # Assert that each device has a master
1808 rolesNotNull = main.TRUE
1809 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001810 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001811 t = main.Thread( target=main.CLIs[i].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001812 name="rolesNotNull-" + str( i ),
1813 args=[ ] )
1814 threads.append( t )
1815 t.start()
1816
1817 for t in threads:
1818 t.join()
1819 rolesNotNull = rolesNotNull and t.result
1820 utilities.assert_equals(
1821 expect=main.TRUE,
1822 actual=rolesNotNull,
1823 onpass="Each device has a master",
1824 onfail="Some devices don't have a master assigned" )
1825
1826 main.step( "Read device roles from ONOS" )
1827 ONOSMastership = []
1828 mastershipCheck = main.FALSE
1829 consistentMastership = True
1830 rolesResults = True
1831 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001832 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001833 t = main.Thread( target=main.CLIs[i].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001834 name="roles-" + str( i ),
1835 args=[] )
1836 threads.append( t )
1837 t.start()
1838
1839 for t in threads:
1840 t.join()
1841 ONOSMastership.append( t.result )
1842
Jon Halla440e872016-03-31 15:15:50 -07001843 for i in range( len( ONOSMastership ) ):
1844 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001845 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
Jon Halla440e872016-03-31 15:15:50 -07001846 main.log.error( "Error in getting ONOS" + node + " roles" )
1847 main.log.warn( "ONOS" + node + " mastership response: " +
1848 repr( ONOSMastership[i] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001849 rolesResults = False
1850 utilities.assert_equals(
1851 expect=True,
1852 actual=rolesResults,
1853 onpass="No error in reading roles output",
1854 onfail="Error in reading roles from ONOS" )
1855
1856 main.step( "Check for consistency in roles from each controller" )
1857 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
1858 main.log.info(
1859 "Switch roles are consistent across all ONOS nodes" )
1860 else:
1861 consistentMastership = False
1862 utilities.assert_equals(
1863 expect=True,
1864 actual=consistentMastership,
1865 onpass="Switch roles are consistent across all ONOS nodes",
1866 onfail="ONOS nodes have different views of switch roles" )
1867
1868 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001869 for i in range( len( ONOSMastership ) ):
1870 node = str( main.activeNodes[i] + 1 )
1871 main.log.warn( "ONOS" + node + " roles: ",
1872 json.dumps( json.loads( ONOSMastership[ i ] ),
1873 sort_keys=True,
1874 indent=4,
1875 separators=( ',', ': ' ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001876
1877 description2 = "Compare switch roles from before failure"
1878 main.step( description2 )
1879 try:
1880 currentJson = json.loads( ONOSMastership[0] )
1881 oldJson = json.loads( mastershipState )
1882 except ( ValueError, TypeError ):
1883 main.log.exception( "Something is wrong with parsing " +
1884 "ONOSMastership[0] or mastershipState" )
1885 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1886 main.log.error( "mastershipState" + repr( mastershipState ) )
1887 main.cleanup()
1888 main.exit()
1889 mastershipCheck = main.TRUE
1890 for i in range( 1, 29 ):
1891 switchDPID = str(
1892 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1893 current = [ switch[ 'master' ] for switch in currentJson
1894 if switchDPID in switch[ 'id' ] ]
1895 old = [ switch[ 'master' ] for switch in oldJson
1896 if switchDPID in switch[ 'id' ] ]
1897 if current == old:
1898 mastershipCheck = mastershipCheck and main.TRUE
1899 else:
1900 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1901 mastershipCheck = main.FALSE
1902 utilities.assert_equals(
1903 expect=main.TRUE,
1904 actual=mastershipCheck,
1905 onpass="Mastership of Switches was not changed",
1906 onfail="Mastership of some switches changed" )
1907 mastershipCheck = mastershipCheck and consistentMastership
1908
1909 main.step( "Get the intents and compare across all nodes" )
1910 ONOSIntents = []
1911 intentCheck = main.FALSE
1912 consistentIntents = True
1913 intentsResults = True
1914 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001915 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07001916 t = main.Thread( target=main.CLIs[i].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001917 name="intents-" + str( i ),
1918 args=[],
1919 kwargs={ 'jsonFormat': True } )
1920 threads.append( t )
1921 t.start()
1922
1923 for t in threads:
1924 t.join()
1925 ONOSIntents.append( t.result )
1926
Jon Halla440e872016-03-31 15:15:50 -07001927 for i in range( len( ONOSIntents) ):
1928 node = str( main.activeNodes[i] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001929 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07001930 main.log.error( "Error in getting ONOS" + node + " intents" )
1931 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001932 repr( ONOSIntents[ i ] ) )
1933 intentsResults = False
1934 utilities.assert_equals(
1935 expect=True,
1936 actual=intentsResults,
1937 onpass="No error in reading intents output",
1938 onfail="Error in reading intents from ONOS" )
1939
1940 main.step( "Check for consistency in Intents from each controller" )
1941 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
1942 main.log.info( "Intents are consistent across all ONOS " +
1943 "nodes" )
1944 else:
1945 consistentIntents = False
1946
1947 # Try to make it easy to figure out what is happening
1948 #
1949 # Intent ONOS1 ONOS2 ...
1950 # 0x01 INSTALLED INSTALLING
1951 # ... ... ...
1952 # ... ... ...
1953 title = " ID"
Jon Halla440e872016-03-31 15:15:50 -07001954 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001955 title += " " * 10 + "ONOS" + str( n + 1 )
1956 main.log.warn( title )
1957 # get all intent keys in the cluster
1958 keys = []
1959 for nodeStr in ONOSIntents:
1960 node = json.loads( nodeStr )
1961 for intent in node:
1962 keys.append( intent.get( 'id' ) )
1963 keys = set( keys )
1964 for key in keys:
1965 row = "%-13s" % key
1966 for nodeStr in ONOSIntents:
1967 node = json.loads( nodeStr )
1968 for intent in node:
1969 if intent.get( 'id' ) == key:
1970 row += "%-15s" % intent.get( 'state' )
1971 main.log.warn( row )
1972 # End table view
1973
1974 utilities.assert_equals(
1975 expect=True,
1976 actual=consistentIntents,
1977 onpass="Intents are consistent across all ONOS nodes",
1978 onfail="ONOS nodes have different views of intents" )
1979 intentStates = []
1980 for node in ONOSIntents: # Iter through ONOS nodes
1981 nodeStates = []
1982 # Iter through intents of a node
1983 try:
1984 for intent in json.loads( node ):
1985 nodeStates.append( intent[ 'state' ] )
1986 except ( ValueError, TypeError ):
1987 main.log.exception( "Error in parsing intents" )
1988 main.log.error( repr( node ) )
1989 intentStates.append( nodeStates )
1990 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1991 main.log.info( dict( out ) )
1992
1993 if intentsResults and not consistentIntents:
Jon Halla440e872016-03-31 15:15:50 -07001994 for i in range( len( main.activeNodes ) ):
1995 node = str( main.activeNodes[i] + 1 )
1996 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001997 main.log.warn( json.dumps(
1998 json.loads( ONOSIntents[ i ] ),
1999 sort_keys=True,
2000 indent=4,
2001 separators=( ',', ': ' ) ) )
2002 elif intentsResults and consistentIntents:
2003 intentCheck = main.TRUE
2004
2005 # NOTE: Store has no durability, so intents are lost across system
2006 # restarts
2007 main.step( "Compare current intents with intents before the failure" )
2008 # NOTE: this requires case 5 to pass for intentState to be set.
2009 # maybe we should stop the test if that fails?
2010 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002011 try:
2012 intentState
2013 except NameError:
2014 main.log.warn( "No previous intent state was saved" )
2015 else:
2016 if intentState and intentState == ONOSIntents[ 0 ]:
2017 sameIntents = main.TRUE
2018 main.log.info( "Intents are consistent with before failure" )
2019 # TODO: possibly the states have changed? we may need to figure out
2020 # what the acceptable states are
2021 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2022 sameIntents = main.TRUE
2023 try:
2024 before = json.loads( intentState )
2025 after = json.loads( ONOSIntents[ 0 ] )
2026 for intent in before:
2027 if intent not in after:
2028 sameIntents = main.FALSE
2029 main.log.debug( "Intent is not currently in ONOS " +
2030 "(at least in the same form):" )
2031 main.log.debug( json.dumps( intent ) )
2032 except ( ValueError, TypeError ):
2033 main.log.exception( "Exception printing intents" )
2034 main.log.debug( repr( ONOSIntents[0] ) )
2035 main.log.debug( repr( intentState ) )
2036 if sameIntents == main.FALSE:
2037 try:
2038 main.log.debug( "ONOS intents before: " )
2039 main.log.debug( json.dumps( json.loads( intentState ),
2040 sort_keys=True, indent=4,
2041 separators=( ',', ': ' ) ) )
2042 main.log.debug( "Current ONOS intents: " )
2043 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2044 sort_keys=True, indent=4,
2045 separators=( ',', ': ' ) ) )
2046 except ( ValueError, TypeError ):
2047 main.log.exception( "Exception printing intents" )
2048 main.log.debug( repr( ONOSIntents[0] ) )
2049 main.log.debug( repr( intentState ) )
2050 utilities.assert_equals(
2051 expect=main.TRUE,
2052 actual=sameIntents,
2053 onpass="Intents are consistent with before failure",
2054 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002055 intentCheck = intentCheck and sameIntents
2056
2057 main.step( "Get the OF Table entries and compare to before " +
2058 "component failure" )
2059 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002060 for i in range( 28 ):
2061 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002062 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07002063 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
2064 FlowTables = FlowTables and curSwitch
2065 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002066 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002067 utilities.assert_equals(
2068 expect=main.TRUE,
2069 actual=FlowTables,
2070 onpass="No changes were found in the flow tables",
2071 onfail="Changes were found in the flow tables" )
2072
2073 main.Mininet2.pingLongKill()
2074 '''
2075 main.step( "Check the continuous pings to ensure that no packets " +
2076 "were dropped during component failure" )
2077 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2078 main.params[ 'TESTONIP' ] )
2079 LossInPings = main.FALSE
2080 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2081 for i in range( 8, 18 ):
2082 main.log.info(
2083 "Checking for a loss in pings along flow from s" +
2084 str( i ) )
2085 LossInPings = main.Mininet2.checkForLoss(
2086 "/tmp/ping.h" +
2087 str( i ) ) or LossInPings
2088 if LossInPings == main.TRUE:
2089 main.log.info( "Loss in ping detected" )
2090 elif LossInPings == main.ERROR:
2091 main.log.info( "There are multiple mininet process running" )
2092 elif LossInPings == main.FALSE:
2093 main.log.info( "No Loss in the pings" )
2094 main.log.info( "No loss of dataplane connectivity" )
2095 utilities.assert_equals(
2096 expect=main.FALSE,
2097 actual=LossInPings,
2098 onpass="No Loss of connectivity",
2099 onfail="Loss of dataplane connectivity detected" )
2100 '''
2101
2102 main.step( "Leadership Election is still functional" )
2103 # Test of LeadershipElection
Jon Halla440e872016-03-31 15:15:50 -07002104 leaderList = []
2105
Jon Hall5cf14d52015-07-16 12:15:19 -07002106 # NOTE: this only works for the sanity test. In case of failures,
2107 # leader will likely change
Jon Halla440e872016-03-31 15:15:50 -07002108 leader = main.nodes[ main.activeNodes[ 0 ] ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002109 leaderResult = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002110
2111 for i in main.activeNodes:
2112 cli = main.CLIs[i]
Jon Hall5cf14d52015-07-16 12:15:19 -07002113 leaderN = cli.electionTestLeader()
Jon Halla440e872016-03-31 15:15:50 -07002114 leaderList.append( leaderN )
Jon Hall5cf14d52015-07-16 12:15:19 -07002115 # verify leader is ONOS1
2116 if leaderN == leader:
2117 # all is well
2118 # NOTE: In failure scenario, this could be a new node, maybe
2119 # check != ONOS1
2120 pass
2121 elif leaderN == main.FALSE:
2122 # error in response
2123 main.log.error( "Something is wrong with " +
2124 "electionTestLeader function, check the" +
2125 " error logs" )
2126 leaderResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002127 elif leaderN is None:
2128 main.log.error( cli.name +
2129 " shows no leader for the election-app was" +
2130 " elected after the old one died" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002131 leaderResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002132 if len( set( leaderList ) ) != 1:
2133 leaderResult = main.FALSE
2134 main.log.error(
2135 "Inconsistent view of leader for the election test app" )
2136 # TODO: print the list
Jon Hall5cf14d52015-07-16 12:15:19 -07002137 utilities.assert_equals(
2138 expect=main.TRUE,
2139 actual=leaderResult,
2140 onpass="Leadership election passed",
2141 onfail="Something went wrong with Leadership election" )
2142
2143 def CASE8( self, main ):
2144 """
2145 Compare topo
2146 """
2147 import json
2148 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002149 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002150 assert main, "main not defined"
2151 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002152 assert main.CLIs, "main.CLIs not defined"
2153 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002154
2155 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002156 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002157 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002158 topoResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002159 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002160 elapsed = 0
2161 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002162 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002163 startTime = time.time()
2164 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002165 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002166 devicesResults = main.TRUE
2167 linksResults = main.TRUE
2168 hostsResults = main.TRUE
2169 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002170 count += 1
2171 cliStart = time.time()
2172 devices = []
2173 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002174 for i in main.activeNodes:
2175 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002176 name="devices-" + str( i ),
Jon Halla440e872016-03-31 15:15:50 -07002177 args=[ main.CLIs[i].devices, [ None ] ],
2178 kwargs= { 'sleep': 5, 'attempts': 5,
2179 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002180 threads.append( t )
2181 t.start()
2182
2183 for t in threads:
2184 t.join()
2185 devices.append( t.result )
2186 hosts = []
2187 ipResult = main.TRUE
2188 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002189 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002190 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002191 name="hosts-" + str( i ),
Jon Halld8f6de82015-12-17 17:04:34 -08002192 args=[ main.CLIs[i].hosts, [ None ] ],
2193 kwargs= { 'sleep': 5, 'attempts': 5,
2194 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002195 threads.append( t )
2196 t.start()
2197
2198 for t in threads:
2199 t.join()
2200 try:
2201 hosts.append( json.loads( t.result ) )
2202 except ( ValueError, TypeError ):
2203 main.log.exception( "Error parsing hosts results" )
2204 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002205 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002206 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002207 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002208 if hosts[ controller ]:
2209 for host in hosts[ controller ]:
2210 if host is None or host.get( 'ipAddresses', [] ) == []:
2211 main.log.error(
2212 "Error with host ipAddresses on controller" +
2213 controllerStr + ": " + str( host ) )
2214 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002215 ports = []
2216 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002217 for i in main.activeNodes:
2218 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002219 name="ports-" + str( i ),
Jon Halla440e872016-03-31 15:15:50 -07002220 args=[ main.CLIs[i].ports, [ None ] ],
2221 kwargs= { 'sleep': 5, 'attempts': 5,
2222 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002223 threads.append( t )
2224 t.start()
2225
2226 for t in threads:
2227 t.join()
2228 ports.append( t.result )
2229 links = []
2230 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002231 for i in main.activeNodes:
2232 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002233 name="links-" + str( i ),
Jon Halla440e872016-03-31 15:15:50 -07002234 args=[ main.CLIs[i].links, [ None ] ],
2235 kwargs= { 'sleep': 5, 'attempts': 5,
2236 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002237 threads.append( t )
2238 t.start()
2239
2240 for t in threads:
2241 t.join()
2242 links.append( t.result )
2243 clusters = []
2244 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002245 for i in main.activeNodes:
2246 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002247 name="clusters-" + str( i ),
Jon Halla440e872016-03-31 15:15:50 -07002248 args=[ main.CLIs[i].clusters, [ None ] ],
2249 kwargs= { 'sleep': 5, 'attempts': 5,
2250 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002251 threads.append( t )
2252 t.start()
2253
2254 for t in threads:
2255 t.join()
2256 clusters.append( t.result )
2257
2258 elapsed = time.time() - startTime
2259 cliTime = time.time() - cliStart
2260 print "Elapsed time: " + str( elapsed )
2261 print "CLI time: " + str( cliTime )
2262
Jon Halla440e872016-03-31 15:15:50 -07002263 if all( e is None for e in devices ) and\
2264 all( e is None for e in hosts ) and\
2265 all( e is None for e in ports ) and\
2266 all( e is None for e in links ) and\
2267 all( e is None for e in clusters ):
2268 topoFailMsg = "Could not get topology from ONOS"
2269 main.log.error( topoFailMsg )
2270 continue # Try again, No use trying to compare
2271
Jon Hall5cf14d52015-07-16 12:15:19 -07002272 mnSwitches = main.Mininet1.getSwitches()
2273 mnLinks = main.Mininet1.getLinks()
2274 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07002275 for controller in range( len( main.activeNodes ) ):
2276 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002277 if devices[ controller ] and ports[ controller ] and\
2278 "Error" not in devices[ controller ] and\
2279 "Error" not in ports[ controller ]:
2280
Jon Hallc6793552016-01-19 14:18:37 -08002281 try:
2282 currentDevicesResult = main.Mininet1.compareSwitches(
2283 mnSwitches,
2284 json.loads( devices[ controller ] ),
2285 json.loads( ports[ controller ] ) )
2286 except ( TypeError, ValueError ) as e:
2287 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2288 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002289 else:
2290 currentDevicesResult = main.FALSE
2291 utilities.assert_equals( expect=main.TRUE,
2292 actual=currentDevicesResult,
2293 onpass="ONOS" + controllerStr +
2294 " Switches view is correct",
2295 onfail="ONOS" + controllerStr +
2296 " Switches view is incorrect" )
2297
2298 if links[ controller ] and "Error" not in links[ controller ]:
2299 currentLinksResult = main.Mininet1.compareLinks(
2300 mnSwitches, mnLinks,
2301 json.loads( links[ controller ] ) )
2302 else:
2303 currentLinksResult = main.FALSE
2304 utilities.assert_equals( expect=main.TRUE,
2305 actual=currentLinksResult,
2306 onpass="ONOS" + controllerStr +
2307 " links view is correct",
2308 onfail="ONOS" + controllerStr +
2309 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002310 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002311 currentHostsResult = main.Mininet1.compareHosts(
2312 mnHosts,
2313 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002314 elif hosts[ controller ] == []:
2315 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002316 else:
2317 currentHostsResult = main.FALSE
2318 utilities.assert_equals( expect=main.TRUE,
2319 actual=currentHostsResult,
2320 onpass="ONOS" + controllerStr +
2321 " hosts exist in Mininet",
2322 onfail="ONOS" + controllerStr +
2323 " hosts don't match Mininet" )
2324 # CHECKING HOST ATTACHMENT POINTS
2325 hostAttachment = True
2326 zeroHosts = False
2327 # FIXME: topo-HA/obelisk specific mappings:
2328 # key is mac and value is dpid
2329 mappings = {}
2330 for i in range( 1, 29 ): # hosts 1 through 28
2331 # set up correct variables:
2332 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
2333 if i == 1:
2334 deviceId = "1000".zfill(16)
2335 elif i == 2:
2336 deviceId = "2000".zfill(16)
2337 elif i == 3:
2338 deviceId = "3000".zfill(16)
2339 elif i == 4:
2340 deviceId = "3004".zfill(16)
2341 elif i == 5:
2342 deviceId = "5000".zfill(16)
2343 elif i == 6:
2344 deviceId = "6000".zfill(16)
2345 elif i == 7:
2346 deviceId = "6007".zfill(16)
2347 elif i >= 8 and i <= 17:
2348 dpid = '3' + str( i ).zfill( 3 )
2349 deviceId = dpid.zfill(16)
2350 elif i >= 18 and i <= 27:
2351 dpid = '6' + str( i ).zfill( 3 )
2352 deviceId = dpid.zfill(16)
2353 elif i == 28:
2354 deviceId = "2800".zfill(16)
2355 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002356 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002357 if hosts[ controller ] == []:
2358 main.log.warn( "There are no hosts discovered" )
2359 zeroHosts = True
2360 else:
2361 for host in hosts[ controller ]:
2362 mac = None
2363 location = None
2364 device = None
2365 port = None
2366 try:
2367 mac = host.get( 'mac' )
2368 assert mac, "mac field could not be found for this host object"
2369
2370 location = host.get( 'location' )
2371 assert location, "location field could not be found for this host object"
2372
2373 # Trim the protocol identifier off deviceId
2374 device = str( location.get( 'elementId' ) ).split(':')[1]
2375 assert device, "elementId field could not be found for this host location object"
2376
2377 port = location.get( 'port' )
2378 assert port, "port field could not be found for this host location object"
2379
2380 # Now check if this matches where they should be
2381 if mac and device and port:
2382 if str( port ) != "1":
2383 main.log.error( "The attachment port is incorrect for " +
2384 "host " + str( mac ) +
2385 ". Expected: 1 Actual: " + str( port) )
2386 hostAttachment = False
2387 if device != mappings[ str( mac ) ]:
2388 main.log.error( "The attachment device is incorrect for " +
2389 "host " + str( mac ) +
2390 ". Expected: " + mappings[ str( mac ) ] +
2391 " Actual: " + device )
2392 hostAttachment = False
2393 else:
2394 hostAttachment = False
2395 except AssertionError:
2396 main.log.exception( "Json object not as expected" )
2397 main.log.error( repr( host ) )
2398 hostAttachment = False
2399 else:
2400 main.log.error( "No hosts json output or \"Error\"" +
2401 " in output. hosts = " +
2402 repr( hosts[ controller ] ) )
2403 if zeroHosts is False:
2404 hostAttachment = True
2405
2406 # END CHECKING HOST ATTACHMENT POINTS
2407 devicesResults = devicesResults and currentDevicesResult
2408 linksResults = linksResults and currentLinksResult
2409 hostsResults = hostsResults and currentHostsResult
2410 hostAttachmentResults = hostAttachmentResults and\
2411 hostAttachment
2412 topoResult = ( devicesResults and linksResults
2413 and hostsResults and ipResult and
2414 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002415 utilities.assert_equals( expect=True,
2416 actual=topoResult,
2417 onpass="ONOS topology matches Mininet",
Jon Halla440e872016-03-31 15:15:50 -07002418 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002419 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002420
2421 # Compare json objects for hosts and dataplane clusters
2422
2423 # hosts
2424 main.step( "Hosts view is consistent across all ONOS nodes" )
2425 consistentHostsResult = main.TRUE
2426 for controller in range( len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07002427 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002428 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002429 if hosts[ controller ] == hosts[ 0 ]:
2430 continue
2431 else: # hosts not consistent
2432 main.log.error( "hosts from ONOS" + controllerStr +
2433 " is inconsistent with ONOS1" )
2434 main.log.warn( repr( hosts[ controller ] ) )
2435 consistentHostsResult = main.FALSE
2436
2437 else:
2438 main.log.error( "Error in getting ONOS hosts from ONOS" +
2439 controllerStr )
2440 consistentHostsResult = main.FALSE
2441 main.log.warn( "ONOS" + controllerStr +
2442 " hosts response: " +
2443 repr( hosts[ controller ] ) )
2444 utilities.assert_equals(
2445 expect=main.TRUE,
2446 actual=consistentHostsResult,
2447 onpass="Hosts view is consistent across all ONOS nodes",
2448 onfail="ONOS nodes have different views of hosts" )
2449
2450 main.step( "Hosts information is correct" )
2451 hostsResults = hostsResults and ipResult
2452 utilities.assert_equals(
2453 expect=main.TRUE,
2454 actual=hostsResults,
2455 onpass="Host information is correct",
2456 onfail="Host information is incorrect" )
2457
2458 main.step( "Host attachment points to the network" )
2459 utilities.assert_equals(
2460 expect=True,
2461 actual=hostAttachmentResults,
2462 onpass="Hosts are correctly attached to the network",
2463 onfail="ONOS did not correctly attach hosts to the network" )
2464
2465 # Strongly connected clusters of devices
2466 main.step( "Clusters view is consistent across all ONOS nodes" )
2467 consistentClustersResult = main.TRUE
2468 for controller in range( len( clusters ) ):
Jon Halla440e872016-03-31 15:15:50 -07002469 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002470 if "Error" not in clusters[ controller ]:
2471 if clusters[ controller ] == clusters[ 0 ]:
2472 continue
2473 else: # clusters not consistent
2474 main.log.error( "clusters from ONOS" +
2475 controllerStr +
2476 " is inconsistent with ONOS1" )
2477 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002478 else:
2479 main.log.error( "Error in getting dataplane clusters " +
2480 "from ONOS" + controllerStr )
2481 consistentClustersResult = main.FALSE
2482 main.log.warn( "ONOS" + controllerStr +
2483 " clusters response: " +
2484 repr( clusters[ controller ] ) )
2485 utilities.assert_equals(
2486 expect=main.TRUE,
2487 actual=consistentClustersResult,
2488 onpass="Clusters view is consistent across all ONOS nodes",
2489 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002490 if not consistentClustersResult:
2491 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002492
2493 main.step( "There is only one SCC" )
2494 # there should always only be one cluster
2495 try:
2496 numClusters = len( json.loads( clusters[ 0 ] ) )
2497 except ( ValueError, TypeError ):
2498 main.log.exception( "Error parsing clusters[0]: " +
2499 repr( clusters[0] ) )
Jon Halla440e872016-03-31 15:15:50 -07002500 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002501 clusterResults = main.FALSE
2502 if numClusters == 1:
2503 clusterResults = main.TRUE
2504 utilities.assert_equals(
2505 expect=1,
2506 actual=numClusters,
2507 onpass="ONOS shows 1 SCC",
2508 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2509
2510 topoResult = ( devicesResults and linksResults
2511 and hostsResults and consistentHostsResult
2512 and consistentClustersResult and clusterResults
2513 and ipResult and hostAttachmentResults )
2514
2515 topoResult = topoResult and int( count <= 2 )
2516 note = "note it takes about " + str( int( cliTime ) ) + \
2517 " seconds for the test to make all the cli calls to fetch " +\
2518 "the topology from each ONOS instance"
2519 main.log.info(
2520 "Very crass estimate for topology discovery/convergence( " +
2521 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2522 str( count ) + " tries" )
2523
2524 main.step( "Device information is correct" )
2525 utilities.assert_equals(
2526 expect=main.TRUE,
2527 actual=devicesResults,
2528 onpass="Device information is correct",
2529 onfail="Device information is incorrect" )
2530
2531 main.step( "Links are correct" )
2532 utilities.assert_equals(
2533 expect=main.TRUE,
2534 actual=linksResults,
2535 onpass="Link are correct",
2536 onfail="Links are incorrect" )
2537
2538 main.step( "Hosts are correct" )
2539 utilities.assert_equals(
2540 expect=main.TRUE,
2541 actual=hostsResults,
2542 onpass="Hosts are correct",
2543 onfail="Hosts are incorrect" )
2544
2545 # FIXME: move this to an ONOS state case
2546 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002547 nodeResults = utilities.retry( main.HA.nodesCheck,
2548 False,
2549 args=[main.activeNodes],
2550 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002551
Jon Hall41d39f12016-04-11 22:54:35 -07002552 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002553 onpass="Nodes check successful",
2554 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002555 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002556 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002557 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hall41d39f12016-04-11 22:54:35 -07002558 main.CLIs[i].name,
2559 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002560
Jon Halld2871c22016-07-26 11:01:14 -07002561 if not topoResult:
2562 main.cleanup()
2563 main.exit()
2564
Jon Hall5cf14d52015-07-16 12:15:19 -07002565 def CASE9( self, main ):
2566 """
2567 Link s3-s28 down
2568 """
2569 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002570 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002571 assert main, "main not defined"
2572 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002573 assert main.CLIs, "main.CLIs not defined"
2574 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002575 # NOTE: You should probably run a topology check after this
2576
2577 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2578
2579 description = "Turn off a link to ensure that Link Discovery " +\
2580 "is working properly"
2581 main.case( description )
2582
2583 main.step( "Kill Link between s3 and s28" )
2584 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2585 main.log.info( "Waiting " + str( linkSleep ) +
2586 " seconds for link down to be discovered" )
2587 time.sleep( linkSleep )
2588 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2589 onpass="Link down successful",
2590 onfail="Failed to bring link down" )
2591 # TODO do some sort of check here
2592
2593 def CASE10( self, main ):
2594 """
2595 Link s3-s28 up
2596 """
2597 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002598 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002599 assert main, "main not defined"
2600 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002601 assert main.CLIs, "main.CLIs not defined"
2602 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002603 # NOTE: You should probably run a topology check after this
2604
2605 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2606
2607 description = "Restore a link to ensure that Link Discovery is " + \
2608 "working properly"
2609 main.case( description )
2610
2611 main.step( "Bring link between s3 and s28 back up" )
2612 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2613 main.log.info( "Waiting " + str( linkSleep ) +
2614 " seconds for link up to be discovered" )
2615 time.sleep( linkSleep )
2616 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2617 onpass="Link up successful",
2618 onfail="Failed to bring link up" )
2619 # TODO do some sort of check here
2620
2621 def CASE11( self, main ):
2622 """
2623 Switch Down
2624 """
2625 # NOTE: You should probably run a topology check after this
2626 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002627 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002628 assert main, "main not defined"
2629 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002630 assert main.CLIs, "main.CLIs not defined"
2631 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002632
2633 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2634
2635 description = "Killing a switch to ensure it is discovered correctly"
Jon Halla440e872016-03-31 15:15:50 -07002636 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002637 main.case( description )
2638 switch = main.params[ 'kill' ][ 'switch' ]
2639 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2640
2641 # TODO: Make this switch parameterizable
2642 main.step( "Kill " + switch )
2643 main.log.info( "Deleting " + switch )
2644 main.Mininet1.delSwitch( switch )
2645 main.log.info( "Waiting " + str( switchSleep ) +
2646 " seconds for switch down to be discovered" )
2647 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002648 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002649 # Peek at the deleted switch
2650 main.log.warn( str( device ) )
2651 result = main.FALSE
2652 if device and device[ 'available' ] is False:
2653 result = main.TRUE
2654 utilities.assert_equals( expect=main.TRUE, actual=result,
2655 onpass="Kill switch successful",
2656 onfail="Failed to kill switch?" )
2657
2658 def CASE12( self, main ):
2659 """
2660 Switch Up
2661 """
2662 # NOTE: You should probably run a topology check after this
2663 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002664 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002665 assert main, "main not defined"
2666 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002667 assert main.CLIs, "main.CLIs not defined"
2668 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002669 assert ONOS1Port, "ONOS1Port not defined"
2670 assert ONOS2Port, "ONOS2Port not defined"
2671 assert ONOS3Port, "ONOS3Port not defined"
2672 assert ONOS4Port, "ONOS4Port not defined"
2673 assert ONOS5Port, "ONOS5Port not defined"
2674 assert ONOS6Port, "ONOS6Port not defined"
2675 assert ONOS7Port, "ONOS7Port not defined"
2676
2677 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2678 switch = main.params[ 'kill' ][ 'switch' ]
2679 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2680 links = main.params[ 'kill' ][ 'links' ].split()
Jon Halla440e872016-03-31 15:15:50 -07002681 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002682 description = "Adding a switch to ensure it is discovered correctly"
2683 main.case( description )
2684
2685 main.step( "Add back " + switch )
2686 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2687 for peer in links:
2688 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07002689 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002690 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2691 main.log.info( "Waiting " + str( switchSleep ) +
2692 " seconds for switch up to be discovered" )
2693 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002694 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002695 # Peek at the deleted switch
2696 main.log.warn( str( device ) )
2697 result = main.FALSE
2698 if device and device[ 'available' ]:
2699 result = main.TRUE
2700 utilities.assert_equals( expect=main.TRUE, actual=result,
2701 onpass="add switch successful",
2702 onfail="Failed to add switch?" )
2703
2704 def CASE13( self, main ):
2705 """
2706 Clean up
2707 """
2708 import os
2709 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002710 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002711 assert main, "main not defined"
2712 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002713 assert main.CLIs, "main.CLIs not defined"
2714 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002715
2716 # printing colors to terminal
2717 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2718 'blue': '\033[94m', 'green': '\033[92m',
2719 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2720 main.case( "Test Cleanup" )
2721 main.step( "Killing tcpdumps" )
2722 main.Mininet2.stopTcpdump()
2723
2724 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002725 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002726 main.step( "Copying MN pcap and ONOS log files to test station" )
2727 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2728 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002729 # NOTE: MN Pcap file is being saved to logdir.
2730 # We scp this file as MN and TestON aren't necessarily the same vm
2731
2732 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002733 # TODO: Load these from params
2734 # NOTE: must end in /
2735 logFolder = "/opt/onos/log/"
2736 logFiles = [ "karaf.log", "karaf.log.1" ]
2737 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002738 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002739 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002740 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002741 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2742 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002743 # std*.log's
2744 # NOTE: must end in /
2745 logFolder = "/opt/onos/var/"
2746 logFiles = [ "stderr.log", "stdout.log" ]
2747 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002748 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002749 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002750 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002751 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2752 logFolder + f, dstName )
2753 else:
2754 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002755
2756 main.step( "Stopping Mininet" )
2757 mnResult = main.Mininet1.stopNet()
2758 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2759 onpass="Mininet stopped",
2760 onfail="MN cleanup NOT successful" )
2761
2762 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002763 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002764 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2765 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002766
2767 try:
2768 timerLog = open( main.logdir + "/Timers.csv", 'w')
2769 # Overwrite with empty line and close
2770 labels = "Gossip Intents"
2771 data = str( gossipTime )
2772 timerLog.write( labels + "\n" + data )
2773 timerLog.close()
2774 except NameError, e:
2775 main.log.exception(e)
2776
2777 def CASE14( self, main ):
2778 """
2779 start election app on all onos nodes
2780 """
Jon Halle1a3b752015-07-22 13:02:46 -07002781 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002782 assert main, "main not defined"
2783 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002784 assert main.CLIs, "main.CLIs not defined"
2785 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002786
2787 main.case("Start Leadership Election app")
2788 main.step( "Install leadership election app" )
Jon Halla440e872016-03-31 15:15:50 -07002789 onosCli = main.CLIs[ main.activeNodes[0] ]
2790 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002791 utilities.assert_equals(
2792 expect=main.TRUE,
2793 actual=appResult,
2794 onpass="Election app installed",
2795 onfail="Something went wrong with installing Leadership election" )
2796
2797 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002798 for i in main.activeNodes:
2799 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002800 time.sleep(5)
2801 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2802 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002803 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002804 expect=True,
2805 actual=sameResult,
2806 onpass="All nodes see the same leaderboards",
2807 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002808
Jon Hall25463a82016-04-13 14:03:52 -07002809 if sameResult:
2810 leader = leaders[ 0 ][ 0 ]
2811 if main.nodes[main.activeNodes[0]].ip_address in leader:
2812 correctLeader = True
2813 else:
2814 correctLeader = False
2815 main.step( "First node was elected leader" )
2816 utilities.assert_equals(
2817 expect=True,
2818 actual=correctLeader,
2819 onpass="Correct leader was elected",
2820 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002821
2822 def CASE15( self, main ):
2823 """
2824 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002825 15.1 Run election on each node
2826 15.2 Check that each node has the same leaders and candidates
2827 15.3 Find current leader and withdraw
2828 15.4 Check that a new node was elected leader
2829 15.5 Check that that new leader was the candidate of old leader
2830 15.6 Run for election on old leader
2831 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2832 15.8 Make sure that the old leader was added to the candidate list
2833
2834 old and new variable prefixes refer to data from before vs after
2835 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002836 """
2837 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002838 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002839 assert main, "main not defined"
2840 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002841 assert main.CLIs, "main.CLIs not defined"
2842 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002843
Jon Halla440e872016-03-31 15:15:50 -07002844 description = "Check that Leadership Election is still functional"
Jon Hall5cf14d52015-07-16 12:15:19 -07002845 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002846 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002847
Jon Halla440e872016-03-31 15:15:50 -07002848 oldLeaders = [] # list of lists of each nodes' candidates before
2849 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002850 oldLeader = '' # the old leader from oldLeaders, None if not same
2851 newLeader = '' # the new leaders fron newLoeaders, None if not same
2852 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2853 expectNoLeader = False # True when there is only one leader
2854 if main.numCtrls == 1:
2855 expectNoLeader = True
2856
2857 main.step( "Run for election on each node" )
2858 electionResult = main.TRUE
2859
Jon Halla440e872016-03-31 15:15:50 -07002860 for i in main.activeNodes: # run test election on each node
2861 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002862 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002863 utilities.assert_equals(
2864 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002865 actual=electionResult,
2866 onpass="All nodes successfully ran for leadership",
2867 onfail="At least one node failed to run for leadership" )
2868
acsmars3a72bde2015-09-02 14:16:22 -07002869 if electionResult == main.FALSE:
2870 main.log.error(
Jon Halla440e872016-03-31 15:15:50 -07002871 "Skipping Test Case because Election Test App isn't loaded" )
acsmars3a72bde2015-09-02 14:16:22 -07002872 main.skipCase()
2873
acsmars71adceb2015-08-31 15:09:26 -07002874 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002875 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002876 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002877 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002878 if sameResult:
2879 oldLeader = oldLeaders[ 0 ][ 0 ]
2880 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07002881 else:
Jon Halla440e872016-03-31 15:15:50 -07002882 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002883 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002884 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002885 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002886 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002887 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002888
2889 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002890 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002891 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002892 if oldLeader is None:
2893 main.log.error( "Leadership isn't consistent." )
2894 withdrawResult = main.FALSE
2895 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002896 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002897 if oldLeader == main.nodes[ i ].ip_address:
2898 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002899 break
2900 else: # FOR/ELSE statement
2901 main.log.error( "Leader election, could not find current leader" )
2902 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002903 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002904 utilities.assert_equals(
2905 expect=main.TRUE,
2906 actual=withdrawResult,
2907 onpass="Node was withdrawn from election",
2908 onfail="Node was not withdrawn from election" )
2909
acsmars71adceb2015-08-31 15:09:26 -07002910 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002911 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07002912 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002913 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002914 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002915 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002916 if newLeaders[ 0 ][ 0 ] == 'none':
2917 main.log.error( "No leader was elected on at least 1 node" )
2918 if not expectNoLeader:
2919 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002920 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002921
2922 # Check that the new leader is not the older leader, which was withdrawn
2923 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002924 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08002925 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07002926 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002927 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002928 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002929 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002930 onpass="Leadership election passed",
2931 onfail="Something went wrong with Leadership election" )
2932
Jon Halla440e872016-03-31 15:15:50 -07002933 main.step( "Check that that new leader was the candidate of old leader" )
2934 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07002935 correctCandidateResult = main.TRUE
2936 if expectNoLeader:
2937 if newLeader == 'none':
2938 main.log.info( "No leader expected. None found. Pass" )
2939 correctCandidateResult = main.TRUE
2940 else:
2941 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2942 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002943 elif len( oldLeaders[0] ) >= 3:
2944 if newLeader == oldLeaders[ 0 ][ 2 ]:
2945 # correct leader was elected
2946 correctCandidateResult = main.TRUE
2947 else:
2948 correctCandidateResult = main.FALSE
2949 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
2950 newLeader, oldLeaders[ 0 ][ 2 ] ) )
2951 else:
2952 main.log.warn( "Could not determine who should be the correct leader" )
2953 main.log.debug( oldLeaders[ 0 ] )
acsmars71adceb2015-08-31 15:09:26 -07002954 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002955 utilities.assert_equals(
2956 expect=main.TRUE,
2957 actual=correctCandidateResult,
2958 onpass="Correct Candidate Elected",
2959 onfail="Incorrect Candidate Elected" )
2960
Jon Hall5cf14d52015-07-16 12:15:19 -07002961 main.step( "Run for election on old leader( just so everyone " +
2962 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002963 if oldLeaderCLI is not None:
2964 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002965 else:
acsmars71adceb2015-08-31 15:09:26 -07002966 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002967 runResult = main.FALSE
2968 utilities.assert_equals(
2969 expect=main.TRUE,
2970 actual=runResult,
2971 onpass="App re-ran for election",
2972 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07002973
acsmars71adceb2015-08-31 15:09:26 -07002974 main.step(
2975 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002976 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07002977 # Get new leaders and candidates
2978 reRunLeaders = []
Jon Hall7a6ebfd2017-03-13 10:58:58 -07002979 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07002980 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07002981
2982 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07002983 if not reRunLeaders[0]:
2984 positionResult = main.FALSE
2985 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07002986 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
2987 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07002988 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002989 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002990 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002991 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002992 onpass="Old leader successfully re-ran for election",
2993 onfail="Something went wrong with Leadership election after " +
2994 "the old leader re-ran for election" )
2995
2996 def CASE16( self, main ):
2997 """
2998 Install Distributed Primitives app
2999 """
3000 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003001 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003002 assert main, "main not defined"
3003 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003004 assert main.CLIs, "main.CLIs not defined"
3005 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003006
3007 # Variables for the distributed primitives tests
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003008 main.pCounterName = "TestON-Partitions"
3009 main.pCounterValue = 0
3010 main.onosSet = set([])
3011 main.onosSetName = "TestON-set"
Jon Hall5cf14d52015-07-16 12:15:19 -07003012
3013 description = "Install Primitives app"
3014 main.case( description )
3015 main.step( "Install Primitives app" )
3016 appName = "org.onosproject.distributedprimitives"
Jon Halla440e872016-03-31 15:15:50 -07003017 node = main.activeNodes[0]
3018 appResults = main.CLIs[node].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003019 utilities.assert_equals( expect=main.TRUE,
3020 actual=appResults,
3021 onpass="Primitives app activated",
3022 onfail="Primitives app not activated" )
3023 time.sleep( 5 ) # To allow all nodes to activate
3024
3025 def CASE17( self, main ):
3026 """
3027 Check for basic functionality with distributed primitives
3028 """
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003029 main.HA.CASE17( main )