blob: 49fc2780759d2f9780ec2ac649708f69b7dec62d [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
Jon Hallb3ed8ed2015-10-28 16:43:55 -070012CASE61: The Failure inducing case.
13CASE62: The Failure recovery case.
Jon Hall5cf14d52015-07-16 12:15:19 -070014CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
Jon Hallb3ed8ed2015-10-28 16:43:55 -070026class HAstopNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -070027
28 def __init__( self ):
29 self.default = ''
30
31 def CASE1( self, main ):
32 """
33 CASE1 is to compile ONOS and push it to the test machines
34
35 Startup sequence:
36 cell <name>
37 onos-verify-cell
38 NOTE: temporary - onos-remove-raft-logs
39 onos-uninstall
40 start mininet
41 git pull
42 mvn clean install
43 onos-package
44 onos-install -f
45 onos-wait-for-start
46 start cli sessions
47 start tcpdump
48 """
Jon Halle1a3b752015-07-22 13:02:46 -070049 import imp
Jon Hall3b489db2015-10-05 14:38:37 -070050 import pexpect
Jon Hallf3d16e72015-12-16 17:45:08 -080051 import time
Jon Halla440e872016-03-31 15:15:50 -070052 import json
Jon Hallb3ed8ed2015-10-28 16:43:55 -070053 main.log.info( "ONOS HA test: Stop a minority of ONOS nodes - " +
Jon Hall5cf14d52015-07-16 12:15:19 -070054 "initialization" )
55 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070056 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070057 "installing ONOS, starting Mininet and ONOS" +\
58 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070059
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
71 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070079 # These are for csv plotting in jenkins
80 global labels
81 global data
82 labels = []
83 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070084
85 # FIXME: just get controller port from params?
86 # TODO: do we really need all these?
87 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
88 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
89 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
90 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
91 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
92 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
93 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
94
Jon Halle1a3b752015-07-22 13:02:46 -070095 try:
Jon Hall53c5e662016-04-13 16:06:56 -070096 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070097 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070098 except Exception as e:
99 main.log.exception( e )
100 main.cleanup()
101 main.exit()
102
103 main.CLIs = []
104 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700105 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700106 for i in range( 1, main.numCtrls + 1 ):
107 try:
108 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
109 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
110 ipList.append( main.nodes[ -1 ].ip_address )
111 except AttributeError:
112 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700113
114 main.step( "Create cell file" )
115 cellAppString = main.params[ 'ENV' ][ 'appString' ]
116 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
117 main.Mininet1.ip_address,
Devin Lim461f0872017-06-05 16:49:33 -0700118 cellAppString, ipList, main.ONOScli1.user_name )
Jon Hall5cf14d52015-07-16 12:15:19 -0700119 main.step( "Applying cell variable to environment" )
120 cellResult = main.ONOSbench.setCell( cellName )
121 verifyResult = main.ONOSbench.verifyCell()
122
123 # FIXME:this is short term fix
124 main.log.info( "Removing raft logs" )
125 main.ONOSbench.onosRemoveRaftLogs()
126
127 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700128 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700129 main.ONOSbench.onosUninstall( node.ip_address )
130
131 # Make sure ONOS is DEAD
132 main.log.info( "Killing any ONOS processes" )
133 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700134 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700135 killed = main.ONOSbench.onosKill( node.ip_address )
136 killResults = killResults and killed
137
138 cleanInstallResult = main.TRUE
139 gitPullResult = main.TRUE
140
141 main.step( "Starting Mininet" )
142 # scp topo file to mininet
143 # TODO: move to params?
144 topoName = "obelisk.py"
145 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700146 main.ONOSbench.scp( main.Mininet1,
147 filePath + topoName,
148 main.Mininet1.home,
149 direction="to" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700150 mnResult = main.Mininet1.startNet()
Jon Hall5cf14d52015-07-16 12:15:19 -0700151 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
152 onpass="Mininet Started",
153 onfail="Error starting Mininet" )
154
155 main.step( "Git checkout and pull " + gitBranch )
156 if PULLCODE:
157 main.ONOSbench.gitCheckout( gitBranch )
158 gitPullResult = main.ONOSbench.gitPull()
159 # values of 1 or 3 are good
160 utilities.assert_lesser( expect=0, actual=gitPullResult,
161 onpass="Git pull successful",
162 onfail="Git pull failed" )
163 main.ONOSbench.getVersion( report=True )
164
165 main.step( "Using mvn clean install" )
166 cleanInstallResult = main.TRUE
167 if PULLCODE and gitPullResult == main.TRUE:
168 cleanInstallResult = main.ONOSbench.cleanInstall()
169 else:
170 main.log.warn( "Did not pull new code so skipping mvn " +
171 "clean install" )
172 utilities.assert_equals( expect=main.TRUE,
173 actual=cleanInstallResult,
174 onpass="MCI successful",
175 onfail="MCI failed" )
176 # GRAPHS
177 # NOTE: important params here:
178 # job = name of Jenkins job
179 # Plot Name = Plot-HA, only can be used if multiple plots
180 # index = The number of the graph under plot name
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700181 job = "HAstopNodes"
Jon Hall5cf14d52015-07-16 12:15:19 -0700182 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700183 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700184 graphs = '<ac:structured-macro ac:name="html">\n'
185 graphs += '<ac:plain-text-body><![CDATA[\n'
186 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800187 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700188 '&width=500&height=300"' +\
189 'noborder="0" width="500" height="300" scrolling="yes" ' +\
190 'seamless="seamless"></iframe>\n'
191 graphs += ']]></ac:plain-text-body>\n'
192 graphs += '</ac:structured-macro>\n'
Jon Hallf37d44d2017-05-24 10:37:30 -0700193 main.log.wiki( graphs )
Jon Hall5cf14d52015-07-16 12:15:19 -0700194
195 main.step( "Creating ONOS package" )
Jon Hall3b489db2015-10-05 14:38:37 -0700196 # copy gen-partions file to ONOS
197 # NOTE: this assumes TestON and ONOS are on the same machine
Jon Hall53c5e662016-04-13 16:06:56 -0700198 srcFile = main.testDir + "/HA/dependencies/onos-gen-partitions"
Jon Hall3b489db2015-10-05 14:38:37 -0700199 dstDir = main.ONOSbench.home + "/tools/test/bin/onos-gen-partitions"
200 cpResult = main.ONOSbench.secureCopy( main.ONOSbench.user_name,
201 main.ONOSbench.ip_address,
202 srcFile,
203 dstDir,
204 pwd=main.ONOSbench.pwd,
205 direction="from" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700206 packageResult = main.ONOSbench.buckBuild()
Jon Hall5cf14d52015-07-16 12:15:19 -0700207 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
208 onpass="ONOS package successful",
209 onfail="ONOS package failed" )
210
211 main.step( "Installing ONOS package" )
212 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700213 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700214 tmpResult = main.ONOSbench.onosInstall( options="-f",
215 node=node.ip_address )
216 onosInstallResult = onosInstallResult and tmpResult
217 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
218 onpass="ONOS install successful",
219 onfail="ONOS install failed" )
Jon Hall3b489db2015-10-05 14:38:37 -0700220 # clean up gen-partitions file
221 try:
222 main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
223 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
224 main.ONOSbench.handle.sendline( "git checkout -- tools/test/bin/onos-gen-partitions" )
225 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
226 main.log.info( " Cleaning custom gen partitions file, response was: \n" +
227 str( main.ONOSbench.handle.before ) )
228 except ( pexpect.TIMEOUT, pexpect.EOF ):
229 main.log.exception( "ONOSbench: pexpect exception found:" +
230 main.ONOSbench.handle.before )
231 main.cleanup()
232 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700233
You Wangf5de25b2017-01-06 15:13:01 -0800234 main.step( "Set up ONOS secure SSH" )
235 secureSshResult = main.TRUE
236 for node in main.nodes:
237 secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=node.ip_address )
238 utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
239 onpass="Test step PASS",
240 onfail="Test step FAIL" )
241
Jon Hall5cf14d52015-07-16 12:15:19 -0700242 main.step( "Checking if ONOS is up yet" )
243 for i in range( 2 ):
244 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700245 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700246 started = main.ONOSbench.isup( node.ip_address )
247 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800248 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700249 onosIsupResult = onosIsupResult and started
250 if onosIsupResult == main.TRUE:
251 break
252 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
253 onpass="ONOS startup successful",
254 onfail="ONOS startup failed" )
255
Jon Hall6509dbf2016-06-21 17:01:17 -0700256 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700257 cliResults = main.TRUE
258 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700259 for i in range( main.numCtrls ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700260 t = main.Thread( target=main.CLIs[ i ].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700261 name="startOnosCli-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -0700262 args=[ main.nodes[ i ].ip_address ] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700263 threads.append( t )
264 t.start()
265
266 for t in threads:
267 t.join()
268 cliResults = cliResults and t.result
269 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
270 onpass="ONOS cli startup successful",
271 onfail="ONOS cli startup failed" )
272
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700273 # Create a list of active nodes for use when some nodes are stopped
274 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
275
Jon Hall5cf14d52015-07-16 12:15:19 -0700276 if main.params[ 'tcpdump' ].lower() == "true":
277 main.step( "Start Packet Capture MN" )
278 main.Mininet2.startTcpdump(
279 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
280 + "-MN.pcap",
281 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
282 port=main.params[ 'MNtcpdump' ][ 'port' ] )
283
Jon Halla440e872016-03-31 15:15:50 -0700284 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700285 nodeResults = utilities.retry( main.HA.nodesCheck,
286 False,
Jon Hallf37d44d2017-05-24 10:37:30 -0700287 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -0700288 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700289
Jon Hall41d39f12016-04-11 22:54:35 -0700290 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700291 onpass="Nodes check successful",
292 onfail="Nodes check NOT successful" )
293
294 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700295 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700296 cli = main.CLIs[ i ]
Jon Halla440e872016-03-31 15:15:50 -0700297 main.log.debug( "{} components not ACTIVE: \n{}".format(
298 cli.name,
299 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700300 main.log.error( "Failed to start ONOS, stopping test" )
301 main.cleanup()
302 main.exit()
303
Jon Hall172b7ba2016-04-07 18:12:20 -0700304 main.step( "Activate apps defined in the params file" )
305 # get data from the params
306 apps = main.params.get( 'apps' )
307 if apps:
Jon Hallf37d44d2017-05-24 10:37:30 -0700308 apps = apps.split( ',' )
Jon Hall172b7ba2016-04-07 18:12:20 -0700309 main.log.warn( apps )
310 activateResult = True
311 for app in apps:
312 main.CLIs[ 0 ].app( app, "Activate" )
313 # TODO: check this worked
314 time.sleep( 10 ) # wait for apps to activate
315 for app in apps:
316 state = main.CLIs[ 0 ].appStatus( app )
317 if state == "ACTIVE":
Jon Hall937bc812017-01-31 16:44:10 -0800318 activateResult = activateResult and True
Jon Hall172b7ba2016-04-07 18:12:20 -0700319 else:
320 main.log.error( "{} is in {} state".format( app, state ) )
Jon Hall937bc812017-01-31 16:44:10 -0800321 activateResult = False
Jon Hall172b7ba2016-04-07 18:12:20 -0700322 utilities.assert_equals( expect=True,
323 actual=activateResult,
324 onpass="Successfully activated apps",
325 onfail="Failed to activate apps" )
326 else:
327 main.log.warn( "No apps were specified to be loaded after startup" )
328
329 main.step( "Set ONOS configurations" )
330 config = main.params.get( 'ONOS_Configuration' )
331 if config:
332 main.log.debug( config )
333 checkResult = main.TRUE
334 for component in config:
Jon Hallf37d44d2017-05-24 10:37:30 -0700335 for setting in config[ component ]:
336 value = config[ component ][ setting ]
Jon Hall172b7ba2016-04-07 18:12:20 -0700337 check = main.CLIs[ 0 ].setCfg( component, setting, value )
338 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
339 checkResult = check and checkResult
340 utilities.assert_equals( expect=main.TRUE,
341 actual=checkResult,
342 onpass="Successfully set config",
343 onfail="Failed to set config" )
344 else:
345 main.log.warn( "No configurations were specified to be changed after startup" )
346
Jon Hall9d2dcad2016-04-08 10:15:20 -0700347 main.step( "App Ids check" )
348 appCheck = main.TRUE
349 threads = []
350 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700351 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall9d2dcad2016-04-08 10:15:20 -0700352 name="appToIDCheck-" + str( i ),
353 args=[] )
354 threads.append( t )
355 t.start()
356
357 for t in threads:
358 t.join()
359 appCheck = appCheck and t.result
360 if appCheck != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700361 node = main.activeNodes[ 0 ]
362 main.log.warn( main.CLIs[ node ].apps() )
363 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700364 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
365 onpass="App Ids seem to be correct",
366 onfail="Something is wrong with app Ids" )
367
Jon Hall5cf14d52015-07-16 12:15:19 -0700368 def CASE2( self, main ):
369 """
370 Assign devices to controllers
371 """
372 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700373 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700374 assert main, "main not defined"
375 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700376 assert main.CLIs, "main.CLIs not defined"
377 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700378 assert ONOS1Port, "ONOS1Port not defined"
379 assert ONOS2Port, "ONOS2Port not defined"
380 assert ONOS3Port, "ONOS3Port not defined"
381 assert ONOS4Port, "ONOS4Port not defined"
382 assert ONOS5Port, "ONOS5Port not defined"
383 assert ONOS6Port, "ONOS6Port not defined"
384 assert ONOS7Port, "ONOS7Port not defined"
385
386 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700387 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700388 "and check that an ONOS node becomes the " +\
389 "master of the device."
390 main.step( "Assign switches to controllers" )
391
392 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700393 for i in range( main.numCtrls ):
394 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700395 swList = []
396 for i in range( 1, 29 ):
397 swList.append( "s" + str( i ) )
398 main.Mininet1.assignSwController( sw=swList, ip=ipList )
399
400 mastershipCheck = main.TRUE
401 for i in range( 1, 29 ):
402 response = main.Mininet1.getSwController( "s" + str( i ) )
403 try:
404 main.log.info( str( response ) )
405 except Exception:
406 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700407 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700408 if re.search( "tcp:" + node.ip_address, response ):
409 mastershipCheck = mastershipCheck and main.TRUE
410 else:
411 main.log.error( "Error, node " + node.ip_address + " is " +
412 "not in the list of controllers s" +
413 str( i ) + " is connecting to." )
414 mastershipCheck = main.FALSE
415 utilities.assert_equals(
416 expect=main.TRUE,
417 actual=mastershipCheck,
418 onpass="Switch mastership assigned correctly",
419 onfail="Switches not assigned correctly to controllers" )
420
421 def CASE21( self, main ):
422 """
423 Assign mastership to controllers
424 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700425 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700426 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700427 assert main, "main not defined"
428 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700429 assert main.CLIs, "main.CLIs not defined"
430 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700431 assert ONOS1Port, "ONOS1Port not defined"
432 assert ONOS2Port, "ONOS2Port not defined"
433 assert ONOS3Port, "ONOS3Port not defined"
434 assert ONOS4Port, "ONOS4Port not defined"
435 assert ONOS5Port, "ONOS5Port not defined"
436 assert ONOS6Port, "ONOS6Port not defined"
437 assert ONOS7Port, "ONOS7Port not defined"
438
439 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700440 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700441 "device. Then manually assign" +\
442 " mastership to specific ONOS nodes using" +\
443 " 'device-role'"
444 main.step( "Assign mastership of switches to specific controllers" )
445 # Manually assign mastership to the controller we want
446 roleCall = main.TRUE
447
Jon Hallf37d44d2017-05-24 10:37:30 -0700448 ipList = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700449 deviceList = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700450 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700451 try:
452 # Assign mastership to specific controllers. This assignment was
453 # determined for a 7 node cluser, but will work with any sized
454 # cluster
455 for i in range( 1, 29 ): # switches 1 through 28
456 # set up correct variables:
457 if i == 1:
458 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700459 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700460 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700461 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700462 c = 1 % main.numCtrls
463 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700464 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700465 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700466 c = 1 % main.numCtrls
467 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700468 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700469 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700470 c = 3 % main.numCtrls
471 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700472 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700473 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700474 c = 2 % main.numCtrls
475 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700476 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700477 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700478 c = 2 % main.numCtrls
479 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700480 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700481 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700482 c = 5 % main.numCtrls
483 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700484 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700485 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700486 c = 4 % main.numCtrls
487 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700488 dpid = '3' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700489 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700490 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700491 c = 6 % main.numCtrls
492 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700493 dpid = '6' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700494 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700495 elif i == 28:
496 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700497 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700498 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700499 else:
500 main.log.error( "You didn't write an else statement for " +
501 "switch s" + str( i ) )
502 roleCall = main.FALSE
503 # Assign switch
504 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
505 # TODO: make this controller dynamic
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700506 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700507 ipList.append( ip )
508 deviceList.append( deviceId )
509 except ( AttributeError, AssertionError ):
510 main.log.exception( "Something is wrong with ONOS device view" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700511 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700512 utilities.assert_equals(
513 expect=main.TRUE,
514 actual=roleCall,
515 onpass="Re-assigned switch mastership to designated controller",
516 onfail="Something wrong with deviceRole calls" )
517
518 main.step( "Check mastership was correctly assigned" )
519 roleCheck = main.TRUE
520 # NOTE: This is due to the fact that device mastership change is not
521 # atomic and is actually a multi step process
522 time.sleep( 5 )
523 for i in range( len( ipList ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700524 ip = ipList[ i ]
525 deviceId = deviceList[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700526 # Check assignment
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700527 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700528 if ip in master:
529 roleCheck = roleCheck and main.TRUE
530 else:
531 roleCheck = roleCheck and main.FALSE
532 main.log.error( "Error, controller " + ip + " is not" +
533 " master " + "of device " +
534 str( deviceId ) + ". Master is " +
535 repr( master ) + "." )
536 utilities.assert_equals(
537 expect=main.TRUE,
538 actual=roleCheck,
539 onpass="Switches were successfully reassigned to designated " +
540 "controller",
541 onfail="Switches were not successfully reassigned" )
542
543 def CASE3( self, main ):
544 """
545 Assign intents
546 """
547 import time
548 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700549 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700550 assert main, "main not defined"
551 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700552 assert main.CLIs, "main.CLIs not defined"
553 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700554 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700555 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700556 "assign predetermined host-to-host intents." +\
557 " After installation, check that the intent" +\
558 " is distributed to all nodes and the state" +\
559 " is INSTALLED"
560
561 # install onos-app-fwd
562 main.step( "Install reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700563 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700564 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700565 utilities.assert_equals( expect=main.TRUE, actual=installResults,
566 onpass="Install fwd successful",
567 onfail="Install fwd failed" )
568
569 main.step( "Check app ids" )
570 appCheck = main.TRUE
571 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700572 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700573 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700574 name="appToIDCheck-" + str( i ),
575 args=[] )
576 threads.append( t )
577 t.start()
578
579 for t in threads:
580 t.join()
581 appCheck = appCheck and t.result
582 if appCheck != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700583 main.log.warn( onosCli.apps() )
584 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700585 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
586 onpass="App Ids seem to be correct",
587 onfail="Something is wrong with app Ids" )
588
589 main.step( "Discovering Hosts( Via pingall for now )" )
590 # FIXME: Once we have a host discovery mechanism, use that instead
591 # REACTIVE FWD test
592 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700593 passMsg = "Reactive Pingall test passed"
594 time1 = time.time()
595 pingResult = main.Mininet1.pingall()
596 time2 = time.time()
597 if not pingResult:
Jon Hallf37d44d2017-05-24 10:37:30 -0700598 main.log.warn( "First pingall failed. Trying again..." )
Jon Hall5cf14d52015-07-16 12:15:19 -0700599 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700600 passMsg += " on the second try"
601 utilities.assert_equals(
602 expect=main.TRUE,
603 actual=pingResult,
Jon Hallf37d44d2017-05-24 10:37:30 -0700604 onpass=passMsg,
Jon Hall96091e62015-09-21 17:34:17 -0700605 onfail="Reactive Pingall failed, " +
606 "one or more ping pairs failed" )
607 main.log.info( "Time for pingall: %2f seconds" %
608 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700609 # timeout for fwd flows
610 time.sleep( 11 )
611 # uninstall onos-app-fwd
612 main.step( "Uninstall reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700613 node = main.activeNodes[ 0 ]
614 uninstallResult = main.CLIs[ node ].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700615 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
616 onpass="Uninstall fwd successful",
617 onfail="Uninstall fwd failed" )
618
619 main.step( "Check app ids" )
620 threads = []
621 appCheck2 = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700622 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700623 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700624 name="appToIDCheck-" + str( i ),
625 args=[] )
626 threads.append( t )
627 t.start()
628
629 for t in threads:
630 t.join()
631 appCheck2 = appCheck2 and t.result
632 if appCheck2 != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700633 node = main.activeNodes[ 0 ]
634 main.log.warn( main.CLIs[ node ].apps() )
635 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700636 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
637 onpass="App Ids seem to be correct",
638 onfail="Something is wrong with app Ids" )
639
640 main.step( "Add host intents via cli" )
641 intentIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700642 # TODO: move the host numbers to params
643 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700644 intentAddResult = True
645 hostResult = main.TRUE
646 for i in range( 8, 18 ):
647 main.log.info( "Adding host intent between h" + str( i ) +
648 " and h" + str( i + 10 ) )
649 host1 = "00:00:00:00:00:" + \
650 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
651 host2 = "00:00:00:00:00:" + \
652 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
653 # NOTE: getHost can return None
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700654 host1Dict = onosCli.getHost( host1 )
655 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700656 host1Id = None
657 host2Id = None
658 if host1Dict and host2Dict:
659 host1Id = host1Dict.get( 'id', None )
660 host2Id = host2Dict.get( 'id', None )
661 if host1Id and host2Id:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700662 nodeNum = ( i % len( main.activeNodes ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700663 node = main.activeNodes[ nodeNum ]
664 tmpId = main.CLIs[ node ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700665 if tmpId:
666 main.log.info( "Added intent with id: " + tmpId )
667 intentIds.append( tmpId )
668 else:
669 main.log.error( "addHostIntent returned: " +
670 repr( tmpId ) )
671 else:
672 main.log.error( "Error, getHost() failed for h" + str( i ) +
673 " and/or h" + str( i + 10 ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700674 node = main.activeNodes[ 0 ]
675 hosts = main.CLIs[ node ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700676 main.log.warn( "Hosts output: " )
677 try:
678 main.log.warn( json.dumps( json.loads( hosts ),
679 sort_keys=True,
680 indent=4,
681 separators=( ',', ': ' ) ) )
682 except ( ValueError, TypeError ):
683 main.log.warn( repr( hosts ) )
684 hostResult = main.FALSE
685 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
686 onpass="Found a host id for each host",
687 onfail="Error looking up host ids" )
688
689 intentStart = time.time()
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700690 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700691 main.log.info( "Submitted intents: " + str( intentIds ) )
692 main.log.info( "Intents in ONOS: " + str( onosIds ) )
693 for intent in intentIds:
694 if intent in onosIds:
695 pass # intent submitted is in onos
696 else:
697 intentAddResult = False
698 if intentAddResult:
699 intentStop = time.time()
700 else:
701 intentStop = None
702 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700703 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700704 intentStates = []
705 installedCheck = True
706 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
707 count = 0
708 try:
709 for intent in json.loads( intents ):
710 state = intent.get( 'state', None )
711 if "INSTALLED" not in state:
712 installedCheck = False
713 intentId = intent.get( 'id', None )
714 intentStates.append( ( intentId, state ) )
715 except ( ValueError, TypeError ):
716 main.log.exception( "Error parsing intents" )
717 # add submitted intents not in the store
718 tmplist = [ i for i, s in intentStates ]
719 missingIntents = False
720 for i in intentIds:
721 if i not in tmplist:
722 intentStates.append( ( i, " - " ) )
723 missingIntents = True
724 intentStates.sort()
725 for i, s in intentStates:
726 count += 1
727 main.log.info( "%-6s%-15s%-15s" %
728 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700729 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700730 try:
731 missing = False
732 if leaders:
733 parsedLeaders = json.loads( leaders )
734 main.log.warn( json.dumps( parsedLeaders,
735 sort_keys=True,
736 indent=4,
737 separators=( ',', ': ' ) ) )
738 # check for all intent partitions
739 topics = []
740 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700741 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700742 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700743 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700744 for topic in topics:
745 if topic not in ONOStopics:
746 main.log.error( "Error: " + topic +
747 " not in leaders" )
748 missing = True
749 else:
750 main.log.error( "leaders() returned None" )
751 except ( ValueError, TypeError ):
752 main.log.exception( "Error parsing leaders" )
753 main.log.error( repr( leaders ) )
754 # Check all nodes
755 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700756 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700757 response = main.CLIs[ i ].leaders( jsonFormat=False )
758 main.log.warn( str( main.CLIs[ i ].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700759 str( response ) )
760
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700761 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700762 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700763 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -0700764 parsedPartitions = json.loads( partitions )
765 main.log.warn( json.dumps( parsedPartitions,
766 sort_keys=True,
767 indent=4,
768 separators=( ',', ': ' ) ) )
769 # TODO check for a leader in all paritions
770 # TODO check for consistency among nodes
771 else:
772 main.log.error( "partitions() returned None" )
773 except ( ValueError, TypeError ):
774 main.log.exception( "Error parsing partitions" )
775 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700776 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700777 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700778 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -0700779 parsedPending = json.loads( pendingMap )
780 main.log.warn( json.dumps( parsedPending,
781 sort_keys=True,
782 indent=4,
783 separators=( ',', ': ' ) ) )
784 # TODO check something here?
785 else:
786 main.log.error( "pendingMap() returned None" )
787 except ( ValueError, TypeError ):
788 main.log.exception( "Error parsing pending map" )
789 main.log.error( repr( pendingMap ) )
790
791 intentAddResult = bool( intentAddResult and not missingIntents and
792 installedCheck )
793 if not intentAddResult:
794 main.log.error( "Error in pushing host intents to ONOS" )
795
796 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700797 for j in range( 100 ):
Jon Hall5cf14d52015-07-16 12:15:19 -0700798 correct = True
799 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700800 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700801 onosIds = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700802 ids = main.CLIs[ i ].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700803 onosIds.append( ids )
Jon Hallf37d44d2017-05-24 10:37:30 -0700804 main.log.debug( "Intents in " + main.CLIs[ i ].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700805 str( sorted( onosIds ) ) )
806 if sorted( ids ) != sorted( intentIds ):
807 main.log.warn( "Set of intent IDs doesn't match" )
808 correct = False
809 break
810 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700811 intents = json.loads( main.CLIs[ i ].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700812 for intent in intents:
813 if intent[ 'state' ] != "INSTALLED":
814 main.log.warn( "Intent " + intent[ 'id' ] +
815 " is " + intent[ 'state' ] )
816 correct = False
817 break
818 if correct:
819 break
820 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700821 time.sleep( 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700822 if not intentStop:
823 intentStop = time.time()
824 global gossipTime
825 gossipTime = intentStop - intentStart
826 main.log.info( "It took about " + str( gossipTime ) +
827 " seconds for all intents to appear in each node" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700828 gossipPeriod = int( main.params[ 'timers' ][ 'gossip' ] )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700829 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700830 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700831 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700832 onpass="ECM anti-entropy for intents worked within " +
833 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700834 onfail="Intent ECM anti-entropy took too long. " +
835 "Expected time:{}, Actual time:{}".format( maxGossipTime,
836 gossipTime ) )
837 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700838 intentAddResult = True
839
840 if not intentAddResult or "key" in pendingMap:
841 import time
842 installedCheck = True
843 main.log.info( "Sleeping 60 seconds to see if intents are found" )
844 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700845 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700846 main.log.info( "Submitted intents: " + str( intentIds ) )
847 main.log.info( "Intents in ONOS: " + str( onosIds ) )
848 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700849 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700850 intentStates = []
851 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
852 count = 0
853 try:
854 for intent in json.loads( intents ):
855 # Iter through intents of a node
856 state = intent.get( 'state', None )
857 if "INSTALLED" not in state:
858 installedCheck = False
859 intentId = intent.get( 'id', None )
860 intentStates.append( ( intentId, state ) )
861 except ( ValueError, TypeError ):
862 main.log.exception( "Error parsing intents" )
863 # add submitted intents not in the store
864 tmplist = [ i for i, s in intentStates ]
865 for i in intentIds:
866 if i not in tmplist:
867 intentStates.append( ( i, " - " ) )
868 intentStates.sort()
869 for i, s in intentStates:
870 count += 1
871 main.log.info( "%-6s%-15s%-15s" %
872 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700873 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700874 try:
875 missing = False
876 if leaders:
877 parsedLeaders = json.loads( leaders )
878 main.log.warn( json.dumps( parsedLeaders,
879 sort_keys=True,
880 indent=4,
881 separators=( ',', ': ' ) ) )
882 # check for all intent partitions
883 # check for election
884 topics = []
885 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700886 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700887 # FIXME: this should only be after we start the app
888 topics.append( "org.onosproject.election" )
889 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700890 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700891 for topic in topics:
892 if topic not in ONOStopics:
893 main.log.error( "Error: " + topic +
894 " not in leaders" )
895 missing = True
896 else:
897 main.log.error( "leaders() returned None" )
898 except ( ValueError, TypeError ):
899 main.log.exception( "Error parsing leaders" )
900 main.log.error( repr( leaders ) )
901 # Check all nodes
902 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700903 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700904 node = main.CLIs[ i ]
905 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -0700906 main.log.warn( str( node.name ) + " leaders output: \n" +
907 str( response ) )
908
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700909 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700910 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700911 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -0700912 parsedPartitions = json.loads( partitions )
913 main.log.warn( json.dumps( parsedPartitions,
914 sort_keys=True,
915 indent=4,
916 separators=( ',', ': ' ) ) )
917 # TODO check for a leader in all paritions
918 # TODO check for consistency among nodes
919 else:
920 main.log.error( "partitions() returned None" )
921 except ( ValueError, TypeError ):
922 main.log.exception( "Error parsing partitions" )
923 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700924 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700925 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700926 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -0700927 parsedPending = json.loads( pendingMap )
928 main.log.warn( json.dumps( parsedPending,
929 sort_keys=True,
930 indent=4,
931 separators=( ',', ': ' ) ) )
932 # TODO check something here?
933 else:
934 main.log.error( "pendingMap() returned None" )
935 except ( ValueError, TypeError ):
936 main.log.exception( "Error parsing pending map" )
937 main.log.error( repr( pendingMap ) )
938
939 def CASE4( self, main ):
940 """
941 Ping across added host intents
942 """
943 import json
944 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700945 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700946 assert main, "main not defined"
947 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700948 assert main.CLIs, "main.CLIs not defined"
949 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700950 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700951 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700952 "functionality and check the state of " +\
953 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700954
Jon Hallf37d44d2017-05-24 10:37:30 -0700955 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700956 main.step( "Check Intent state" )
957 installedCheck = False
958 loopCount = 0
959 while not installedCheck and loopCount < 40:
960 installedCheck = True
961 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700962 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700963 intentStates = []
964 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
965 count = 0
966 # Iter through intents of a node
967 try:
968 for intent in json.loads( intents ):
969 state = intent.get( 'state', None )
970 if "INSTALLED" not in state:
971 installedCheck = False
972 intentId = intent.get( 'id', None )
973 intentStates.append( ( intentId, state ) )
974 except ( ValueError, TypeError ):
975 main.log.exception( "Error parsing intents." )
976 # Print states
977 intentStates.sort()
978 for i, s in intentStates:
979 count += 1
980 main.log.info( "%-6s%-15s%-15s" %
981 ( str( count ), str( i ), str( s ) ) )
982 if not installedCheck:
983 time.sleep( 1 )
984 loopCount += 1
985 utilities.assert_equals( expect=True, actual=installedCheck,
986 onpass="Intents are all INSTALLED",
987 onfail="Intents are not all in " +
988 "INSTALLED state" )
989
Jon Hall9d2dcad2016-04-08 10:15:20 -0700990 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700991 PingResult = main.TRUE
992 for i in range( 8, 18 ):
993 ping = main.Mininet1.pingHost( src="h" + str( i ),
994 target="h" + str( i + 10 ) )
995 PingResult = PingResult and ping
996 if ping == main.FALSE:
997 main.log.warn( "Ping failed between h" + str( i ) +
998 " and h" + str( i + 10 ) )
999 elif ping == main.TRUE:
1000 main.log.info( "Ping test passed!" )
1001 # Don't set PingResult or you'd override failures
1002 if PingResult == main.FALSE:
1003 main.log.error(
1004 "Intents have not been installed correctly, pings failed." )
1005 # TODO: pretty print
1006 main.log.warn( "ONOS1 intents: " )
1007 try:
1008 tmpIntents = onosCli.intents()
1009 main.log.warn( json.dumps( json.loads( tmpIntents ),
1010 sort_keys=True,
1011 indent=4,
1012 separators=( ',', ': ' ) ) )
1013 except ( ValueError, TypeError ):
1014 main.log.warn( repr( tmpIntents ) )
1015 utilities.assert_equals(
1016 expect=main.TRUE,
1017 actual=PingResult,
1018 onpass="Intents have been installed correctly and pings work",
1019 onfail="Intents have not been installed correctly, pings failed." )
1020
Jon Hall5cf14d52015-07-16 12:15:19 -07001021 main.step( "Check leadership of topics" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001022 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001023 topicCheck = main.TRUE
1024 try:
1025 if leaders:
1026 parsedLeaders = json.loads( leaders )
1027 main.log.warn( json.dumps( parsedLeaders,
1028 sort_keys=True,
1029 indent=4,
1030 separators=( ',', ': ' ) ) )
1031 # check for all intent partitions
1032 # check for election
1033 # TODO: Look at Devices as topics now that it uses this system
1034 topics = []
1035 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001036 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001037 # FIXME: this should only be after we start the app
1038 # FIXME: topics.append( "org.onosproject.election" )
1039 # Print leaders output
1040 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -07001041 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -07001042 for topic in topics:
1043 if topic not in ONOStopics:
1044 main.log.error( "Error: " + topic +
1045 " not in leaders" )
1046 topicCheck = main.FALSE
1047 else:
1048 main.log.error( "leaders() returned None" )
1049 topicCheck = main.FALSE
1050 except ( ValueError, TypeError ):
1051 topicCheck = main.FALSE
1052 main.log.exception( "Error parsing leaders" )
1053 main.log.error( repr( leaders ) )
1054 # TODO: Check for a leader of these topics
1055 # Check all nodes
1056 if topicCheck:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001057 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001058 node = main.CLIs[ i ]
1059 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -07001060 main.log.warn( str( node.name ) + " leaders output: \n" +
1061 str( response ) )
1062
1063 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1064 onpass="intent Partitions is in leaders",
1065 onfail="Some topics were lost " )
1066 # Print partitions
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001067 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001068 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001069 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -07001070 parsedPartitions = json.loads( partitions )
1071 main.log.warn( json.dumps( parsedPartitions,
1072 sort_keys=True,
1073 indent=4,
1074 separators=( ',', ': ' ) ) )
1075 # TODO check for a leader in all paritions
1076 # TODO check for consistency among nodes
1077 else:
1078 main.log.error( "partitions() returned None" )
1079 except ( ValueError, TypeError ):
1080 main.log.exception( "Error parsing partitions" )
1081 main.log.error( repr( partitions ) )
1082 # Print Pending Map
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001083 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001084 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001085 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -07001086 parsedPending = json.loads( pendingMap )
1087 main.log.warn( json.dumps( parsedPending,
1088 sort_keys=True,
1089 indent=4,
1090 separators=( ',', ': ' ) ) )
1091 # TODO check something here?
1092 else:
1093 main.log.error( "pendingMap() returned None" )
1094 except ( ValueError, TypeError ):
1095 main.log.exception( "Error parsing pending map" )
1096 main.log.error( repr( pendingMap ) )
1097
1098 if not installedCheck:
1099 main.log.info( "Waiting 60 seconds to see if the state of " +
1100 "intents change" )
1101 time.sleep( 60 )
1102 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001103 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001104 intentStates = []
1105 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1106 count = 0
1107 # Iter through intents of a node
1108 try:
1109 for intent in json.loads( intents ):
1110 state = intent.get( 'state', None )
1111 if "INSTALLED" not in state:
1112 installedCheck = False
1113 intentId = intent.get( 'id', None )
1114 intentStates.append( ( intentId, state ) )
1115 except ( ValueError, TypeError ):
1116 main.log.exception( "Error parsing intents." )
1117 intentStates.sort()
1118 for i, s in intentStates:
1119 count += 1
1120 main.log.info( "%-6s%-15s%-15s" %
1121 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001122 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001123 try:
1124 missing = False
1125 if leaders:
1126 parsedLeaders = json.loads( leaders )
1127 main.log.warn( json.dumps( parsedLeaders,
1128 sort_keys=True,
1129 indent=4,
1130 separators=( ',', ': ' ) ) )
1131 # check for all intent partitions
1132 # check for election
1133 topics = []
1134 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001135 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001136 # FIXME: this should only be after we start the app
1137 topics.append( "org.onosproject.election" )
1138 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -07001139 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -07001140 for topic in topics:
1141 if topic not in ONOStopics:
1142 main.log.error( "Error: " + topic +
1143 " not in leaders" )
1144 missing = True
1145 else:
1146 main.log.error( "leaders() returned None" )
1147 except ( ValueError, TypeError ):
1148 main.log.exception( "Error parsing leaders" )
1149 main.log.error( repr( leaders ) )
1150 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001151 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001152 node = main.CLIs[ i ]
1153 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -07001154 main.log.warn( str( node.name ) + " leaders output: \n" +
1155 str( response ) )
1156
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001157 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001158 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001159 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -07001160 parsedPartitions = json.loads( partitions )
1161 main.log.warn( json.dumps( parsedPartitions,
1162 sort_keys=True,
1163 indent=4,
1164 separators=( ',', ': ' ) ) )
1165 # TODO check for a leader in all paritions
1166 # TODO check for consistency among nodes
1167 else:
1168 main.log.error( "partitions() returned None" )
1169 except ( ValueError, TypeError ):
1170 main.log.exception( "Error parsing partitions" )
1171 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001172 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001173 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001174 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -07001175 parsedPending = json.loads( pendingMap )
1176 main.log.warn( json.dumps( parsedPending,
1177 sort_keys=True,
1178 indent=4,
1179 separators=( ',', ': ' ) ) )
1180 # TODO check something here?
1181 else:
1182 main.log.error( "pendingMap() returned None" )
1183 except ( ValueError, TypeError ):
1184 main.log.exception( "Error parsing pending map" )
1185 main.log.error( repr( pendingMap ) )
1186 # Print flowrules
Jon Hall41d39f12016-04-11 22:54:35 -07001187 main.log.debug( onosCli.flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001188 main.step( "Wait a minute then ping again" )
1189 # the wait is above
1190 PingResult = main.TRUE
1191 for i in range( 8, 18 ):
1192 ping = main.Mininet1.pingHost( src="h" + str( i ),
1193 target="h" + str( i + 10 ) )
1194 PingResult = PingResult and ping
1195 if ping == main.FALSE:
1196 main.log.warn( "Ping failed between h" + str( i ) +
1197 " and h" + str( i + 10 ) )
1198 elif ping == main.TRUE:
1199 main.log.info( "Ping test passed!" )
1200 # Don't set PingResult or you'd override failures
1201 if PingResult == main.FALSE:
1202 main.log.error(
1203 "Intents have not been installed correctly, pings failed." )
1204 # TODO: pretty print
1205 main.log.warn( "ONOS1 intents: " )
1206 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001207 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001208 main.log.warn( json.dumps( json.loads( tmpIntents ),
1209 sort_keys=True,
1210 indent=4,
1211 separators=( ',', ': ' ) ) )
1212 except ( ValueError, TypeError ):
1213 main.log.warn( repr( tmpIntents ) )
1214 utilities.assert_equals(
1215 expect=main.TRUE,
1216 actual=PingResult,
1217 onpass="Intents have been installed correctly and pings work",
1218 onfail="Intents have not been installed correctly, pings failed." )
1219
1220 def CASE5( self, main ):
1221 """
1222 Reading state of ONOS
1223 """
1224 import json
1225 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001226 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001227 assert main, "main not defined"
1228 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001229 assert main.CLIs, "main.CLIs not defined"
1230 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001231
1232 main.case( "Setting up and gathering data for current state" )
1233 # The general idea for this test case is to pull the state of
1234 # ( intents,flows, topology,... ) from each ONOS node
1235 # We can then compare them with each other and also with past states
1236
1237 main.step( "Check that each switch has a master" )
1238 global mastershipState
1239 mastershipState = '[]'
1240
1241 # Assert that each device has a master
1242 rolesNotNull = main.TRUE
1243 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001244 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001245 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001246 name="rolesNotNull-" + str( i ),
1247 args=[] )
1248 threads.append( t )
1249 t.start()
1250
1251 for t in threads:
1252 t.join()
1253 rolesNotNull = rolesNotNull and t.result
1254 utilities.assert_equals(
1255 expect=main.TRUE,
1256 actual=rolesNotNull,
1257 onpass="Each device has a master",
1258 onfail="Some devices don't have a master assigned" )
1259
1260 main.step( "Get the Mastership of each switch from each controller" )
1261 ONOSMastership = []
1262 mastershipCheck = main.FALSE
1263 consistentMastership = True
1264 rolesResults = True
1265 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001266 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001267 t = main.Thread( target=main.CLIs[ i ].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001268 name="roles-" + str( i ),
1269 args=[] )
1270 threads.append( t )
1271 t.start()
1272
1273 for t in threads:
1274 t.join()
1275 ONOSMastership.append( t.result )
1276
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001277 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001278 node = str( main.activeNodes[ i ] + 1 )
1279 if not ONOSMastership[ i ] or "Error" in ONOSMastership[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001280 main.log.error( "Error in getting ONOS" + node + " roles" )
1281 main.log.warn( "ONOS" + node + " mastership response: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001282 repr( ONOSMastership[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001283 rolesResults = False
1284 utilities.assert_equals(
1285 expect=True,
1286 actual=rolesResults,
1287 onpass="No error in reading roles output",
1288 onfail="Error in reading roles from ONOS" )
1289
1290 main.step( "Check for consistency in roles from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001291 if all( [ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001292 main.log.info(
1293 "Switch roles are consistent across all ONOS nodes" )
1294 else:
1295 consistentMastership = False
1296 utilities.assert_equals(
1297 expect=True,
1298 actual=consistentMastership,
1299 onpass="Switch roles are consistent across all ONOS nodes",
1300 onfail="ONOS nodes have different views of switch roles" )
1301
1302 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001303 for i in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001304 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001305 try:
1306 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001307 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001308 json.dumps(
1309 json.loads( ONOSMastership[ i ] ),
1310 sort_keys=True,
1311 indent=4,
1312 separators=( ',', ': ' ) ) )
1313 except ( ValueError, TypeError ):
1314 main.log.warn( repr( ONOSMastership[ i ] ) )
1315 elif rolesResults and consistentMastership:
1316 mastershipCheck = main.TRUE
1317 mastershipState = ONOSMastership[ 0 ]
1318
1319 main.step( "Get the intents from each controller" )
1320 global intentState
1321 intentState = []
1322 ONOSIntents = []
1323 intentCheck = main.FALSE
1324 consistentIntents = True
1325 intentsResults = True
1326 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001327 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001328 t = main.Thread( target=main.CLIs[ i ].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001329 name="intents-" + str( i ),
1330 args=[],
1331 kwargs={ 'jsonFormat': True } )
1332 threads.append( t )
1333 t.start()
1334
1335 for t in threads:
1336 t.join()
1337 ONOSIntents.append( t.result )
1338
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001339 for i in range( len( ONOSIntents ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001340 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001341 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001342 main.log.error( "Error in getting ONOS" + node + " intents" )
1343 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001344 repr( ONOSIntents[ i ] ) )
1345 intentsResults = False
1346 utilities.assert_equals(
1347 expect=True,
1348 actual=intentsResults,
1349 onpass="No error in reading intents output",
1350 onfail="Error in reading intents from ONOS" )
1351
1352 main.step( "Check for consistency in Intents from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001353 if all( [ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001354 main.log.info( "Intents are consistent across all ONOS " +
1355 "nodes" )
1356 else:
1357 consistentIntents = False
1358 main.log.error( "Intents not consistent" )
1359 utilities.assert_equals(
1360 expect=True,
1361 actual=consistentIntents,
1362 onpass="Intents are consistent across all ONOS nodes",
1363 onfail="ONOS nodes have different views of intents" )
1364
1365 if intentsResults:
1366 # Try to make it easy to figure out what is happening
1367 #
1368 # Intent ONOS1 ONOS2 ...
1369 # 0x01 INSTALLED INSTALLING
1370 # ... ... ...
1371 # ... ... ...
1372 title = " Id"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001373 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001374 title += " " * 10 + "ONOS" + str( n + 1 )
1375 main.log.warn( title )
1376 # get all intent keys in the cluster
1377 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001378 try:
1379 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001380 for nodeStr in ONOSIntents:
1381 node = json.loads( nodeStr )
1382 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001383 keys.append( intent.get( 'id' ) )
1384 keys = set( keys )
1385 # For each intent key, print the state on each node
1386 for key in keys:
1387 row = "%-13s" % key
1388 for nodeStr in ONOSIntents:
1389 node = json.loads( nodeStr )
1390 for intent in node:
1391 if intent.get( 'id', "Error" ) == key:
1392 row += "%-15s" % intent.get( 'state' )
1393 main.log.warn( row )
1394 # End of intent state table
1395 except ValueError as e:
1396 main.log.exception( e )
1397 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001398
1399 if intentsResults and not consistentIntents:
1400 # print the json objects
Jon Hallf37d44d2017-05-24 10:37:30 -07001401 n = str( main.activeNodes[ -1 ] + 1 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001402 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001403 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1404 sort_keys=True,
1405 indent=4,
1406 separators=( ',', ': ' ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001407 for i in range( len( ONOSIntents ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001408 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001409 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001410 main.log.debug( "ONOS" + node + " intents: " )
Jon Hallf37d44d2017-05-24 10:37:30 -07001411 main.log.debug( json.dumps( json.loads( ONOSIntents[ i ] ),
Jon Hall5cf14d52015-07-16 12:15:19 -07001412 sort_keys=True,
1413 indent=4,
1414 separators=( ',', ': ' ) ) )
1415 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001416 main.log.debug( "ONOS" + node + " intents match ONOS" +
1417 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001418 elif intentsResults and consistentIntents:
1419 intentCheck = main.TRUE
1420 intentState = ONOSIntents[ 0 ]
1421
1422 main.step( "Get the flows from each controller" )
1423 global flowState
1424 flowState = []
1425 ONOSFlows = []
1426 ONOSFlowsJson = []
1427 flowCheck = main.FALSE
1428 consistentFlows = True
1429 flowsResults = True
1430 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001431 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001432 t = main.Thread( target=main.CLIs[ i ].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001433 name="flows-" + str( i ),
1434 args=[],
1435 kwargs={ 'jsonFormat': True } )
1436 threads.append( t )
1437 t.start()
1438
1439 # NOTE: Flows command can take some time to run
Jon Hallf37d44d2017-05-24 10:37:30 -07001440 time.sleep( 30 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001441 for t in threads:
1442 t.join()
1443 result = t.result
1444 ONOSFlows.append( result )
1445
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001446 for i in range( len( ONOSFlows ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001447 num = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001448 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1449 main.log.error( "Error in getting ONOS" + num + " flows" )
1450 main.log.warn( "ONOS" + num + " flows response: " +
1451 repr( ONOSFlows[ i ] ) )
1452 flowsResults = False
1453 ONOSFlowsJson.append( None )
1454 else:
1455 try:
1456 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1457 except ( ValueError, TypeError ):
1458 # FIXME: change this to log.error?
1459 main.log.exception( "Error in parsing ONOS" + num +
1460 " response as json." )
1461 main.log.error( repr( ONOSFlows[ i ] ) )
1462 ONOSFlowsJson.append( None )
1463 flowsResults = False
1464 utilities.assert_equals(
1465 expect=True,
1466 actual=flowsResults,
1467 onpass="No error in reading flows output",
1468 onfail="Error in reading flows from ONOS" )
1469
1470 main.step( "Check for consistency in Flows from each controller" )
1471 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1472 if all( tmp ):
1473 main.log.info( "Flow count is consistent across all ONOS nodes" )
1474 else:
1475 consistentFlows = False
1476 utilities.assert_equals(
1477 expect=True,
1478 actual=consistentFlows,
1479 onpass="The flow count is consistent across all ONOS nodes",
1480 onfail="ONOS nodes have different flow counts" )
1481
1482 if flowsResults and not consistentFlows:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001483 for i in range( len( ONOSFlows ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001484 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001485 try:
1486 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001487 "ONOS" + node + " flows: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001488 json.dumps( json.loads( ONOSFlows[ i ] ), sort_keys=True,
Jon Hall5cf14d52015-07-16 12:15:19 -07001489 indent=4, separators=( ',', ': ' ) ) )
1490 except ( ValueError, TypeError ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001491 main.log.warn( "ONOS" + node + " flows: " +
1492 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001493 elif flowsResults and consistentFlows:
1494 flowCheck = main.TRUE
1495 flowState = ONOSFlows[ 0 ]
1496
1497 main.step( "Get the OF Table entries" )
1498 global flows
1499 flows = []
1500 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001501 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001502 if flowCheck == main.FALSE:
1503 for table in flows:
1504 main.log.warn( table )
1505 # TODO: Compare switch flow tables with ONOS flow tables
1506
1507 main.step( "Start continuous pings" )
1508 main.Mininet2.pingLong(
1509 src=main.params[ 'PING' ][ 'source1' ],
1510 target=main.params[ 'PING' ][ 'target1' ],
1511 pingTime=500 )
1512 main.Mininet2.pingLong(
1513 src=main.params[ 'PING' ][ 'source2' ],
1514 target=main.params[ 'PING' ][ 'target2' ],
1515 pingTime=500 )
1516 main.Mininet2.pingLong(
1517 src=main.params[ 'PING' ][ 'source3' ],
1518 target=main.params[ 'PING' ][ 'target3' ],
1519 pingTime=500 )
1520 main.Mininet2.pingLong(
1521 src=main.params[ 'PING' ][ 'source4' ],
1522 target=main.params[ 'PING' ][ 'target4' ],
1523 pingTime=500 )
1524 main.Mininet2.pingLong(
1525 src=main.params[ 'PING' ][ 'source5' ],
1526 target=main.params[ 'PING' ][ 'target5' ],
1527 pingTime=500 )
1528 main.Mininet2.pingLong(
1529 src=main.params[ 'PING' ][ 'source6' ],
1530 target=main.params[ 'PING' ][ 'target6' ],
1531 pingTime=500 )
1532 main.Mininet2.pingLong(
1533 src=main.params[ 'PING' ][ 'source7' ],
1534 target=main.params[ 'PING' ][ 'target7' ],
1535 pingTime=500 )
1536 main.Mininet2.pingLong(
1537 src=main.params[ 'PING' ][ 'source8' ],
1538 target=main.params[ 'PING' ][ 'target8' ],
1539 pingTime=500 )
1540 main.Mininet2.pingLong(
1541 src=main.params[ 'PING' ][ 'source9' ],
1542 target=main.params[ 'PING' ][ 'target9' ],
1543 pingTime=500 )
1544 main.Mininet2.pingLong(
1545 src=main.params[ 'PING' ][ 'source10' ],
1546 target=main.params[ 'PING' ][ 'target10' ],
1547 pingTime=500 )
1548
1549 main.step( "Collecting topology information from ONOS" )
1550 devices = []
1551 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001552 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001553 t = main.Thread( target=main.CLIs[ i ].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001554 name="devices-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001555 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001556 threads.append( t )
1557 t.start()
1558
1559 for t in threads:
1560 t.join()
1561 devices.append( t.result )
1562 hosts = []
1563 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001564 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001565 t = main.Thread( target=main.CLIs[ i ].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001566 name="hosts-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001567 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001568 threads.append( t )
1569 t.start()
1570
1571 for t in threads:
1572 t.join()
1573 try:
1574 hosts.append( json.loads( t.result ) )
1575 except ( ValueError, TypeError ):
1576 # FIXME: better handling of this, print which node
1577 # Maybe use thread name?
1578 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001579 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001580 hosts.append( None )
1581
1582 ports = []
1583 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001584 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001585 t = main.Thread( target=main.CLIs[ i ].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001586 name="ports-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001587 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001588 threads.append( t )
1589 t.start()
1590
1591 for t in threads:
1592 t.join()
1593 ports.append( t.result )
1594 links = []
1595 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001596 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001597 t = main.Thread( target=main.CLIs[ i ].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001598 name="links-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001599 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001600 threads.append( t )
1601 t.start()
1602
1603 for t in threads:
1604 t.join()
1605 links.append( t.result )
1606 clusters = []
1607 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001608 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001609 t = main.Thread( target=main.CLIs[ i ].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001610 name="clusters-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001611 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001612 threads.append( t )
1613 t.start()
1614
1615 for t in threads:
1616 t.join()
1617 clusters.append( t.result )
1618 # Compare json objects for hosts and dataplane clusters
1619
1620 # hosts
1621 main.step( "Host view is consistent across ONOS nodes" )
1622 consistentHostsResult = main.TRUE
1623 for controller in range( len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001624 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001625 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001626 if hosts[ controller ] == hosts[ 0 ]:
1627 continue
1628 else: # hosts not consistent
1629 main.log.error( "hosts from ONOS" +
1630 controllerStr +
1631 " is inconsistent with ONOS1" )
1632 main.log.warn( repr( hosts[ controller ] ) )
1633 consistentHostsResult = main.FALSE
1634
1635 else:
1636 main.log.error( "Error in getting ONOS hosts from ONOS" +
1637 controllerStr )
1638 consistentHostsResult = main.FALSE
1639 main.log.warn( "ONOS" + controllerStr +
1640 " hosts response: " +
1641 repr( hosts[ controller ] ) )
1642 utilities.assert_equals(
1643 expect=main.TRUE,
1644 actual=consistentHostsResult,
1645 onpass="Hosts view is consistent across all ONOS nodes",
1646 onfail="ONOS nodes have different views of hosts" )
1647
1648 main.step( "Each host has an IP address" )
1649 ipResult = main.TRUE
1650 for controller in range( 0, len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001651 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001652 if hosts[ controller ]:
1653 for host in hosts[ controller ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07001654 if not host.get( 'ipAddresses', [] ):
Jon Hallf3d16e72015-12-16 17:45:08 -08001655 main.log.error( "Error with host ips on controller" +
1656 controllerStr + ": " + str( host ) )
1657 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001658 utilities.assert_equals(
1659 expect=main.TRUE,
1660 actual=ipResult,
1661 onpass="The ips of the hosts aren't empty",
1662 onfail="The ip of at least one host is missing" )
1663
1664 # Strongly connected clusters of devices
1665 main.step( "Cluster view is consistent across ONOS nodes" )
1666 consistentClustersResult = main.TRUE
1667 for controller in range( len( clusters ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001668 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001669 if "Error" not in clusters[ controller ]:
1670 if clusters[ controller ] == clusters[ 0 ]:
1671 continue
1672 else: # clusters not consistent
1673 main.log.error( "clusters from ONOS" + controllerStr +
1674 " is inconsistent with ONOS1" )
1675 consistentClustersResult = main.FALSE
1676
1677 else:
1678 main.log.error( "Error in getting dataplane clusters " +
1679 "from ONOS" + controllerStr )
1680 consistentClustersResult = main.FALSE
1681 main.log.warn( "ONOS" + controllerStr +
1682 " clusters response: " +
1683 repr( clusters[ controller ] ) )
1684 utilities.assert_equals(
1685 expect=main.TRUE,
1686 actual=consistentClustersResult,
1687 onpass="Clusters view is consistent across all ONOS nodes",
1688 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001689 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001690 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001691
Jon Hall5cf14d52015-07-16 12:15:19 -07001692 # there should always only be one cluster
1693 main.step( "Cluster view correct across ONOS nodes" )
1694 try:
1695 numClusters = len( json.loads( clusters[ 0 ] ) )
1696 except ( ValueError, TypeError ):
1697 main.log.exception( "Error parsing clusters[0]: " +
1698 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001699 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001700 clusterResults = main.FALSE
1701 if numClusters == 1:
1702 clusterResults = main.TRUE
1703 utilities.assert_equals(
1704 expect=1,
1705 actual=numClusters,
1706 onpass="ONOS shows 1 SCC",
1707 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1708
1709 main.step( "Comparing ONOS topology to MN" )
1710 devicesResults = main.TRUE
1711 linksResults = main.TRUE
1712 hostsResults = main.TRUE
1713 mnSwitches = main.Mininet1.getSwitches()
1714 mnLinks = main.Mininet1.getLinks()
1715 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001716 for controller in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001717 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001718 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001719 "Error" not in devices[ controller ] and\
1720 "Error" not in ports[ controller ]:
1721 currentDevicesResult = main.Mininet1.compareSwitches(
1722 mnSwitches,
1723 json.loads( devices[ controller ] ),
1724 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001725 else:
1726 currentDevicesResult = main.FALSE
1727 utilities.assert_equals( expect=main.TRUE,
1728 actual=currentDevicesResult,
1729 onpass="ONOS" + controllerStr +
1730 " Switches view is correct",
1731 onfail="ONOS" + controllerStr +
1732 " Switches view is incorrect" )
1733 if links[ controller ] and "Error" not in links[ controller ]:
1734 currentLinksResult = main.Mininet1.compareLinks(
1735 mnSwitches, mnLinks,
1736 json.loads( links[ controller ] ) )
1737 else:
1738 currentLinksResult = main.FALSE
1739 utilities.assert_equals( expect=main.TRUE,
1740 actual=currentLinksResult,
1741 onpass="ONOS" + controllerStr +
1742 " links view is correct",
1743 onfail="ONOS" + controllerStr +
1744 " links view is incorrect" )
1745
Jon Hall657cdf62015-12-17 14:40:51 -08001746 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001747 currentHostsResult = main.Mininet1.compareHosts(
1748 mnHosts,
1749 hosts[ controller ] )
1750 else:
1751 currentHostsResult = main.FALSE
1752 utilities.assert_equals( expect=main.TRUE,
1753 actual=currentHostsResult,
1754 onpass="ONOS" + controllerStr +
1755 " hosts exist in Mininet",
1756 onfail="ONOS" + controllerStr +
1757 " hosts don't match Mininet" )
1758
1759 devicesResults = devicesResults and currentDevicesResult
1760 linksResults = linksResults and currentLinksResult
1761 hostsResults = hostsResults and currentHostsResult
1762
1763 main.step( "Device information is correct" )
1764 utilities.assert_equals(
1765 expect=main.TRUE,
1766 actual=devicesResults,
1767 onpass="Device information is correct",
1768 onfail="Device information is incorrect" )
1769
1770 main.step( "Links are correct" )
1771 utilities.assert_equals(
1772 expect=main.TRUE,
1773 actual=linksResults,
1774 onpass="Link are correct",
1775 onfail="Links are incorrect" )
1776
1777 main.step( "Hosts are correct" )
1778 utilities.assert_equals(
1779 expect=main.TRUE,
1780 actual=hostsResults,
1781 onpass="Hosts are correct",
1782 onfail="Hosts are incorrect" )
1783
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001784 def CASE61( self, main ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001785 """
1786 The Failure case.
1787 """
Jon Halle1a3b752015-07-22 13:02:46 -07001788 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001789 assert main, "main not defined"
1790 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001791 assert main.CLIs, "main.CLIs not defined"
1792 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001793 main.case( "Stop minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -07001794
1795 main.step( "Checking ONOS Logs for errors" )
1796 for node in main.nodes:
1797 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1798 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1799
Jon Hall3b489db2015-10-05 14:38:37 -07001800 n = len( main.nodes ) # Number of nodes
1801 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
1802 main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
1803 if n > 3:
1804 main.kill.append( p - 1 )
1805 # NOTE: This only works for cluster sizes of 3,5, or 7.
1806
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001807 main.step( "Stopping " + str( len( main.kill ) ) + " ONOS nodes" )
Jon Hall3b489db2015-10-05 14:38:37 -07001808 killResults = main.TRUE
1809 for i in main.kill:
1810 killResults = killResults and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001811 main.ONOSbench.onosStop( main.nodes[ i ].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001812 main.activeNodes.remove( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001813 utilities.assert_equals( expect=main.TRUE, actual=killResults,
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001814 onpass="ONOS nodes stopped successfully",
1815 onfail="ONOS nodes NOT successfully stopped" )
1816
Jon Halld2871c22016-07-26 11:01:14 -07001817 main.step( "Checking ONOS nodes" )
1818 nodeResults = utilities.retry( main.HA.nodesCheck,
1819 False,
Jon Hallf37d44d2017-05-24 10:37:30 -07001820 args=[ main.activeNodes ],
Jon Halld2871c22016-07-26 11:01:14 -07001821 sleep=15,
1822 attempts=5 )
1823
1824 utilities.assert_equals( expect=True, actual=nodeResults,
1825 onpass="Nodes check successful",
1826 onfail="Nodes check NOT successful" )
1827
1828 if not nodeResults:
1829 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001830 cli = main.CLIs[ i ]
Jon Halld2871c22016-07-26 11:01:14 -07001831 main.log.debug( "{} components not ACTIVE: \n{}".format(
1832 cli.name,
1833 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
1834 main.log.error( "Failed to start ONOS, stopping test" )
1835 main.cleanup()
1836 main.exit()
1837
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001838 def CASE62( self, main ):
1839 """
1840 The bring up stopped nodes
1841 """
1842 import time
1843 assert main.numCtrls, "main.numCtrls not defined"
1844 assert main, "main not defined"
1845 assert utilities.assert_equals, "utilities.assert_equals not defined"
1846 assert main.CLIs, "main.CLIs not defined"
1847 assert main.nodes, "main.nodes not defined"
1848 assert main.kill, "main.kill not defined"
1849 main.case( "Restart minority of ONOS nodes" )
1850
1851 main.step( "Restarting " + str( len( main.kill ) ) + " ONOS nodes" )
1852 startResults = main.TRUE
1853 restartTime = time.time()
1854 for i in main.kill:
1855 startResults = startResults and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001856 main.ONOSbench.onosStart( main.nodes[ i ].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001857 utilities.assert_equals( expect=main.TRUE, actual=startResults,
1858 onpass="ONOS nodes started successfully",
1859 onfail="ONOS nodes NOT successfully started" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001860
1861 main.step( "Checking if ONOS is up yet" )
1862 count = 0
1863 onosIsupResult = main.FALSE
1864 while onosIsupResult == main.FALSE and count < 10:
Jon Hall3b489db2015-10-05 14:38:37 -07001865 onosIsupResult = main.TRUE
1866 for i in main.kill:
1867 onosIsupResult = onosIsupResult and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001868 main.ONOSbench.isup( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001869 count = count + 1
Jon Hall5cf14d52015-07-16 12:15:19 -07001870 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1871 onpass="ONOS restarted successfully",
1872 onfail="ONOS restart NOT successful" )
1873
Jon Halle1a3b752015-07-22 13:02:46 -07001874 main.step( "Restarting ONOS main.CLIs" )
Jon Hall3b489db2015-10-05 14:38:37 -07001875 cliResults = main.TRUE
1876 for i in main.kill:
1877 cliResults = cliResults and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001878 main.CLIs[ i ].startOnosCli( main.nodes[ i ].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001879 main.activeNodes.append( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001880 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1881 onpass="ONOS cli restarted",
1882 onfail="ONOS cli did not restart" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001883 main.activeNodes.sort()
1884 try:
1885 assert list( set( main.activeNodes ) ) == main.activeNodes,\
1886 "List of active nodes has duplicates, this likely indicates something was run out of order"
1887 except AssertionError:
1888 main.log.exception( "" )
1889 main.cleanup()
1890 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -07001891
1892 # Grab the time of restart so we chan check how long the gossip
1893 # protocol has had time to work
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001894 main.restartTime = time.time() - restartTime
Jon Hall5cf14d52015-07-16 12:15:19 -07001895 main.log.debug( "Restart time: " + str( main.restartTime ) )
Jon Halld2871c22016-07-26 11:01:14 -07001896
1897 main.step( "Checking ONOS nodes" )
1898 nodeResults = utilities.retry( main.HA.nodesCheck,
1899 False,
Jon Hallf37d44d2017-05-24 10:37:30 -07001900 args=[ main.activeNodes ],
Jon Halld2871c22016-07-26 11:01:14 -07001901 sleep=15,
1902 attempts=5 )
1903
1904 utilities.assert_equals( expect=True, actual=nodeResults,
1905 onpass="Nodes check successful",
1906 onfail="Nodes check NOT successful" )
1907
1908 if not nodeResults:
1909 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001910 cli = main.CLIs[ i ]
Jon Halld2871c22016-07-26 11:01:14 -07001911 main.log.debug( "{} components not ACTIVE: \n{}".format(
1912 cli.name,
1913 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
1914 main.log.error( "Failed to start ONOS, stopping test" )
1915 main.cleanup()
1916 main.exit()
Jon Hallf37d44d2017-05-24 10:37:30 -07001917 node = main.activeNodes[ 0 ]
1918 main.log.debug( main.CLIs[ node ].nodes( jsonFormat=False ) )
1919 main.log.debug( main.CLIs[ node ].leaders( jsonFormat=False ) )
1920 main.log.debug( main.CLIs[ node ].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001921
Jon Halla440e872016-03-31 15:15:50 -07001922 main.step( "Rerun for election on the node(s) that were killed" )
1923 runResults = main.TRUE
1924 for i in main.kill:
1925 runResults = runResults and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001926 main.CLIs[ i ].electionTestRun()
Jon Halla440e872016-03-31 15:15:50 -07001927 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1928 onpass="ONOS nodes reran for election topic",
1929 onfail="Errror rerunning for election" )
1930
Jon Hall5cf14d52015-07-16 12:15:19 -07001931 def CASE7( self, main ):
1932 """
1933 Check state after ONOS failure
1934 """
1935 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001936 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001937 assert main, "main not defined"
1938 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001939 assert main.CLIs, "main.CLIs not defined"
1940 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001941 try:
1942 main.kill
1943 except AttributeError:
1944 main.kill = []
1945
Jon Hall5cf14d52015-07-16 12:15:19 -07001946 main.case( "Running ONOS Constant State Tests" )
1947
1948 main.step( "Check that each switch has a master" )
1949 # Assert that each device has a master
1950 rolesNotNull = main.TRUE
1951 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001952 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001953 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001954 name="rolesNotNull-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001955 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001956 threads.append( t )
1957 t.start()
1958
1959 for t in threads:
1960 t.join()
1961 rolesNotNull = rolesNotNull and t.result
1962 utilities.assert_equals(
1963 expect=main.TRUE,
1964 actual=rolesNotNull,
1965 onpass="Each device has a master",
1966 onfail="Some devices don't have a master assigned" )
1967
1968 main.step( "Read device roles from ONOS" )
1969 ONOSMastership = []
Jon Halla440e872016-03-31 15:15:50 -07001970 mastershipCheck = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001971 consistentMastership = True
1972 rolesResults = True
1973 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001974 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001975 t = main.Thread( target=main.CLIs[ i ].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001976 name="roles-" + str( i ),
1977 args=[] )
1978 threads.append( t )
1979 t.start()
1980
1981 for t in threads:
1982 t.join()
1983 ONOSMastership.append( t.result )
1984
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001985 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001986 node = str( main.activeNodes[ i ] + 1 )
1987 if not ONOSMastership[ i ] or "Error" in ONOSMastership[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001988 main.log.error( "Error in getting ONOS" + node + " roles" )
1989 main.log.warn( "ONOS" + node + " mastership response: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001990 repr( ONOSMastership[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001991 rolesResults = False
1992 utilities.assert_equals(
1993 expect=True,
1994 actual=rolesResults,
1995 onpass="No error in reading roles output",
1996 onfail="Error in reading roles from ONOS" )
1997
1998 main.step( "Check for consistency in roles from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001999 if all( [ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002000 main.log.info(
2001 "Switch roles are consistent across all ONOS nodes" )
2002 else:
2003 consistentMastership = False
2004 utilities.assert_equals(
2005 expect=True,
2006 actual=consistentMastership,
2007 onpass="Switch roles are consistent across all ONOS nodes",
2008 onfail="ONOS nodes have different views of switch roles" )
2009
2010 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002011 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002012 node = str( main.activeNodes[ i ] + 1 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002013 main.log.warn( "ONOS" + node + " roles: ",
2014 json.dumps( json.loads( ONOSMastership[ i ] ),
2015 sort_keys=True,
2016 indent=4,
2017 separators=( ',', ': ' ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002018
2019 # NOTE: we expect mastership to change on controller failure
Jon Hall5cf14d52015-07-16 12:15:19 -07002020
2021 main.step( "Get the intents and compare across all nodes" )
2022 ONOSIntents = []
2023 intentCheck = main.FALSE
2024 consistentIntents = True
2025 intentsResults = True
2026 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002027 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002028 t = main.Thread( target=main.CLIs[ i ].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07002029 name="intents-" + str( i ),
2030 args=[],
2031 kwargs={ 'jsonFormat': True } )
2032 threads.append( t )
2033 t.start()
2034
2035 for t in threads:
2036 t.join()
2037 ONOSIntents.append( t.result )
2038
Jon Hallf37d44d2017-05-24 10:37:30 -07002039 for i in range( len( ONOSIntents ) ):
2040 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002041 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002042 main.log.error( "Error in getting ONOS" + node + " intents" )
2043 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002044 repr( ONOSIntents[ i ] ) )
2045 intentsResults = False
2046 utilities.assert_equals(
2047 expect=True,
2048 actual=intentsResults,
2049 onpass="No error in reading intents output",
2050 onfail="Error in reading intents from ONOS" )
2051
2052 main.step( "Check for consistency in Intents from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002053 if all( [ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002054 main.log.info( "Intents are consistent across all ONOS " +
2055 "nodes" )
2056 else:
2057 consistentIntents = False
2058
2059 # Try to make it easy to figure out what is happening
2060 #
2061 # Intent ONOS1 ONOS2 ...
2062 # 0x01 INSTALLED INSTALLING
2063 # ... ... ...
2064 # ... ... ...
2065 title = " ID"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002066 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002067 title += " " * 10 + "ONOS" + str( n + 1 )
2068 main.log.warn( title )
2069 # get all intent keys in the cluster
2070 keys = []
2071 for nodeStr in ONOSIntents:
2072 node = json.loads( nodeStr )
2073 for intent in node:
2074 keys.append( intent.get( 'id' ) )
2075 keys = set( keys )
2076 for key in keys:
2077 row = "%-13s" % key
2078 for nodeStr in ONOSIntents:
2079 node = json.loads( nodeStr )
2080 for intent in node:
2081 if intent.get( 'id' ) == key:
2082 row += "%-15s" % intent.get( 'state' )
2083 main.log.warn( row )
2084 # End table view
2085
2086 utilities.assert_equals(
2087 expect=True,
2088 actual=consistentIntents,
2089 onpass="Intents are consistent across all ONOS nodes",
2090 onfail="ONOS nodes have different views of intents" )
2091 intentStates = []
2092 for node in ONOSIntents: # Iter through ONOS nodes
2093 nodeStates = []
2094 # Iter through intents of a node
2095 try:
2096 for intent in json.loads( node ):
2097 nodeStates.append( intent[ 'state' ] )
2098 except ( ValueError, TypeError ):
2099 main.log.exception( "Error in parsing intents" )
2100 main.log.error( repr( node ) )
2101 intentStates.append( nodeStates )
Jon Hallf37d44d2017-05-24 10:37:30 -07002102 out = [ ( i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002103 main.log.info( dict( out ) )
2104
2105 if intentsResults and not consistentIntents:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002106 for i in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002107 node = str( main.activeNodes[ i ] + 1 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002108 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002109 main.log.warn( json.dumps(
2110 json.loads( ONOSIntents[ i ] ),
2111 sort_keys=True,
2112 indent=4,
2113 separators=( ',', ': ' ) ) )
2114 elif intentsResults and consistentIntents:
2115 intentCheck = main.TRUE
2116
2117 # NOTE: Store has no durability, so intents are lost across system
2118 # restarts
2119 main.step( "Compare current intents with intents before the failure" )
2120 # NOTE: this requires case 5 to pass for intentState to be set.
2121 # maybe we should stop the test if that fails?
2122 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002123 try:
2124 intentState
2125 except NameError:
2126 main.log.warn( "No previous intent state was saved" )
2127 else:
2128 if intentState and intentState == ONOSIntents[ 0 ]:
2129 sameIntents = main.TRUE
2130 main.log.info( "Intents are consistent with before failure" )
2131 # TODO: possibly the states have changed? we may need to figure out
2132 # what the acceptable states are
2133 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2134 sameIntents = main.TRUE
2135 try:
2136 before = json.loads( intentState )
2137 after = json.loads( ONOSIntents[ 0 ] )
2138 for intent in before:
2139 if intent not in after:
2140 sameIntents = main.FALSE
2141 main.log.debug( "Intent is not currently in ONOS " +
2142 "(at least in the same form):" )
2143 main.log.debug( json.dumps( intent ) )
2144 except ( ValueError, TypeError ):
2145 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002146 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002147 main.log.debug( repr( intentState ) )
2148 if sameIntents == main.FALSE:
2149 try:
2150 main.log.debug( "ONOS intents before: " )
2151 main.log.debug( json.dumps( json.loads( intentState ),
2152 sort_keys=True, indent=4,
2153 separators=( ',', ': ' ) ) )
2154 main.log.debug( "Current ONOS intents: " )
2155 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2156 sort_keys=True, indent=4,
2157 separators=( ',', ': ' ) ) )
2158 except ( ValueError, TypeError ):
2159 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002160 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002161 main.log.debug( repr( intentState ) )
2162 utilities.assert_equals(
2163 expect=main.TRUE,
2164 actual=sameIntents,
2165 onpass="Intents are consistent with before failure",
2166 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002167 intentCheck = intentCheck and sameIntents
2168
2169 main.step( "Get the OF Table entries and compare to before " +
2170 "component failure" )
2171 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002172 for i in range( 28 ):
2173 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002174 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hallf37d44d2017-05-24 10:37:30 -07002175 curSwitch = main.Mininet1.flowTableComp( flows[ i ], tmpFlows )
Jon Hall41d39f12016-04-11 22:54:35 -07002176 FlowTables = FlowTables and curSwitch
2177 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002178 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002179 utilities.assert_equals(
2180 expect=main.TRUE,
2181 actual=FlowTables,
2182 onpass="No changes were found in the flow tables",
2183 onfail="Changes were found in the flow tables" )
2184
2185 main.Mininet2.pingLongKill()
Jon Hallf37d44d2017-05-24 10:37:30 -07002186 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002187 main.step( "Check the continuous pings to ensure that no packets " +
2188 "were dropped during component failure" )
2189 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2190 main.params[ 'TESTONIP' ] )
2191 LossInPings = main.FALSE
2192 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2193 for i in range( 8, 18 ):
2194 main.log.info(
2195 "Checking for a loss in pings along flow from s" +
2196 str( i ) )
2197 LossInPings = main.Mininet2.checkForLoss(
2198 "/tmp/ping.h" +
2199 str( i ) ) or LossInPings
2200 if LossInPings == main.TRUE:
2201 main.log.info( "Loss in ping detected" )
2202 elif LossInPings == main.ERROR:
2203 main.log.info( "There are multiple mininet process running" )
2204 elif LossInPings == main.FALSE:
2205 main.log.info( "No Loss in the pings" )
2206 main.log.info( "No loss of dataplane connectivity" )
2207 utilities.assert_equals(
2208 expect=main.FALSE,
2209 actual=LossInPings,
2210 onpass="No Loss of connectivity",
2211 onfail="Loss of dataplane connectivity detected" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002212 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002213 main.step( "Leadership Election is still functional" )
2214 # Test of LeadershipElection
2215 leaderList = []
Jon Hall5cf14d52015-07-16 12:15:19 -07002216
Jon Hall3b489db2015-10-05 14:38:37 -07002217 restarted = []
2218 for i in main.kill:
Jon Hallf37d44d2017-05-24 10:37:30 -07002219 restarted.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002220 leaderResult = main.TRUE
Jon Hall3b489db2015-10-05 14:38:37 -07002221
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002222 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002223 cli = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002224 leaderN = cli.electionTestLeader()
2225 leaderList.append( leaderN )
2226 if leaderN == main.FALSE:
2227 # error in response
2228 main.log.error( "Something is wrong with " +
2229 "electionTestLeader function, check the" +
2230 " error logs" )
2231 leaderResult = main.FALSE
2232 elif leaderN is None:
2233 main.log.error( cli.name +
2234 " shows no leader for the election-app was" +
2235 " elected after the old one died" )
2236 leaderResult = main.FALSE
2237 elif leaderN in restarted:
2238 main.log.error( cli.name + " shows " + str( leaderN ) +
2239 " as leader for the election-app, but it " +
2240 "was restarted" )
2241 leaderResult = main.FALSE
2242 if len( set( leaderList ) ) != 1:
2243 leaderResult = main.FALSE
2244 main.log.error(
2245 "Inconsistent view of leader for the election test app" )
2246 # TODO: print the list
2247 utilities.assert_equals(
2248 expect=main.TRUE,
2249 actual=leaderResult,
2250 onpass="Leadership election passed",
2251 onfail="Something went wrong with Leadership election" )
2252
2253 def CASE8( self, main ):
2254 """
2255 Compare topo
2256 """
2257 import json
2258 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002259 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002260 assert main, "main not defined"
2261 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002262 assert main.CLIs, "main.CLIs not defined"
2263 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002264
2265 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002266 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002267 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002268 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002269 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002270 elapsed = 0
2271 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002272 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002273 startTime = time.time()
2274 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002275 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002276 devicesResults = main.TRUE
2277 linksResults = main.TRUE
2278 hostsResults = main.TRUE
2279 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002280 count += 1
2281 cliStart = time.time()
2282 devices = []
2283 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002284 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002285 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002286 name="devices-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002287 args=[ main.CLIs[ i ].devices, [ None ] ],
2288 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002289 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002290 threads.append( t )
2291 t.start()
2292
2293 for t in threads:
2294 t.join()
2295 devices.append( t.result )
2296 hosts = []
2297 ipResult = main.TRUE
2298 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002299 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002300 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002301 name="hosts-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002302 args=[ main.CLIs[ i ].hosts, [ None ] ],
2303 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Halld8f6de82015-12-17 17:04:34 -08002304 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002305 threads.append( t )
2306 t.start()
2307
2308 for t in threads:
2309 t.join()
2310 try:
2311 hosts.append( json.loads( t.result ) )
2312 except ( ValueError, TypeError ):
2313 main.log.exception( "Error parsing hosts results" )
2314 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002315 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002316 for controller in range( 0, len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002317 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002318 if hosts[ controller ]:
2319 for host in hosts[ controller ]:
2320 if host is None or host.get( 'ipAddresses', [] ) == []:
2321 main.log.error(
2322 "Error with host ipAddresses on controller" +
2323 controllerStr + ": " + str( host ) )
2324 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002325 ports = []
2326 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002327 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002328 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002329 name="ports-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002330 args=[ main.CLIs[ i ].ports, [ None ] ],
2331 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002332 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002333 threads.append( t )
2334 t.start()
2335
2336 for t in threads:
2337 t.join()
2338 ports.append( t.result )
2339 links = []
2340 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002341 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002342 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002343 name="links-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002344 args=[ main.CLIs[ i ].links, [ None ] ],
2345 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002346 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002347 threads.append( t )
2348 t.start()
2349
2350 for t in threads:
2351 t.join()
2352 links.append( t.result )
2353 clusters = []
2354 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002355 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002356 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002357 name="clusters-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002358 args=[ main.CLIs[ i ].clusters, [ None ] ],
2359 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002360 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002361 threads.append( t )
2362 t.start()
2363
2364 for t in threads:
2365 t.join()
2366 clusters.append( t.result )
2367
2368 elapsed = time.time() - startTime
2369 cliTime = time.time() - cliStart
2370 print "Elapsed time: " + str( elapsed )
2371 print "CLI time: " + str( cliTime )
2372
Jon Hall6e709752016-02-01 13:38:46 -08002373 if all( e is None for e in devices ) and\
2374 all( e is None for e in hosts ) and\
2375 all( e is None for e in ports ) and\
2376 all( e is None for e in links ) and\
2377 all( e is None for e in clusters ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002378 topoFailMsg = "Could not get topology from ONOS"
2379 main.log.error( topoFailMsg )
2380 continue # Try again, No use trying to compare
Jon Hall6e709752016-02-01 13:38:46 -08002381
Jon Hall5cf14d52015-07-16 12:15:19 -07002382 mnSwitches = main.Mininet1.getSwitches()
2383 mnLinks = main.Mininet1.getLinks()
2384 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002385 for controller in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002386 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002387 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07002388 "Error" not in devices[ controller ] and\
2389 "Error" not in ports[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002390
Jon Hallc6793552016-01-19 14:18:37 -08002391 try:
2392 currentDevicesResult = main.Mininet1.compareSwitches(
2393 mnSwitches,
2394 json.loads( devices[ controller ] ),
2395 json.loads( ports[ controller ] ) )
2396 except ( TypeError, ValueError ) as e:
2397 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2398 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002399 else:
2400 currentDevicesResult = main.FALSE
2401 utilities.assert_equals( expect=main.TRUE,
2402 actual=currentDevicesResult,
2403 onpass="ONOS" + controllerStr +
2404 " Switches view is correct",
2405 onfail="ONOS" + controllerStr +
2406 " Switches view is incorrect" )
2407
2408 if links[ controller ] and "Error" not in links[ controller ]:
2409 currentLinksResult = main.Mininet1.compareLinks(
2410 mnSwitches, mnLinks,
2411 json.loads( links[ controller ] ) )
2412 else:
2413 currentLinksResult = main.FALSE
2414 utilities.assert_equals( expect=main.TRUE,
2415 actual=currentLinksResult,
2416 onpass="ONOS" + controllerStr +
2417 " links view is correct",
2418 onfail="ONOS" + controllerStr +
2419 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002420 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002421 currentHostsResult = main.Mininet1.compareHosts(
2422 mnHosts,
2423 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002424 elif hosts[ controller ] == []:
2425 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002426 else:
2427 currentHostsResult = main.FALSE
2428 utilities.assert_equals( expect=main.TRUE,
2429 actual=currentHostsResult,
2430 onpass="ONOS" + controllerStr +
2431 " hosts exist in Mininet",
2432 onfail="ONOS" + controllerStr +
2433 " hosts don't match Mininet" )
2434 # CHECKING HOST ATTACHMENT POINTS
2435 hostAttachment = True
2436 zeroHosts = False
2437 # FIXME: topo-HA/obelisk specific mappings:
2438 # key is mac and value is dpid
2439 mappings = {}
2440 for i in range( 1, 29 ): # hosts 1 through 28
2441 # set up correct variables:
Jon Hallf37d44d2017-05-24 10:37:30 -07002442 macId = "00:" * 5 + hex( i ).split( "0x" )[ 1 ].upper().zfill( 2 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002443 if i == 1:
Jon Hallf37d44d2017-05-24 10:37:30 -07002444 deviceId = "1000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002445 elif i == 2:
Jon Hallf37d44d2017-05-24 10:37:30 -07002446 deviceId = "2000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002447 elif i == 3:
Jon Hallf37d44d2017-05-24 10:37:30 -07002448 deviceId = "3000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002449 elif i == 4:
Jon Hallf37d44d2017-05-24 10:37:30 -07002450 deviceId = "3004".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002451 elif i == 5:
Jon Hallf37d44d2017-05-24 10:37:30 -07002452 deviceId = "5000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002453 elif i == 6:
Jon Hallf37d44d2017-05-24 10:37:30 -07002454 deviceId = "6000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002455 elif i == 7:
Jon Hallf37d44d2017-05-24 10:37:30 -07002456 deviceId = "6007".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002457 elif i >= 8 and i <= 17:
2458 dpid = '3' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07002459 deviceId = dpid.zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002460 elif i >= 18 and i <= 27:
2461 dpid = '6' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07002462 deviceId = dpid.zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002463 elif i == 28:
Jon Hallf37d44d2017-05-24 10:37:30 -07002464 deviceId = "2800".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002465 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002466 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002467 if hosts[ controller ] == []:
2468 main.log.warn( "There are no hosts discovered" )
2469 zeroHosts = True
2470 else:
2471 for host in hosts[ controller ]:
2472 mac = None
2473 location = None
2474 device = None
2475 port = None
2476 try:
2477 mac = host.get( 'mac' )
2478 assert mac, "mac field could not be found for this host object"
2479
2480 location = host.get( 'location' )
2481 assert location, "location field could not be found for this host object"
2482
2483 # Trim the protocol identifier off deviceId
Jon Hallf37d44d2017-05-24 10:37:30 -07002484 device = str( location.get( 'elementId' ) ).split( ':' )[ 1 ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002485 assert device, "elementId field could not be found for this host location object"
2486
2487 port = location.get( 'port' )
2488 assert port, "port field could not be found for this host location object"
2489
2490 # Now check if this matches where they should be
2491 if mac and device and port:
2492 if str( port ) != "1":
2493 main.log.error( "The attachment port is incorrect for " +
2494 "host " + str( mac ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07002495 ". Expected: 1 Actual: " + str( port ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002496 hostAttachment = False
2497 if device != mappings[ str( mac ) ]:
2498 main.log.error( "The attachment device is incorrect for " +
2499 "host " + str( mac ) +
2500 ". Expected: " + mappings[ str( mac ) ] +
2501 " Actual: " + device )
2502 hostAttachment = False
2503 else:
2504 hostAttachment = False
2505 except AssertionError:
2506 main.log.exception( "Json object not as expected" )
2507 main.log.error( repr( host ) )
2508 hostAttachment = False
2509 else:
2510 main.log.error( "No hosts json output or \"Error\"" +
2511 " in output. hosts = " +
2512 repr( hosts[ controller ] ) )
2513 if zeroHosts is False:
2514 hostAttachment = True
2515
2516 # END CHECKING HOST ATTACHMENT POINTS
2517 devicesResults = devicesResults and currentDevicesResult
2518 linksResults = linksResults and currentLinksResult
2519 hostsResults = hostsResults and currentHostsResult
2520 hostAttachmentResults = hostAttachmentResults and\
2521 hostAttachment
Jon Halla440e872016-03-31 15:15:50 -07002522 topoResult = ( devicesResults and linksResults
2523 and hostsResults and ipResult and
2524 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002525 utilities.assert_equals( expect=True,
2526 actual=topoResult,
2527 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002528 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002529 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002530
2531 # Compare json objects for hosts and dataplane clusters
2532
2533 # hosts
2534 main.step( "Hosts view is consistent across all ONOS nodes" )
2535 consistentHostsResult = main.TRUE
2536 for controller in range( len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002537 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002538 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002539 if hosts[ controller ] == hosts[ 0 ]:
2540 continue
2541 else: # hosts not consistent
2542 main.log.error( "hosts from ONOS" + controllerStr +
2543 " is inconsistent with ONOS1" )
2544 main.log.warn( repr( hosts[ controller ] ) )
2545 consistentHostsResult = main.FALSE
2546
2547 else:
2548 main.log.error( "Error in getting ONOS hosts from ONOS" +
2549 controllerStr )
2550 consistentHostsResult = main.FALSE
2551 main.log.warn( "ONOS" + controllerStr +
2552 " hosts response: " +
2553 repr( hosts[ controller ] ) )
2554 utilities.assert_equals(
2555 expect=main.TRUE,
2556 actual=consistentHostsResult,
2557 onpass="Hosts view is consistent across all ONOS nodes",
2558 onfail="ONOS nodes have different views of hosts" )
2559
2560 main.step( "Hosts information is correct" )
2561 hostsResults = hostsResults and ipResult
2562 utilities.assert_equals(
2563 expect=main.TRUE,
2564 actual=hostsResults,
2565 onpass="Host information is correct",
2566 onfail="Host information is incorrect" )
2567
2568 main.step( "Host attachment points to the network" )
2569 utilities.assert_equals(
2570 expect=True,
2571 actual=hostAttachmentResults,
2572 onpass="Hosts are correctly attached to the network",
2573 onfail="ONOS did not correctly attach hosts to the network" )
2574
2575 # Strongly connected clusters of devices
2576 main.step( "Clusters view is consistent across all ONOS nodes" )
2577 consistentClustersResult = main.TRUE
2578 for controller in range( len( clusters ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002579 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002580 if "Error" not in clusters[ controller ]:
2581 if clusters[ controller ] == clusters[ 0 ]:
2582 continue
2583 else: # clusters not consistent
2584 main.log.error( "clusters from ONOS" +
2585 controllerStr +
2586 " is inconsistent with ONOS1" )
2587 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002588 else:
2589 main.log.error( "Error in getting dataplane clusters " +
2590 "from ONOS" + controllerStr )
2591 consistentClustersResult = main.FALSE
2592 main.log.warn( "ONOS" + controllerStr +
2593 " clusters response: " +
2594 repr( clusters[ controller ] ) )
2595 utilities.assert_equals(
2596 expect=main.TRUE,
2597 actual=consistentClustersResult,
2598 onpass="Clusters view is consistent across all ONOS nodes",
2599 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002600 if not consistentClustersResult:
2601 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002602
2603 main.step( "There is only one SCC" )
2604 # there should always only be one cluster
2605 try:
2606 numClusters = len( json.loads( clusters[ 0 ] ) )
2607 except ( ValueError, TypeError ):
2608 main.log.exception( "Error parsing clusters[0]: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07002609 repr( clusters[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002610 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002611 clusterResults = main.FALSE
2612 if numClusters == 1:
2613 clusterResults = main.TRUE
2614 utilities.assert_equals(
2615 expect=1,
2616 actual=numClusters,
2617 onpass="ONOS shows 1 SCC",
2618 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2619
2620 topoResult = ( devicesResults and linksResults
2621 and hostsResults and consistentHostsResult
2622 and consistentClustersResult and clusterResults
2623 and ipResult and hostAttachmentResults )
2624
2625 topoResult = topoResult and int( count <= 2 )
2626 note = "note it takes about " + str( int( cliTime ) ) + \
2627 " seconds for the test to make all the cli calls to fetch " +\
2628 "the topology from each ONOS instance"
2629 main.log.info(
2630 "Very crass estimate for topology discovery/convergence( " +
2631 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2632 str( count ) + " tries" )
2633
2634 main.step( "Device information is correct" )
2635 utilities.assert_equals(
2636 expect=main.TRUE,
2637 actual=devicesResults,
2638 onpass="Device information is correct",
2639 onfail="Device information is incorrect" )
2640
2641 main.step( "Links are correct" )
2642 utilities.assert_equals(
2643 expect=main.TRUE,
2644 actual=linksResults,
2645 onpass="Link are correct",
2646 onfail="Links are incorrect" )
2647
Jon Halla440e872016-03-31 15:15:50 -07002648 main.step( "Hosts are correct" )
2649 utilities.assert_equals(
2650 expect=main.TRUE,
2651 actual=hostsResults,
2652 onpass="Hosts are correct",
2653 onfail="Hosts are incorrect" )
2654
Jon Hall5cf14d52015-07-16 12:15:19 -07002655 # FIXME: move this to an ONOS state case
2656 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002657 nodeResults = utilities.retry( main.HA.nodesCheck,
2658 False,
Jon Hallf37d44d2017-05-24 10:37:30 -07002659 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -07002660 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002661
Jon Hall41d39f12016-04-11 22:54:35 -07002662 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002663 onpass="Nodes check successful",
2664 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002665 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002666 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002667 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hallf37d44d2017-05-24 10:37:30 -07002668 main.CLIs[ i ].name,
2669 main.CLIs[ i ].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002670
Jon Halld2871c22016-07-26 11:01:14 -07002671 if not topoResult:
2672 main.cleanup()
2673 main.exit()
2674
Jon Hall5cf14d52015-07-16 12:15:19 -07002675 def CASE9( self, main ):
2676 """
2677 Link s3-s28 down
2678 """
2679 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002680 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002681 assert main, "main not defined"
2682 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002683 assert main.CLIs, "main.CLIs not defined"
2684 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002685 # NOTE: You should probably run a topology check after this
2686
2687 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2688
2689 description = "Turn off a link to ensure that Link Discovery " +\
2690 "is working properly"
2691 main.case( description )
2692
2693 main.step( "Kill Link between s3 and s28" )
2694 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2695 main.log.info( "Waiting " + str( linkSleep ) +
2696 " seconds for link down to be discovered" )
2697 time.sleep( linkSleep )
2698 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2699 onpass="Link down successful",
2700 onfail="Failed to bring link down" )
2701 # TODO do some sort of check here
2702
2703 def CASE10( self, main ):
2704 """
2705 Link s3-s28 up
2706 """
2707 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002708 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002709 assert main, "main not defined"
2710 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002711 assert main.CLIs, "main.CLIs not defined"
2712 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002713 # NOTE: You should probably run a topology check after this
2714
2715 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2716
2717 description = "Restore a link to ensure that Link Discovery is " + \
2718 "working properly"
2719 main.case( description )
2720
2721 main.step( "Bring link between s3 and s28 back up" )
2722 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2723 main.log.info( "Waiting " + str( linkSleep ) +
2724 " seconds for link up to be discovered" )
2725 time.sleep( linkSleep )
2726 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2727 onpass="Link up successful",
2728 onfail="Failed to bring link up" )
2729 # TODO do some sort of check here
2730
2731 def CASE11( self, main ):
2732 """
2733 Switch Down
2734 """
2735 # NOTE: You should probably run a topology check after this
2736 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002737 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002738 assert main, "main not defined"
2739 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002740 assert main.CLIs, "main.CLIs not defined"
2741 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002742
2743 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2744
2745 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallf37d44d2017-05-24 10:37:30 -07002746 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002747 main.case( description )
2748 switch = main.params[ 'kill' ][ 'switch' ]
2749 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2750
2751 # TODO: Make this switch parameterizable
2752 main.step( "Kill " + switch )
2753 main.log.info( "Deleting " + switch )
2754 main.Mininet1.delSwitch( switch )
2755 main.log.info( "Waiting " + str( switchSleep ) +
2756 " seconds for switch down to be discovered" )
2757 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002758 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002759 # Peek at the deleted switch
2760 main.log.warn( str( device ) )
2761 result = main.FALSE
2762 if device and device[ 'available' ] is False:
2763 result = main.TRUE
2764 utilities.assert_equals( expect=main.TRUE, actual=result,
2765 onpass="Kill switch successful",
2766 onfail="Failed to kill switch?" )
2767
2768 def CASE12( self, main ):
2769 """
2770 Switch Up
2771 """
2772 # NOTE: You should probably run a topology check after this
2773 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002774 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002775 assert main, "main not defined"
2776 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002777 assert main.CLIs, "main.CLIs not defined"
2778 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002779 assert ONOS1Port, "ONOS1Port not defined"
2780 assert ONOS2Port, "ONOS2Port not defined"
2781 assert ONOS3Port, "ONOS3Port not defined"
2782 assert ONOS4Port, "ONOS4Port not defined"
2783 assert ONOS5Port, "ONOS5Port not defined"
2784 assert ONOS6Port, "ONOS6Port not defined"
2785 assert ONOS7Port, "ONOS7Port not defined"
2786
2787 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2788 switch = main.params[ 'kill' ][ 'switch' ]
2789 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2790 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallf37d44d2017-05-24 10:37:30 -07002791 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002792 description = "Adding a switch to ensure it is discovered correctly"
2793 main.case( description )
2794
2795 main.step( "Add back " + switch )
2796 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2797 for peer in links:
2798 main.Mininet1.addLink( switch, peer )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002799 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002800 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2801 main.log.info( "Waiting " + str( switchSleep ) +
2802 " seconds for switch up to be discovered" )
2803 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002804 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002805 # Peek at the deleted switch
2806 main.log.warn( str( device ) )
2807 result = main.FALSE
2808 if device and device[ 'available' ]:
2809 result = main.TRUE
2810 utilities.assert_equals( expect=main.TRUE, actual=result,
2811 onpass="add switch successful",
2812 onfail="Failed to add switch?" )
2813
2814 def CASE13( self, main ):
2815 """
2816 Clean up
2817 """
2818 import os
2819 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002820 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002821 assert main, "main not defined"
2822 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002823 assert main.CLIs, "main.CLIs not defined"
2824 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002825
2826 # printing colors to terminal
2827 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2828 'blue': '\033[94m', 'green': '\033[92m',
2829 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2830 main.case( "Test Cleanup" )
2831 main.step( "Killing tcpdumps" )
2832 main.Mininet2.stopTcpdump()
2833
2834 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002835 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002836 main.step( "Copying MN pcap and ONOS log files to test station" )
2837 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2838 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002839 # NOTE: MN Pcap file is being saved to logdir.
2840 # We scp this file as MN and TestON aren't necessarily the same vm
2841
2842 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002843 # TODO: Load these from params
2844 # NOTE: must end in /
2845 logFolder = "/opt/onos/log/"
2846 logFiles = [ "karaf.log", "karaf.log.1" ]
2847 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002848 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002849 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002850 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002851 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2852 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002853 # std*.log's
2854 # NOTE: must end in /
2855 logFolder = "/opt/onos/var/"
2856 logFiles = [ "stderr.log", "stdout.log" ]
2857 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002858 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002859 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002860 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002861 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2862 logFolder + f, dstName )
2863 else:
2864 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002865
2866 main.step( "Stopping Mininet" )
2867 mnResult = main.Mininet1.stopNet()
2868 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2869 onpass="Mininet stopped",
2870 onfail="MN cleanup NOT successful" )
2871
2872 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002873 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002874 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2875 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002876
2877 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07002878 timerLog = open( main.logdir + "/Timers.csv", 'w' )
Jon Hall5cf14d52015-07-16 12:15:19 -07002879 # Overwrite with empty line and close
2880 labels = "Gossip Intents, Restart"
2881 data = str( gossipTime ) + ", " + str( main.restartTime )
2882 timerLog.write( labels + "\n" + data )
2883 timerLog.close()
Jon Hallf37d44d2017-05-24 10:37:30 -07002884 except NameError as e:
2885 main.log.exception( e )
Jon Hall5cf14d52015-07-16 12:15:19 -07002886
2887 def CASE14( self, main ):
2888 """
2889 start election app on all onos nodes
2890 """
Jon Halle1a3b752015-07-22 13:02:46 -07002891 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002892 assert main, "main not defined"
2893 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002894 assert main.CLIs, "main.CLIs not defined"
2895 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002896
Jon Hallf37d44d2017-05-24 10:37:30 -07002897 main.case( "Start Leadership Election app" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002898 main.step( "Install leadership election app" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002899 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002900 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002901 utilities.assert_equals(
2902 expect=main.TRUE,
2903 actual=appResult,
2904 onpass="Election app installed",
2905 onfail="Something went wrong with installing Leadership election" )
2906
2907 main.step( "Run for election on each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002908 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002909 main.CLIs[ i ].electionTestRun()
2910 time.sleep( 5 )
2911 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall25463a82016-04-13 14:03:52 -07002912 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002913 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002914 expect=True,
2915 actual=sameResult,
2916 onpass="All nodes see the same leaderboards",
2917 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002918
Jon Hall25463a82016-04-13 14:03:52 -07002919 if sameResult:
2920 leader = leaders[ 0 ][ 0 ]
Jon Hallf37d44d2017-05-24 10:37:30 -07002921 if main.nodes[ main.activeNodes[ 0 ] ].ip_address in leader:
Jon Hall25463a82016-04-13 14:03:52 -07002922 correctLeader = True
2923 else:
2924 correctLeader = False
2925 main.step( "First node was elected leader" )
2926 utilities.assert_equals(
2927 expect=True,
2928 actual=correctLeader,
2929 onpass="Correct leader was elected",
2930 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002931
2932 def CASE15( self, main ):
2933 """
2934 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002935 15.1 Run election on each node
2936 15.2 Check that each node has the same leaders and candidates
2937 15.3 Find current leader and withdraw
2938 15.4 Check that a new node was elected leader
2939 15.5 Check that that new leader was the candidate of old leader
2940 15.6 Run for election on old leader
2941 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2942 15.8 Make sure that the old leader was added to the candidate list
2943
2944 old and new variable prefixes refer to data from before vs after
2945 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002946 """
2947 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002948 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002949 assert main, "main not defined"
2950 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002951 assert main.CLIs, "main.CLIs not defined"
2952 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002953
Jon Hall5cf14d52015-07-16 12:15:19 -07002954 description = "Check that Leadership Election is still functional"
2955 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002956 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002957
Jon Halla440e872016-03-31 15:15:50 -07002958 oldLeaders = [] # list of lists of each nodes' candidates before
2959 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002960 oldLeader = '' # the old leader from oldLeaders, None if not same
2961 newLeader = '' # the new leaders fron newLoeaders, None if not same
2962 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2963 expectNoLeader = False # True when there is only one leader
2964 if main.numCtrls == 1:
2965 expectNoLeader = True
2966
2967 main.step( "Run for election on each node" )
2968 electionResult = main.TRUE
2969
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002970 for i in main.activeNodes: # run test election on each node
Jon Hallf37d44d2017-05-24 10:37:30 -07002971 if main.CLIs[ i ].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002972 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002973 utilities.assert_equals(
2974 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002975 actual=electionResult,
2976 onpass="All nodes successfully ran for leadership",
2977 onfail="At least one node failed to run for leadership" )
2978
acsmars3a72bde2015-09-02 14:16:22 -07002979 if electionResult == main.FALSE:
2980 main.log.error(
2981 "Skipping Test Case because Election Test App isn't loaded" )
2982 main.skipCase()
2983
acsmars71adceb2015-08-31 15:09:26 -07002984 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002985 failMessage = "Nodes have different leaderboards"
Jon Hallf37d44d2017-05-24 10:37:30 -07002986 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002987 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002988 if sameResult:
2989 oldLeader = oldLeaders[ 0 ][ 0 ]
2990 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07002991 else:
Jon Halla440e872016-03-31 15:15:50 -07002992 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002993 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002994 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002995 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002996 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002997 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002998
2999 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07003000 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07003001 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07003002 if oldLeader is None:
3003 main.log.error( "Leadership isn't consistent." )
3004 withdrawResult = main.FALSE
3005 # Get the CLI of the oldLeader
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003006 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07003007 if oldLeader == main.nodes[ i ].ip_address:
3008 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07003009 break
3010 else: # FOR/ELSE statement
3011 main.log.error( "Leader election, could not find current leader" )
3012 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07003013 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07003014 utilities.assert_equals(
3015 expect=main.TRUE,
3016 actual=withdrawResult,
3017 onpass="Node was withdrawn from election",
3018 onfail="Node was not withdrawn from election" )
3019
acsmars71adceb2015-08-31 15:09:26 -07003020 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07003021 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07003022 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07003023 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07003024 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07003025 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07003026 if newLeaders[ 0 ][ 0 ] == 'none':
3027 main.log.error( "No leader was elected on at least 1 node" )
3028 if not expectNoLeader:
3029 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07003030 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07003031
3032 # Check that the new leader is not the older leader, which was withdrawn
3033 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07003034 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08003035 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07003036 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003037 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003038 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003039 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003040 onpass="Leadership election passed",
3041 onfail="Something went wrong with Leadership election" )
3042
Jon Halla440e872016-03-31 15:15:50 -07003043 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08003044 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07003045 correctCandidateResult = main.TRUE
3046 if expectNoLeader:
3047 if newLeader == 'none':
3048 main.log.info( "No leader expected. None found. Pass" )
3049 correctCandidateResult = main.TRUE
3050 else:
3051 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3052 correctCandidateResult = main.FALSE
Jon Hallf37d44d2017-05-24 10:37:30 -07003053 elif len( oldLeaders[ 0 ] ) >= 3:
Jon Halla440e872016-03-31 15:15:50 -07003054 if newLeader == oldLeaders[ 0 ][ 2 ]:
3055 # correct leader was elected
3056 correctCandidateResult = main.TRUE
3057 else:
3058 correctCandidateResult = main.FALSE
3059 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3060 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003061 else:
3062 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003063 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003064 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07003065 utilities.assert_equals(
3066 expect=main.TRUE,
3067 actual=correctCandidateResult,
3068 onpass="Correct Candidate Elected",
3069 onfail="Incorrect Candidate Elected" )
3070
Jon Hall5cf14d52015-07-16 12:15:19 -07003071 main.step( "Run for election on old leader( just so everyone " +
3072 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07003073 if oldLeaderCLI is not None:
3074 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003075 else:
acsmars71adceb2015-08-31 15:09:26 -07003076 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003077 runResult = main.FALSE
3078 utilities.assert_equals(
3079 expect=main.TRUE,
3080 actual=runResult,
3081 onpass="App re-ran for election",
3082 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003083
acsmars71adceb2015-08-31 15:09:26 -07003084 main.step(
3085 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003086 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003087 # Get new leaders and candidates
3088 reRunLeaders = []
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003089 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003090 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003091
3092 # Check that the re-elected node is last on the candidate List
Jon Hallf37d44d2017-05-24 10:37:30 -07003093 if not reRunLeaders[ 0 ]:
Jon Hall3a7843a2016-04-12 03:01:09 -07003094 positionResult = main.FALSE
3095 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07003096 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader ),
Jon Halla440e872016-03-31 15:15:50 -07003097 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07003098 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003099 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003100 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003101 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003102 onpass="Old leader successfully re-ran for election",
3103 onfail="Something went wrong with Leadership election after " +
3104 "the old leader re-ran for election" )
3105
3106 def CASE16( self, main ):
3107 """
3108 Install Distributed Primitives app
3109 """
3110 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003111 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003112 assert main, "main not defined"
3113 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003114 assert main.CLIs, "main.CLIs not defined"
3115 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003116
3117 # Variables for the distributed primitives tests
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003118 main.pCounterName = "TestON-Partitions"
3119 main.pCounterValue = 0
Jon Hallf37d44d2017-05-24 10:37:30 -07003120 main.onosSet = set( [] )
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003121 main.onosSetName = "TestON-set"
Jon Hall5cf14d52015-07-16 12:15:19 -07003122
3123 description = "Install Primitives app"
3124 main.case( description )
3125 main.step( "Install Primitives app" )
3126 appName = "org.onosproject.distributedprimitives"
Jon Hallf37d44d2017-05-24 10:37:30 -07003127 node = main.activeNodes[ 0 ]
3128 appResults = main.CLIs[ node ].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003129 utilities.assert_equals( expect=main.TRUE,
3130 actual=appResults,
3131 onpass="Primitives app activated",
3132 onfail="Primitives app not activated" )
3133 time.sleep( 5 ) # To allow all nodes to activate
3134
3135 def CASE17( self, main ):
3136 """
3137 Check for basic functionality with distributed primitives
3138 """
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003139 main.HA.CASE17( main )