blob: 4ef70018171ecab5ddfa13a1bd6092d83d813659 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 a minority of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
Jon Hallb3ed8ed2015-10-28 16:43:55 -070012CASE61: The Failure inducing case.
13CASE62: The Failure recovery case.
Jon Hall5cf14d52015-07-16 12:15:19 -070014CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
Jon Hallb3ed8ed2015-10-28 16:43:55 -070026class HAkillNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -070027
28 def __init__( self ):
29 self.default = ''
30
31 def CASE1( self, main ):
32 """
33 CASE1 is to compile ONOS and push it to the test machines
34
35 Startup sequence:
36 cell <name>
37 onos-verify-cell
38 NOTE: temporary - onos-remove-raft-logs
39 onos-uninstall
40 start mininet
41 git pull
42 mvn clean install
43 onos-package
44 onos-install -f
45 onos-wait-for-start
46 start cli sessions
47 start tcpdump
48 """
Jon Halle1a3b752015-07-22 13:02:46 -070049 import imp
Jon Hall3b489db2015-10-05 14:38:37 -070050 import pexpect
Jon Hall6e709752016-02-01 13:38:46 -080051 import time
Jon Halla440e872016-03-31 15:15:50 -070052 import json
Jon Hall6e709752016-02-01 13:38:46 -080053 main.log.info( "ONOS HA test: Restart a minority of ONOS nodes - " +
Jon Hall5cf14d52015-07-16 12:15:19 -070054 "initialization" )
55 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070056 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070057 "installing ONOS, starting Mininet and ONOS" +\
58 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070059
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
71 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070072 global ONOS1Port
73 global ONOS2Port
74 global ONOS3Port
75 global ONOS4Port
76 global ONOS5Port
77 global ONOS6Port
78 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070079 # These are for csv plotting in jenkins
80 global labels
81 global data
82 labels = []
83 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070084
85 # FIXME: just get controller port from params?
86 # TODO: do we really need all these?
87 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
88 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
89 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
90 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
91 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
92 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
93 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
94
Jon Halle1a3b752015-07-22 13:02:46 -070095 try:
Jon Hall53c5e662016-04-13 16:06:56 -070096 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070097 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070098 except Exception as e:
99 main.log.exception( e )
100 main.cleanup()
101 main.exit()
102
103 main.CLIs = []
104 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700105 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700106 for i in range( 1, main.numCtrls + 1 ):
107 try:
108 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
109 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
110 ipList.append( main.nodes[ -1 ].ip_address )
111 except AttributeError:
112 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700113
114 main.step( "Create cell file" )
115 cellAppString = main.params[ 'ENV' ][ 'appString' ]
116 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
117 main.Mininet1.ip_address,
Devin Limdc78e202017-06-09 18:30:07 -0700118 cellAppString, ipList, main.ONOScli1.karafUser )
Jon Hall5cf14d52015-07-16 12:15:19 -0700119 main.step( "Applying cell variable to environment" )
120 cellResult = main.ONOSbench.setCell( cellName )
121 verifyResult = main.ONOSbench.verifyCell()
122
123 # FIXME:this is short term fix
124 main.log.info( "Removing raft logs" )
125 main.ONOSbench.onosRemoveRaftLogs()
126
127 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700128 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700129 main.ONOSbench.onosUninstall( node.ip_address )
130
131 # Make sure ONOS is DEAD
132 main.log.info( "Killing any ONOS processes" )
133 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700134 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700135 killed = main.ONOSbench.onosKill( node.ip_address )
136 killResults = killResults and killed
137
Jon Hall5cf14d52015-07-16 12:15:19 -0700138 gitPullResult = main.TRUE
139
140 main.step( "Starting Mininet" )
141 # scp topo file to mininet
142 # TODO: move to params?
143 topoName = "obelisk.py"
144 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700145 main.ONOSbench.scp( main.Mininet1,
146 filePath + topoName,
147 main.Mininet1.home,
148 direction="to" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700149 mnResult = main.Mininet1.startNet()
Jon Hall5cf14d52015-07-16 12:15:19 -0700150 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
151 onpass="Mininet Started",
152 onfail="Error starting Mininet" )
153
154 main.step( "Git checkout and pull " + gitBranch )
155 if PULLCODE:
156 main.ONOSbench.gitCheckout( gitBranch )
157 gitPullResult = main.ONOSbench.gitPull()
158 # values of 1 or 3 are good
159 utilities.assert_lesser( expect=0, actual=gitPullResult,
160 onpass="Git pull successful",
161 onfail="Git pull failed" )
162 main.ONOSbench.getVersion( report=True )
163
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700164 main.step( "Make sure ONOS service doesn't automatically respawn" )
165 handle = main.ONOSbench.handle
166 handle.sendline( "sed -i -e 's/^respawn$/#respawn/g' tools/package/init/onos.conf" )
167 handle.expect( "\$" ) # $ from the command
Jon Hall334ba942017-01-19 17:02:17 -0800168 handle.sendline( "sed -i -e 's/^Restart=always/Restart=no/g' tools/package/init/onos.service" )
169 handle.expect( "\$" ) # $ from the command
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700170 handle.expect( "\$" ) # $ from the prompt
171
Jon Hall5cf14d52015-07-16 12:15:19 -0700172 # GRAPHS
173 # NOTE: important params here:
174 # job = name of Jenkins job
175 # Plot Name = Plot-HA, only can be used if multiple plots
176 # index = The number of the graph under plot name
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700177 job = "HAkillNodes"
Jon Hall5cf14d52015-07-16 12:15:19 -0700178 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700179 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700180 graphs = '<ac:structured-macro ac:name="html">\n'
181 graphs += '<ac:plain-text-body><![CDATA[\n'
182 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800183 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700184 '&width=500&height=300"' +\
185 'noborder="0" width="500" height="300" scrolling="yes" ' +\
186 'seamless="seamless"></iframe>\n'
187 graphs += ']]></ac:plain-text-body>\n'
188 graphs += '</ac:structured-macro>\n'
Jon Hallf37d44d2017-05-24 10:37:30 -0700189 main.log.wiki( graphs )
Jon Hall5cf14d52015-07-16 12:15:19 -0700190
191 main.step( "Creating ONOS package" )
Jon Hall3b489db2015-10-05 14:38:37 -0700192 # copy gen-partions file to ONOS
193 # NOTE: this assumes TestON and ONOS are on the same machine
Jon Hall53c5e662016-04-13 16:06:56 -0700194 srcFile = main.testDir + "/HA/dependencies/onos-gen-partitions"
Jon Hall3b489db2015-10-05 14:38:37 -0700195 dstDir = main.ONOSbench.home + "/tools/test/bin/onos-gen-partitions"
196 cpResult = main.ONOSbench.secureCopy( main.ONOSbench.user_name,
197 main.ONOSbench.ip_address,
198 srcFile,
199 dstDir,
200 pwd=main.ONOSbench.pwd,
201 direction="from" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700202 packageResult = main.ONOSbench.buckBuild()
Jon Hall5cf14d52015-07-16 12:15:19 -0700203 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
204 onpass="ONOS package successful",
205 onfail="ONOS package failed" )
206
207 main.step( "Installing ONOS package" )
208 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700209 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700210 tmpResult = main.ONOSbench.onosInstall( options="-f",
211 node=node.ip_address )
212 onosInstallResult = onosInstallResult and tmpResult
213 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
214 onpass="ONOS install successful",
215 onfail="ONOS install failed" )
Jon Hall3b489db2015-10-05 14:38:37 -0700216 # clean up gen-partitions file
217 try:
218 main.ONOSbench.handle.sendline( "cd " + main.ONOSbench.home )
219 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
220 main.ONOSbench.handle.sendline( "git checkout -- tools/test/bin/onos-gen-partitions" )
221 main.ONOSbench.handle.expect( main.ONOSbench.home + "\$" )
222 main.log.info( " Cleaning custom gen partitions file, response was: \n" +
223 str( main.ONOSbench.handle.before ) )
224 except ( pexpect.TIMEOUT, pexpect.EOF ):
225 main.log.exception( "ONOSbench: pexpect exception found:" +
226 main.ONOSbench.handle.before )
227 main.cleanup()
228 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700229
You Wangf5de25b2017-01-06 15:13:01 -0800230 main.step( "Set up ONOS secure SSH" )
231 secureSshResult = main.TRUE
232 for node in main.nodes:
233 secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=node.ip_address )
234 utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
235 onpass="Test step PASS",
236 onfail="Test step FAIL" )
237
Jon Hall5cf14d52015-07-16 12:15:19 -0700238 main.step( "Checking if ONOS is up yet" )
239 for i in range( 2 ):
240 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700241 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700242 started = main.ONOSbench.isup( node.ip_address )
243 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800244 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700245 onosIsupResult = onosIsupResult and started
246 if onosIsupResult == main.TRUE:
247 break
248 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
249 onpass="ONOS startup successful",
250 onfail="ONOS startup failed" )
251
Jon Hall6509dbf2016-06-21 17:01:17 -0700252 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700253 cliResults = main.TRUE
254 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700255 for i in range( main.numCtrls ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700256 t = main.Thread( target=main.CLIs[ i ].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700257 name="startOnosCli-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -0700258 args=[ main.nodes[ i ].ip_address ] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700259 threads.append( t )
260 t.start()
261
262 for t in threads:
263 t.join()
264 cliResults = cliResults and t.result
265 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
266 onpass="ONOS cli startup successful",
267 onfail="ONOS cli startup failed" )
268
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700269 # Create a list of active nodes for use when some nodes are stopped
270 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
271
Jon Hall5cf14d52015-07-16 12:15:19 -0700272 if main.params[ 'tcpdump' ].lower() == "true":
273 main.step( "Start Packet Capture MN" )
274 main.Mininet2.startTcpdump(
275 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
276 + "-MN.pcap",
277 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
278 port=main.params[ 'MNtcpdump' ][ 'port' ] )
279
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700280 main.step( "Clean up ONOS service changes" )
281 handle.sendline( "git checkout -- tools/package/init/onos.conf" )
Jon Hall334ba942017-01-19 17:02:17 -0800282 handle.sendline( "git checkout -- tools/package/init/onos.service" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700283 handle.expect( "\$" )
284
Jon Halla440e872016-03-31 15:15:50 -0700285 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700286 nodeResults = utilities.retry( main.HA.nodesCheck,
287 False,
Jon Hallf37d44d2017-05-24 10:37:30 -0700288 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -0700289 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700290
Jon Hall41d39f12016-04-11 22:54:35 -0700291 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700292 onpass="Nodes check successful",
293 onfail="Nodes check NOT successful" )
294
295 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700296 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700297 cli = main.CLIs[ i ]
Jon Halla440e872016-03-31 15:15:50 -0700298 main.log.debug( "{} components not ACTIVE: \n{}".format(
299 cli.name,
300 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700301 main.log.error( "Failed to start ONOS, stopping test" )
302 main.cleanup()
303 main.exit()
304
Jon Hall172b7ba2016-04-07 18:12:20 -0700305 main.step( "Activate apps defined in the params file" )
306 # get data from the params
307 apps = main.params.get( 'apps' )
308 if apps:
Jon Hallf37d44d2017-05-24 10:37:30 -0700309 apps = apps.split( ',' )
Jon Hall172b7ba2016-04-07 18:12:20 -0700310 main.log.warn( apps )
311 activateResult = True
312 for app in apps:
313 main.CLIs[ 0 ].app( app, "Activate" )
314 # TODO: check this worked
315 time.sleep( 10 ) # wait for apps to activate
316 for app in apps:
317 state = main.CLIs[ 0 ].appStatus( app )
318 if state == "ACTIVE":
Jon Hall937bc812017-01-31 16:44:10 -0800319 activateResult = activateResult and True
Jon Hall172b7ba2016-04-07 18:12:20 -0700320 else:
321 main.log.error( "{} is in {} state".format( app, state ) )
Jon Hall937bc812017-01-31 16:44:10 -0800322 activateResult = False
Jon Hall172b7ba2016-04-07 18:12:20 -0700323 utilities.assert_equals( expect=True,
324 actual=activateResult,
325 onpass="Successfully activated apps",
326 onfail="Failed to activate apps" )
327 else:
328 main.log.warn( "No apps were specified to be loaded after startup" )
329
330 main.step( "Set ONOS configurations" )
331 config = main.params.get( 'ONOS_Configuration' )
332 if config:
333 main.log.debug( config )
334 checkResult = main.TRUE
335 for component in config:
Jon Hallf37d44d2017-05-24 10:37:30 -0700336 for setting in config[ component ]:
337 value = config[ component ][ setting ]
Jon Hall172b7ba2016-04-07 18:12:20 -0700338 check = main.CLIs[ 0 ].setCfg( component, setting, value )
339 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
340 checkResult = check and checkResult
341 utilities.assert_equals( expect=main.TRUE,
342 actual=checkResult,
343 onpass="Successfully set config",
344 onfail="Failed to set config" )
345 else:
346 main.log.warn( "No configurations were specified to be changed after startup" )
347
Jon Hall9d2dcad2016-04-08 10:15:20 -0700348 main.step( "App Ids check" )
349 appCheck = main.TRUE
350 threads = []
351 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700352 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall9d2dcad2016-04-08 10:15:20 -0700353 name="appToIDCheck-" + str( i ),
354 args=[] )
355 threads.append( t )
356 t.start()
357
358 for t in threads:
359 t.join()
360 appCheck = appCheck and t.result
361 if appCheck != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700362 node = main.activeNodes[ 0 ]
363 main.log.warn( main.CLIs[ node ].apps() )
364 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700365 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
366 onpass="App Ids seem to be correct",
367 onfail="Something is wrong with app Ids" )
368
Jon Hall5cf14d52015-07-16 12:15:19 -0700369 def CASE2( self, main ):
370 """
371 Assign devices to controllers
372 """
373 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700374 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700375 assert main, "main not defined"
376 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700377 assert main.CLIs, "main.CLIs not defined"
378 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700379 assert ONOS1Port, "ONOS1Port not defined"
380 assert ONOS2Port, "ONOS2Port not defined"
381 assert ONOS3Port, "ONOS3Port not defined"
382 assert ONOS4Port, "ONOS4Port not defined"
383 assert ONOS5Port, "ONOS5Port not defined"
384 assert ONOS6Port, "ONOS6Port not defined"
385 assert ONOS7Port, "ONOS7Port not defined"
386
387 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700388 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700389 "and check that an ONOS node becomes the " +\
390 "master of the device."
391 main.step( "Assign switches to controllers" )
392
393 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700394 for i in range( main.numCtrls ):
395 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700396 swList = []
397 for i in range( 1, 29 ):
398 swList.append( "s" + str( i ) )
399 main.Mininet1.assignSwController( sw=swList, ip=ipList )
400
401 mastershipCheck = main.TRUE
402 for i in range( 1, 29 ):
403 response = main.Mininet1.getSwController( "s" + str( i ) )
404 try:
405 main.log.info( str( response ) )
406 except Exception:
407 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700408 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700409 if re.search( "tcp:" + node.ip_address, response ):
410 mastershipCheck = mastershipCheck and main.TRUE
411 else:
412 main.log.error( "Error, node " + node.ip_address + " is " +
413 "not in the list of controllers s" +
414 str( i ) + " is connecting to." )
415 mastershipCheck = main.FALSE
416 utilities.assert_equals(
417 expect=main.TRUE,
418 actual=mastershipCheck,
419 onpass="Switch mastership assigned correctly",
420 onfail="Switches not assigned correctly to controllers" )
421
422 def CASE21( self, main ):
423 """
424 Assign mastership to controllers
425 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700426 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700427 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700428 assert main, "main not defined"
429 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700430 assert main.CLIs, "main.CLIs not defined"
431 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700432 assert ONOS1Port, "ONOS1Port not defined"
433 assert ONOS2Port, "ONOS2Port not defined"
434 assert ONOS3Port, "ONOS3Port not defined"
435 assert ONOS4Port, "ONOS4Port not defined"
436 assert ONOS5Port, "ONOS5Port not defined"
437 assert ONOS6Port, "ONOS6Port not defined"
438 assert ONOS7Port, "ONOS7Port not defined"
439
440 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700441 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700442 "device. Then manually assign" +\
443 " mastership to specific ONOS nodes using" +\
444 " 'device-role'"
445 main.step( "Assign mastership of switches to specific controllers" )
446 # Manually assign mastership to the controller we want
447 roleCall = main.TRUE
448
Jon Hallf37d44d2017-05-24 10:37:30 -0700449 ipList = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700450 deviceList = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700451 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700452 try:
453 # Assign mastership to specific controllers. This assignment was
454 # determined for a 7 node cluser, but will work with any sized
455 # cluster
456 for i in range( 1, 29 ): # switches 1 through 28
457 # set up correct variables:
458 if i == 1:
459 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700460 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700461 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700462 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700463 c = 1 % main.numCtrls
464 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700465 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700466 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700467 c = 1 % main.numCtrls
468 ip = main.nodes[ c ].ip_address # ONOS2
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700469 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700470 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700471 c = 3 % main.numCtrls
472 ip = main.nodes[ c ].ip_address # ONOS4
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700473 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700474 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700475 c = 2 % main.numCtrls
476 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700477 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700478 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700479 c = 2 % main.numCtrls
480 ip = main.nodes[ c ].ip_address # ONOS3
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700481 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700482 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700483 c = 5 % main.numCtrls
484 ip = main.nodes[ c ].ip_address # ONOS6
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700485 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700486 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700487 c = 4 % main.numCtrls
488 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700489 dpid = '3' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700490 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700491 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700492 c = 6 % main.numCtrls
493 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700494 dpid = '6' + str( i ).zfill( 3 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700495 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700496 elif i == 28:
497 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700498 ip = main.nodes[ c ].ip_address # ONOS1
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700499 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700500 else:
501 main.log.error( "You didn't write an else statement for " +
502 "switch s" + str( i ) )
503 roleCall = main.FALSE
504 # Assign switch
505 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
506 # TODO: make this controller dynamic
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700507 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700508 ipList.append( ip )
509 deviceList.append( deviceId )
510 except ( AttributeError, AssertionError ):
511 main.log.exception( "Something is wrong with ONOS device view" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700512 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700513 utilities.assert_equals(
514 expect=main.TRUE,
515 actual=roleCall,
516 onpass="Re-assigned switch mastership to designated controller",
517 onfail="Something wrong with deviceRole calls" )
518
519 main.step( "Check mastership was correctly assigned" )
520 roleCheck = main.TRUE
521 # NOTE: This is due to the fact that device mastership change is not
522 # atomic and is actually a multi step process
523 time.sleep( 5 )
524 for i in range( len( ipList ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700525 ip = ipList[ i ]
526 deviceId = deviceList[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700527 # Check assignment
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700528 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700529 if ip in master:
530 roleCheck = roleCheck and main.TRUE
531 else:
532 roleCheck = roleCheck and main.FALSE
533 main.log.error( "Error, controller " + ip + " is not" +
534 " master " + "of device " +
535 str( deviceId ) + ". Master is " +
536 repr( master ) + "." )
537 utilities.assert_equals(
538 expect=main.TRUE,
539 actual=roleCheck,
540 onpass="Switches were successfully reassigned to designated " +
541 "controller",
542 onfail="Switches were not successfully reassigned" )
543
544 def CASE3( self, main ):
545 """
546 Assign intents
547 """
548 import time
549 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700550 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700551 assert main, "main not defined"
552 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700553 assert main.CLIs, "main.CLIs not defined"
554 assert main.nodes, "main.nodes not defined"
Jon Halla440e872016-03-31 15:15:50 -0700555 try:
556 labels
557 except NameError:
558 main.log.error( "labels not defined, setting to []" )
559 labels = []
560 try:
561 data
562 except NameError:
563 main.log.error( "data not defined, setting to []" )
564 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700565 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700566 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700567 "assign predetermined host-to-host intents." +\
568 " After installation, check that the intent" +\
569 " is distributed to all nodes and the state" +\
570 " is INSTALLED"
571
572 # install onos-app-fwd
573 main.step( "Install reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700574 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700575 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700576 utilities.assert_equals( expect=main.TRUE, actual=installResults,
577 onpass="Install fwd successful",
578 onfail="Install fwd failed" )
579
580 main.step( "Check app ids" )
581 appCheck = main.TRUE
582 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700583 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700584 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700585 name="appToIDCheck-" + str( i ),
586 args=[] )
587 threads.append( t )
588 t.start()
589
590 for t in threads:
591 t.join()
592 appCheck = appCheck and t.result
593 if appCheck != main.TRUE:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700594 main.log.warn( onosCli.apps() )
595 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700596 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
597 onpass="App Ids seem to be correct",
598 onfail="Something is wrong with app Ids" )
599
600 main.step( "Discovering Hosts( Via pingall for now )" )
601 # FIXME: Once we have a host discovery mechanism, use that instead
602 # REACTIVE FWD test
603 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700604 passMsg = "Reactive Pingall test passed"
605 time1 = time.time()
606 pingResult = main.Mininet1.pingall()
607 time2 = time.time()
608 if not pingResult:
Jon Hallf37d44d2017-05-24 10:37:30 -0700609 main.log.warn( "First pingall failed. Trying again..." )
Jon Hall5cf14d52015-07-16 12:15:19 -0700610 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700611 passMsg += " on the second try"
612 utilities.assert_equals(
613 expect=main.TRUE,
614 actual=pingResult,
Jon Hallf37d44d2017-05-24 10:37:30 -0700615 onpass=passMsg,
Jon Hall96091e62015-09-21 17:34:17 -0700616 onfail="Reactive Pingall failed, " +
617 "one or more ping pairs failed" )
618 main.log.info( "Time for pingall: %2f seconds" %
619 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700620 # timeout for fwd flows
621 time.sleep( 11 )
622 # uninstall onos-app-fwd
623 main.step( "Uninstall reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700624 node = main.activeNodes[ 0 ]
625 uninstallResult = main.CLIs[ node ].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700626 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
627 onpass="Uninstall fwd successful",
628 onfail="Uninstall fwd failed" )
629
630 main.step( "Check app ids" )
631 threads = []
632 appCheck2 = main.TRUE
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700633 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700634 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700635 name="appToIDCheck-" + str( i ),
636 args=[] )
637 threads.append( t )
638 t.start()
639
640 for t in threads:
641 t.join()
642 appCheck2 = appCheck2 and t.result
643 if appCheck2 != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700644 node = main.activeNodes[ 0 ]
645 main.log.warn( main.CLIs[ node ].apps() )
646 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700647 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
648 onpass="App Ids seem to be correct",
649 onfail="Something is wrong with app Ids" )
650
651 main.step( "Add host intents via cli" )
652 intentIds = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700653 # TODO: move the host numbers to params
654 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700655 intentAddResult = True
656 hostResult = main.TRUE
657 for i in range( 8, 18 ):
658 main.log.info( "Adding host intent between h" + str( i ) +
659 " and h" + str( i + 10 ) )
660 host1 = "00:00:00:00:00:" + \
661 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
662 host2 = "00:00:00:00:00:" + \
663 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
664 # NOTE: getHost can return None
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700665 host1Dict = onosCli.getHost( host1 )
666 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700667 host1Id = None
668 host2Id = None
669 if host1Dict and host2Dict:
670 host1Id = host1Dict.get( 'id', None )
671 host2Id = host2Dict.get( 'id', None )
672 if host1Id and host2Id:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700673 nodeNum = ( i % len( main.activeNodes ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700674 node = main.activeNodes[ nodeNum ]
675 tmpId = main.CLIs[ node ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700676 if tmpId:
677 main.log.info( "Added intent with id: " + tmpId )
678 intentIds.append( tmpId )
679 else:
680 main.log.error( "addHostIntent returned: " +
681 repr( tmpId ) )
682 else:
683 main.log.error( "Error, getHost() failed for h" + str( i ) +
684 " and/or h" + str( i + 10 ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700685 node = main.activeNodes[ 0 ]
686 hosts = main.CLIs[ node ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700687 main.log.warn( "Hosts output: " )
688 try:
689 main.log.warn( json.dumps( json.loads( hosts ),
690 sort_keys=True,
691 indent=4,
692 separators=( ',', ': ' ) ) )
693 except ( ValueError, TypeError ):
694 main.log.warn( repr( hosts ) )
695 hostResult = main.FALSE
696 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
697 onpass="Found a host id for each host",
698 onfail="Error looking up host ids" )
699
700 intentStart = time.time()
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700701 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700702 main.log.info( "Submitted intents: " + str( intentIds ) )
703 main.log.info( "Intents in ONOS: " + str( onosIds ) )
704 for intent in intentIds:
705 if intent in onosIds:
706 pass # intent submitted is in onos
707 else:
708 intentAddResult = False
709 if intentAddResult:
710 intentStop = time.time()
711 else:
712 intentStop = None
713 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700714 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700715 intentStates = []
716 installedCheck = True
717 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
718 count = 0
719 try:
720 for intent in json.loads( intents ):
721 state = intent.get( 'state', None )
722 if "INSTALLED" not in state:
723 installedCheck = False
724 intentId = intent.get( 'id', None )
725 intentStates.append( ( intentId, state ) )
726 except ( ValueError, TypeError ):
727 main.log.exception( "Error parsing intents" )
728 # add submitted intents not in the store
729 tmplist = [ i for i, s in intentStates ]
730 missingIntents = False
731 for i in intentIds:
732 if i not in tmplist:
733 intentStates.append( ( i, " - " ) )
734 missingIntents = True
735 intentStates.sort()
736 for i, s in intentStates:
737 count += 1
738 main.log.info( "%-6s%-15s%-15s" %
739 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700740 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700741 try:
742 missing = False
743 if leaders:
744 parsedLeaders = json.loads( leaders )
745 main.log.warn( json.dumps( parsedLeaders,
746 sort_keys=True,
747 indent=4,
748 separators=( ',', ': ' ) ) )
749 # check for all intent partitions
750 topics = []
751 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700752 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700753 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700754 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700755 for topic in topics:
756 if topic not in ONOStopics:
757 main.log.error( "Error: " + topic +
758 " not in leaders" )
759 missing = True
760 else:
761 main.log.error( "leaders() returned None" )
762 except ( ValueError, TypeError ):
763 main.log.exception( "Error parsing leaders" )
764 main.log.error( repr( leaders ) )
765 # Check all nodes
766 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700767 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700768 response = main.CLIs[ i ].leaders( jsonFormat=False )
769 main.log.warn( str( main.CLIs[ i ].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700770 str( response ) )
771
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700772 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700773 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700774 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -0700775 parsedPartitions = json.loads( partitions )
776 main.log.warn( json.dumps( parsedPartitions,
777 sort_keys=True,
778 indent=4,
779 separators=( ',', ': ' ) ) )
780 # TODO check for a leader in all paritions
781 # TODO check for consistency among nodes
782 else:
783 main.log.error( "partitions() returned None" )
784 except ( ValueError, TypeError ):
785 main.log.exception( "Error parsing partitions" )
786 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700787 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700788 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700789 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -0700790 parsedPending = json.loads( pendingMap )
791 main.log.warn( json.dumps( parsedPending,
792 sort_keys=True,
793 indent=4,
794 separators=( ',', ': ' ) ) )
795 # TODO check something here?
796 else:
797 main.log.error( "pendingMap() returned None" )
798 except ( ValueError, TypeError ):
799 main.log.exception( "Error parsing pending map" )
800 main.log.error( repr( pendingMap ) )
801
802 intentAddResult = bool( intentAddResult and not missingIntents and
803 installedCheck )
804 if not intentAddResult:
805 main.log.error( "Error in pushing host intents to ONOS" )
806
807 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700808 for j in range( 100 ):
Jon Hall5cf14d52015-07-16 12:15:19 -0700809 correct = True
810 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700811 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700812 onosIds = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700813 ids = main.CLIs[ i ].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700814 onosIds.append( ids )
Jon Hallf37d44d2017-05-24 10:37:30 -0700815 main.log.debug( "Intents in " + main.CLIs[ i ].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700816 str( sorted( onosIds ) ) )
817 if sorted( ids ) != sorted( intentIds ):
818 main.log.warn( "Set of intent IDs doesn't match" )
819 correct = False
820 break
821 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700822 intents = json.loads( main.CLIs[ i ].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700823 for intent in intents:
824 if intent[ 'state' ] != "INSTALLED":
825 main.log.warn( "Intent " + intent[ 'id' ] +
826 " is " + intent[ 'state' ] )
827 correct = False
828 break
829 if correct:
830 break
831 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700832 time.sleep( 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700833 if not intentStop:
834 intentStop = time.time()
835 global gossipTime
836 gossipTime = intentStop - intentStart
837 main.log.info( "It took about " + str( gossipTime ) +
838 " seconds for all intents to appear in each node" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700839 gossipPeriod = int( main.params[ 'timers' ][ 'gossip' ] )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700840 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700841 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700842 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700843 onpass="ECM anti-entropy for intents worked within " +
844 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700845 onfail="Intent ECM anti-entropy took too long. " +
846 "Expected time:{}, Actual time:{}".format( maxGossipTime,
847 gossipTime ) )
848 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700849 intentAddResult = True
850
851 if not intentAddResult or "key" in pendingMap:
852 import time
853 installedCheck = True
854 main.log.info( "Sleeping 60 seconds to see if intents are found" )
855 time.sleep( 60 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700856 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700857 main.log.info( "Submitted intents: " + str( intentIds ) )
858 main.log.info( "Intents in ONOS: " + str( onosIds ) )
859 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700860 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700861 intentStates = []
862 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
863 count = 0
864 try:
865 for intent in json.loads( intents ):
866 # Iter through intents of a node
867 state = intent.get( 'state', None )
868 if "INSTALLED" not in state:
869 installedCheck = False
870 intentId = intent.get( 'id', None )
871 intentStates.append( ( intentId, state ) )
872 except ( ValueError, TypeError ):
873 main.log.exception( "Error parsing intents" )
874 # add submitted intents not in the store
875 tmplist = [ i for i, s in intentStates ]
876 for i in intentIds:
877 if i not in tmplist:
878 intentStates.append( ( i, " - " ) )
879 intentStates.sort()
880 for i, s in intentStates:
881 count += 1
882 main.log.info( "%-6s%-15s%-15s" %
883 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700884 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700885 try:
886 missing = False
887 if leaders:
888 parsedLeaders = json.loads( leaders )
889 main.log.warn( json.dumps( parsedLeaders,
890 sort_keys=True,
891 indent=4,
892 separators=( ',', ': ' ) ) )
893 # check for all intent partitions
894 # check for election
895 topics = []
896 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700897 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700898 # FIXME: this should only be after we start the app
899 topics.append( "org.onosproject.election" )
900 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700901 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700902 for topic in topics:
903 if topic not in ONOStopics:
904 main.log.error( "Error: " + topic +
905 " not in leaders" )
906 missing = True
907 else:
908 main.log.error( "leaders() returned None" )
909 except ( ValueError, TypeError ):
910 main.log.exception( "Error parsing leaders" )
911 main.log.error( repr( leaders ) )
912 # Check all nodes
913 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700914 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700915 node = main.CLIs[ i ]
916 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -0700917 main.log.warn( str( node.name ) + " leaders output: \n" +
918 str( response ) )
919
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700920 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700921 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700922 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -0700923 parsedPartitions = json.loads( partitions )
924 main.log.warn( json.dumps( parsedPartitions,
925 sort_keys=True,
926 indent=4,
927 separators=( ',', ': ' ) ) )
928 # TODO check for a leader in all paritions
929 # TODO check for consistency among nodes
930 else:
931 main.log.error( "partitions() returned None" )
932 except ( ValueError, TypeError ):
933 main.log.exception( "Error parsing partitions" )
934 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700935 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700936 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700937 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -0700938 parsedPending = json.loads( pendingMap )
939 main.log.warn( json.dumps( parsedPending,
940 sort_keys=True,
941 indent=4,
942 separators=( ',', ': ' ) ) )
943 # TODO check something here?
944 else:
945 main.log.error( "pendingMap() returned None" )
946 except ( ValueError, TypeError ):
947 main.log.exception( "Error parsing pending map" )
948 main.log.error( repr( pendingMap ) )
949
950 def CASE4( self, main ):
951 """
952 Ping across added host intents
953 """
954 import json
955 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700956 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700957 assert main, "main not defined"
958 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700959 assert main.CLIs, "main.CLIs not defined"
960 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700961 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700962 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700963 "functionality and check the state of " +\
964 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700965
Jon Hallf37d44d2017-05-24 10:37:30 -0700966 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700967 main.step( "Check Intent state" )
968 installedCheck = False
969 loopCount = 0
970 while not installedCheck and loopCount < 40:
971 installedCheck = True
972 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700973 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700974 intentStates = []
975 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
976 count = 0
977 # Iter through intents of a node
978 try:
979 for intent in json.loads( intents ):
980 state = intent.get( 'state', None )
981 if "INSTALLED" not in state:
982 installedCheck = False
983 intentId = intent.get( 'id', None )
984 intentStates.append( ( intentId, state ) )
985 except ( ValueError, TypeError ):
986 main.log.exception( "Error parsing intents." )
987 # Print states
988 intentStates.sort()
989 for i, s in intentStates:
990 count += 1
991 main.log.info( "%-6s%-15s%-15s" %
992 ( str( count ), str( i ), str( s ) ) )
993 if not installedCheck:
994 time.sleep( 1 )
995 loopCount += 1
996 utilities.assert_equals( expect=True, actual=installedCheck,
997 onpass="Intents are all INSTALLED",
998 onfail="Intents are not all in " +
999 "INSTALLED state" )
1000
Jon Hall9d2dcad2016-04-08 10:15:20 -07001001 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -07001002 PingResult = main.TRUE
1003 for i in range( 8, 18 ):
1004 ping = main.Mininet1.pingHost( src="h" + str( i ),
1005 target="h" + str( i + 10 ) )
1006 PingResult = PingResult and ping
1007 if ping == main.FALSE:
1008 main.log.warn( "Ping failed between h" + str( i ) +
1009 " and h" + str( i + 10 ) )
1010 elif ping == main.TRUE:
1011 main.log.info( "Ping test passed!" )
1012 # Don't set PingResult or you'd override failures
1013 if PingResult == main.FALSE:
1014 main.log.error(
1015 "Intents have not been installed correctly, pings failed." )
1016 # TODO: pretty print
1017 main.log.warn( "ONOS1 intents: " )
1018 try:
1019 tmpIntents = onosCli.intents()
1020 main.log.warn( json.dumps( json.loads( tmpIntents ),
1021 sort_keys=True,
1022 indent=4,
1023 separators=( ',', ': ' ) ) )
1024 except ( ValueError, TypeError ):
1025 main.log.warn( repr( tmpIntents ) )
1026 utilities.assert_equals(
1027 expect=main.TRUE,
1028 actual=PingResult,
1029 onpass="Intents have been installed correctly and pings work",
1030 onfail="Intents have not been installed correctly, pings failed." )
1031
Jon Hall5cf14d52015-07-16 12:15:19 -07001032 main.step( "Check leadership of topics" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001033 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001034 topicCheck = main.TRUE
1035 try:
1036 if leaders:
1037 parsedLeaders = json.loads( leaders )
1038 main.log.warn( json.dumps( parsedLeaders,
1039 sort_keys=True,
1040 indent=4,
1041 separators=( ',', ': ' ) ) )
1042 # check for all intent partitions
1043 # check for election
1044 # TODO: Look at Devices as topics now that it uses this system
1045 topics = []
1046 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001047 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001048 # FIXME: this should only be after we start the app
1049 # FIXME: topics.append( "org.onosproject.election" )
1050 # Print leaders output
1051 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -07001052 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -07001053 for topic in topics:
1054 if topic not in ONOStopics:
1055 main.log.error( "Error: " + topic +
1056 " not in leaders" )
1057 topicCheck = main.FALSE
1058 else:
1059 main.log.error( "leaders() returned None" )
1060 topicCheck = main.FALSE
1061 except ( ValueError, TypeError ):
1062 topicCheck = main.FALSE
1063 main.log.exception( "Error parsing leaders" )
1064 main.log.error( repr( leaders ) )
1065 # TODO: Check for a leader of these topics
1066 # Check all nodes
1067 if topicCheck:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001068 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001069 node = main.CLIs[ i ]
1070 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -07001071 main.log.warn( str( node.name ) + " leaders output: \n" +
1072 str( response ) )
1073
1074 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1075 onpass="intent Partitions is in leaders",
1076 onfail="Some topics were lost " )
1077 # Print partitions
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001078 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001079 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001080 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -07001081 parsedPartitions = json.loads( partitions )
1082 main.log.warn( json.dumps( parsedPartitions,
1083 sort_keys=True,
1084 indent=4,
1085 separators=( ',', ': ' ) ) )
1086 # TODO check for a leader in all paritions
1087 # TODO check for consistency among nodes
1088 else:
1089 main.log.error( "partitions() returned None" )
1090 except ( ValueError, TypeError ):
1091 main.log.exception( "Error parsing partitions" )
1092 main.log.error( repr( partitions ) )
1093 # Print Pending Map
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001094 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001095 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001096 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -07001097 parsedPending = json.loads( pendingMap )
1098 main.log.warn( json.dumps( parsedPending,
1099 sort_keys=True,
1100 indent=4,
1101 separators=( ',', ': ' ) ) )
1102 # TODO check something here?
1103 else:
1104 main.log.error( "pendingMap() returned None" )
1105 except ( ValueError, TypeError ):
1106 main.log.exception( "Error parsing pending map" )
1107 main.log.error( repr( pendingMap ) )
1108
1109 if not installedCheck:
1110 main.log.info( "Waiting 60 seconds to see if the state of " +
1111 "intents change" )
1112 time.sleep( 60 )
1113 # Print the intent states
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001114 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001115 intentStates = []
1116 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1117 count = 0
1118 # Iter through intents of a node
1119 try:
1120 for intent in json.loads( intents ):
1121 state = intent.get( 'state', None )
1122 if "INSTALLED" not in state:
1123 installedCheck = False
1124 intentId = intent.get( 'id', None )
1125 intentStates.append( ( intentId, state ) )
1126 except ( ValueError, TypeError ):
1127 main.log.exception( "Error parsing intents." )
1128 intentStates.sort()
1129 for i, s in intentStates:
1130 count += 1
1131 main.log.info( "%-6s%-15s%-15s" %
1132 ( str( count ), str( i ), str( s ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001133 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001134 try:
1135 missing = False
1136 if leaders:
1137 parsedLeaders = json.loads( leaders )
1138 main.log.warn( json.dumps( parsedLeaders,
1139 sort_keys=True,
1140 indent=4,
1141 separators=( ',', ': ' ) ) )
1142 # check for all intent partitions
1143 # check for election
1144 topics = []
1145 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001146 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001147 # FIXME: this should only be after we start the app
1148 topics.append( "org.onosproject.election" )
1149 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -07001150 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -07001151 for topic in topics:
1152 if topic not in ONOStopics:
1153 main.log.error( "Error: " + topic +
1154 " not in leaders" )
1155 missing = True
1156 else:
1157 main.log.error( "leaders() returned None" )
1158 except ( ValueError, TypeError ):
1159 main.log.exception( "Error parsing leaders" )
1160 main.log.error( repr( leaders ) )
1161 if missing:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001162 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001163 node = main.CLIs[ i ]
1164 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -07001165 main.log.warn( str( node.name ) + " leaders output: \n" +
1166 str( response ) )
1167
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001168 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001169 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001170 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -07001171 parsedPartitions = json.loads( partitions )
1172 main.log.warn( json.dumps( parsedPartitions,
1173 sort_keys=True,
1174 indent=4,
1175 separators=( ',', ': ' ) ) )
1176 # TODO check for a leader in all paritions
1177 # TODO check for consistency among nodes
1178 else:
1179 main.log.error( "partitions() returned None" )
1180 except ( ValueError, TypeError ):
1181 main.log.exception( "Error parsing partitions" )
1182 main.log.error( repr( partitions ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001183 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001184 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001185 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -07001186 parsedPending = json.loads( pendingMap )
1187 main.log.warn( json.dumps( parsedPending,
1188 sort_keys=True,
1189 indent=4,
1190 separators=( ',', ': ' ) ) )
1191 # TODO check something here?
1192 else:
1193 main.log.error( "pendingMap() returned None" )
1194 except ( ValueError, TypeError ):
1195 main.log.exception( "Error parsing pending map" )
1196 main.log.error( repr( pendingMap ) )
1197 # Print flowrules
Jon Hall41d39f12016-04-11 22:54:35 -07001198 main.log.debug( onosCli.flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001199 main.step( "Wait a minute then ping again" )
1200 # the wait is above
1201 PingResult = main.TRUE
1202 for i in range( 8, 18 ):
1203 ping = main.Mininet1.pingHost( src="h" + str( i ),
1204 target="h" + str( i + 10 ) )
1205 PingResult = PingResult and ping
1206 if ping == main.FALSE:
1207 main.log.warn( "Ping failed between h" + str( i ) +
1208 " and h" + str( i + 10 ) )
1209 elif ping == main.TRUE:
1210 main.log.info( "Ping test passed!" )
1211 # Don't set PingResult or you'd override failures
1212 if PingResult == main.FALSE:
1213 main.log.error(
1214 "Intents have not been installed correctly, pings failed." )
1215 # TODO: pretty print
1216 main.log.warn( "ONOS1 intents: " )
1217 try:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001218 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001219 main.log.warn( json.dumps( json.loads( tmpIntents ),
1220 sort_keys=True,
1221 indent=4,
1222 separators=( ',', ': ' ) ) )
1223 except ( ValueError, TypeError ):
1224 main.log.warn( repr( tmpIntents ) )
1225 utilities.assert_equals(
1226 expect=main.TRUE,
1227 actual=PingResult,
1228 onpass="Intents have been installed correctly and pings work",
1229 onfail="Intents have not been installed correctly, pings failed." )
1230
1231 def CASE5( self, main ):
1232 """
1233 Reading state of ONOS
1234 """
1235 import json
1236 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001237 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001238 assert main, "main not defined"
1239 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001240 assert main.CLIs, "main.CLIs not defined"
1241 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001242
1243 main.case( "Setting up and gathering data for current state" )
1244 # The general idea for this test case is to pull the state of
1245 # ( intents,flows, topology,... ) from each ONOS node
1246 # We can then compare them with each other and also with past states
1247
1248 main.step( "Check that each switch has a master" )
1249 global mastershipState
1250 mastershipState = '[]'
1251
1252 # Assert that each device has a master
1253 rolesNotNull = main.TRUE
1254 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001255 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001256 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001257 name="rolesNotNull-" + str( i ),
1258 args=[] )
1259 threads.append( t )
1260 t.start()
1261
1262 for t in threads:
1263 t.join()
1264 rolesNotNull = rolesNotNull and t.result
1265 utilities.assert_equals(
1266 expect=main.TRUE,
1267 actual=rolesNotNull,
1268 onpass="Each device has a master",
1269 onfail="Some devices don't have a master assigned" )
1270
1271 main.step( "Get the Mastership of each switch from each controller" )
1272 ONOSMastership = []
1273 mastershipCheck = main.FALSE
1274 consistentMastership = True
1275 rolesResults = True
1276 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001277 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001278 t = main.Thread( target=main.CLIs[ i ].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001279 name="roles-" + str( i ),
1280 args=[] )
1281 threads.append( t )
1282 t.start()
1283
1284 for t in threads:
1285 t.join()
1286 ONOSMastership.append( t.result )
1287
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001288 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001289 node = str( main.activeNodes[ i ] + 1 )
1290 if not ONOSMastership[ i ] or "Error" in ONOSMastership[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001291 main.log.error( "Error in getting ONOS" + node + " roles" )
1292 main.log.warn( "ONOS" + node + " mastership response: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001293 repr( ONOSMastership[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001294 rolesResults = False
1295 utilities.assert_equals(
1296 expect=True,
1297 actual=rolesResults,
1298 onpass="No error in reading roles output",
1299 onfail="Error in reading roles from ONOS" )
1300
1301 main.step( "Check for consistency in roles from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001302 if all( [ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001303 main.log.info(
1304 "Switch roles are consistent across all ONOS nodes" )
1305 else:
1306 consistentMastership = False
1307 utilities.assert_equals(
1308 expect=True,
1309 actual=consistentMastership,
1310 onpass="Switch roles are consistent across all ONOS nodes",
1311 onfail="ONOS nodes have different views of switch roles" )
1312
1313 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001314 for i in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001315 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001316 try:
1317 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001318 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001319 json.dumps(
1320 json.loads( ONOSMastership[ i ] ),
1321 sort_keys=True,
1322 indent=4,
1323 separators=( ',', ': ' ) ) )
1324 except ( ValueError, TypeError ):
1325 main.log.warn( repr( ONOSMastership[ i ] ) )
1326 elif rolesResults and consistentMastership:
1327 mastershipCheck = main.TRUE
1328 mastershipState = ONOSMastership[ 0 ]
1329
1330 main.step( "Get the intents from each controller" )
1331 global intentState
1332 intentState = []
1333 ONOSIntents = []
1334 intentCheck = main.FALSE
1335 consistentIntents = True
1336 intentsResults = True
1337 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001338 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001339 t = main.Thread( target=main.CLIs[ i ].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001340 name="intents-" + str( i ),
1341 args=[],
1342 kwargs={ 'jsonFormat': True } )
1343 threads.append( t )
1344 t.start()
1345
1346 for t in threads:
1347 t.join()
1348 ONOSIntents.append( t.result )
1349
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001350 for i in range( len( ONOSIntents ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001351 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001352 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001353 main.log.error( "Error in getting ONOS" + node + " intents" )
1354 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001355 repr( ONOSIntents[ i ] ) )
1356 intentsResults = False
1357 utilities.assert_equals(
1358 expect=True,
1359 actual=intentsResults,
1360 onpass="No error in reading intents output",
1361 onfail="Error in reading intents from ONOS" )
1362
1363 main.step( "Check for consistency in Intents from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001364 if all( [ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001365 main.log.info( "Intents are consistent across all ONOS " +
1366 "nodes" )
1367 else:
1368 consistentIntents = False
1369 main.log.error( "Intents not consistent" )
1370 utilities.assert_equals(
1371 expect=True,
1372 actual=consistentIntents,
1373 onpass="Intents are consistent across all ONOS nodes",
1374 onfail="ONOS nodes have different views of intents" )
1375
1376 if intentsResults:
1377 # Try to make it easy to figure out what is happening
1378 #
1379 # Intent ONOS1 ONOS2 ...
1380 # 0x01 INSTALLED INSTALLING
1381 # ... ... ...
1382 # ... ... ...
1383 title = " Id"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001384 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001385 title += " " * 10 + "ONOS" + str( n + 1 )
1386 main.log.warn( title )
1387 # get all intent keys in the cluster
1388 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001389 try:
1390 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001391 for nodeStr in ONOSIntents:
1392 node = json.loads( nodeStr )
1393 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001394 keys.append( intent.get( 'id' ) )
1395 keys = set( keys )
1396 # For each intent key, print the state on each node
1397 for key in keys:
1398 row = "%-13s" % key
1399 for nodeStr in ONOSIntents:
1400 node = json.loads( nodeStr )
1401 for intent in node:
1402 if intent.get( 'id', "Error" ) == key:
1403 row += "%-15s" % intent.get( 'state' )
1404 main.log.warn( row )
1405 # End of intent state table
1406 except ValueError as e:
1407 main.log.exception( e )
1408 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001409
1410 if intentsResults and not consistentIntents:
1411 # print the json objects
Jon Hallf37d44d2017-05-24 10:37:30 -07001412 n = str( main.activeNodes[ -1 ] + 1 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001413 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001414 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1415 sort_keys=True,
1416 indent=4,
1417 separators=( ',', ': ' ) ) )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001418 for i in range( len( ONOSIntents ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001419 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001420 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001421 main.log.debug( "ONOS" + node + " intents: " )
Jon Hallf37d44d2017-05-24 10:37:30 -07001422 main.log.debug( json.dumps( json.loads( ONOSIntents[ i ] ),
Jon Hall5cf14d52015-07-16 12:15:19 -07001423 sort_keys=True,
1424 indent=4,
1425 separators=( ',', ': ' ) ) )
1426 else:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001427 main.log.debug( "ONOS" + node + " intents match ONOS" +
1428 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001429 elif intentsResults and consistentIntents:
1430 intentCheck = main.TRUE
1431 intentState = ONOSIntents[ 0 ]
1432
1433 main.step( "Get the flows from each controller" )
1434 global flowState
1435 flowState = []
1436 ONOSFlows = []
1437 ONOSFlowsJson = []
1438 flowCheck = main.FALSE
1439 consistentFlows = True
1440 flowsResults = True
1441 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001442 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001443 t = main.Thread( target=main.CLIs[ i ].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001444 name="flows-" + str( i ),
1445 args=[],
1446 kwargs={ 'jsonFormat': True } )
1447 threads.append( t )
1448 t.start()
1449
1450 # NOTE: Flows command can take some time to run
Jon Hallf37d44d2017-05-24 10:37:30 -07001451 time.sleep( 30 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001452 for t in threads:
1453 t.join()
1454 result = t.result
1455 ONOSFlows.append( result )
1456
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001457 for i in range( len( ONOSFlows ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001458 num = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001459 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1460 main.log.error( "Error in getting ONOS" + num + " flows" )
1461 main.log.warn( "ONOS" + num + " flows response: " +
1462 repr( ONOSFlows[ i ] ) )
1463 flowsResults = False
1464 ONOSFlowsJson.append( None )
1465 else:
1466 try:
1467 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1468 except ( ValueError, TypeError ):
1469 # FIXME: change this to log.error?
1470 main.log.exception( "Error in parsing ONOS" + num +
1471 " response as json." )
1472 main.log.error( repr( ONOSFlows[ i ] ) )
1473 ONOSFlowsJson.append( None )
1474 flowsResults = False
1475 utilities.assert_equals(
1476 expect=True,
1477 actual=flowsResults,
1478 onpass="No error in reading flows output",
1479 onfail="Error in reading flows from ONOS" )
1480
1481 main.step( "Check for consistency in Flows from each controller" )
1482 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1483 if all( tmp ):
1484 main.log.info( "Flow count is consistent across all ONOS nodes" )
1485 else:
1486 consistentFlows = False
1487 utilities.assert_equals(
1488 expect=True,
1489 actual=consistentFlows,
1490 onpass="The flow count is consistent across all ONOS nodes",
1491 onfail="ONOS nodes have different flow counts" )
1492
1493 if flowsResults and not consistentFlows:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001494 for i in range( len( ONOSFlows ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001495 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001496 try:
1497 main.log.warn(
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001498 "ONOS" + node + " flows: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001499 json.dumps( json.loads( ONOSFlows[ i ] ), sort_keys=True,
Jon Hall5cf14d52015-07-16 12:15:19 -07001500 indent=4, separators=( ',', ': ' ) ) )
1501 except ( ValueError, TypeError ):
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001502 main.log.warn( "ONOS" + node + " flows: " +
1503 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001504 elif flowsResults and consistentFlows:
1505 flowCheck = main.TRUE
1506 flowState = ONOSFlows[ 0 ]
1507
1508 main.step( "Get the OF Table entries" )
1509 global flows
1510 flows = []
1511 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001512 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001513 if flowCheck == main.FALSE:
1514 for table in flows:
1515 main.log.warn( table )
1516 # TODO: Compare switch flow tables with ONOS flow tables
1517
1518 main.step( "Start continuous pings" )
1519 main.Mininet2.pingLong(
1520 src=main.params[ 'PING' ][ 'source1' ],
1521 target=main.params[ 'PING' ][ 'target1' ],
1522 pingTime=500 )
1523 main.Mininet2.pingLong(
1524 src=main.params[ 'PING' ][ 'source2' ],
1525 target=main.params[ 'PING' ][ 'target2' ],
1526 pingTime=500 )
1527 main.Mininet2.pingLong(
1528 src=main.params[ 'PING' ][ 'source3' ],
1529 target=main.params[ 'PING' ][ 'target3' ],
1530 pingTime=500 )
1531 main.Mininet2.pingLong(
1532 src=main.params[ 'PING' ][ 'source4' ],
1533 target=main.params[ 'PING' ][ 'target4' ],
1534 pingTime=500 )
1535 main.Mininet2.pingLong(
1536 src=main.params[ 'PING' ][ 'source5' ],
1537 target=main.params[ 'PING' ][ 'target5' ],
1538 pingTime=500 )
1539 main.Mininet2.pingLong(
1540 src=main.params[ 'PING' ][ 'source6' ],
1541 target=main.params[ 'PING' ][ 'target6' ],
1542 pingTime=500 )
1543 main.Mininet2.pingLong(
1544 src=main.params[ 'PING' ][ 'source7' ],
1545 target=main.params[ 'PING' ][ 'target7' ],
1546 pingTime=500 )
1547 main.Mininet2.pingLong(
1548 src=main.params[ 'PING' ][ 'source8' ],
1549 target=main.params[ 'PING' ][ 'target8' ],
1550 pingTime=500 )
1551 main.Mininet2.pingLong(
1552 src=main.params[ 'PING' ][ 'source9' ],
1553 target=main.params[ 'PING' ][ 'target9' ],
1554 pingTime=500 )
1555 main.Mininet2.pingLong(
1556 src=main.params[ 'PING' ][ 'source10' ],
1557 target=main.params[ 'PING' ][ 'target10' ],
1558 pingTime=500 )
1559
1560 main.step( "Collecting topology information from ONOS" )
1561 devices = []
1562 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001563 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001564 t = main.Thread( target=main.CLIs[ i ].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001565 name="devices-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001566 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001567 threads.append( t )
1568 t.start()
1569
1570 for t in threads:
1571 t.join()
1572 devices.append( t.result )
1573 hosts = []
1574 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001575 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001576 t = main.Thread( target=main.CLIs[ i ].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001577 name="hosts-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001578 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001579 threads.append( t )
1580 t.start()
1581
1582 for t in threads:
1583 t.join()
1584 try:
1585 hosts.append( json.loads( t.result ) )
1586 except ( ValueError, TypeError ):
1587 # FIXME: better handling of this, print which node
1588 # Maybe use thread name?
1589 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001590 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001591 hosts.append( None )
1592
1593 ports = []
1594 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001595 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001596 t = main.Thread( target=main.CLIs[ i ].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001597 name="ports-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001598 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001599 threads.append( t )
1600 t.start()
1601
1602 for t in threads:
1603 t.join()
1604 ports.append( t.result )
1605 links = []
1606 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001607 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001608 t = main.Thread( target=main.CLIs[ i ].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001609 name="links-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001610 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001611 threads.append( t )
1612 t.start()
1613
1614 for t in threads:
1615 t.join()
1616 links.append( t.result )
1617 clusters = []
1618 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001619 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001620 t = main.Thread( target=main.CLIs[ i ].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001621 name="clusters-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001622 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001623 threads.append( t )
1624 t.start()
1625
1626 for t in threads:
1627 t.join()
1628 clusters.append( t.result )
1629 # Compare json objects for hosts and dataplane clusters
1630
1631 # hosts
1632 main.step( "Host view is consistent across ONOS nodes" )
1633 consistentHostsResult = main.TRUE
1634 for controller in range( len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001635 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001636 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001637 if hosts[ controller ] == hosts[ 0 ]:
1638 continue
1639 else: # hosts not consistent
1640 main.log.error( "hosts from ONOS" +
1641 controllerStr +
1642 " is inconsistent with ONOS1" )
1643 main.log.warn( repr( hosts[ controller ] ) )
1644 consistentHostsResult = main.FALSE
1645
1646 else:
1647 main.log.error( "Error in getting ONOS hosts from ONOS" +
1648 controllerStr )
1649 consistentHostsResult = main.FALSE
1650 main.log.warn( "ONOS" + controllerStr +
1651 " hosts response: " +
1652 repr( hosts[ controller ] ) )
1653 utilities.assert_equals(
1654 expect=main.TRUE,
1655 actual=consistentHostsResult,
1656 onpass="Hosts view is consistent across all ONOS nodes",
1657 onfail="ONOS nodes have different views of hosts" )
1658
1659 main.step( "Each host has an IP address" )
1660 ipResult = main.TRUE
1661 for controller in range( 0, len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001662 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001663 if hosts[ controller ]:
1664 for host in hosts[ controller ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07001665 if not host.get( 'ipAddresses', [] ):
Jon Hallf3d16e72015-12-16 17:45:08 -08001666 main.log.error( "Error with host ips on controller" +
1667 controllerStr + ": " + str( host ) )
1668 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001669 utilities.assert_equals(
1670 expect=main.TRUE,
1671 actual=ipResult,
1672 onpass="The ips of the hosts aren't empty",
1673 onfail="The ip of at least one host is missing" )
1674
1675 # Strongly connected clusters of devices
1676 main.step( "Cluster view is consistent across ONOS nodes" )
1677 consistentClustersResult = main.TRUE
1678 for controller in range( len( clusters ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001679 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001680 if "Error" not in clusters[ controller ]:
1681 if clusters[ controller ] == clusters[ 0 ]:
1682 continue
1683 else: # clusters not consistent
1684 main.log.error( "clusters from ONOS" + controllerStr +
1685 " is inconsistent with ONOS1" )
1686 consistentClustersResult = main.FALSE
1687
1688 else:
1689 main.log.error( "Error in getting dataplane clusters " +
1690 "from ONOS" + controllerStr )
1691 consistentClustersResult = main.FALSE
1692 main.log.warn( "ONOS" + controllerStr +
1693 " clusters response: " +
1694 repr( clusters[ controller ] ) )
1695 utilities.assert_equals(
1696 expect=main.TRUE,
1697 actual=consistentClustersResult,
1698 onpass="Clusters view is consistent across all ONOS nodes",
1699 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001700 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001701 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001702
Jon Hall5cf14d52015-07-16 12:15:19 -07001703 # there should always only be one cluster
1704 main.step( "Cluster view correct across ONOS nodes" )
1705 try:
1706 numClusters = len( json.loads( clusters[ 0 ] ) )
1707 except ( ValueError, TypeError ):
1708 main.log.exception( "Error parsing clusters[0]: " +
1709 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001710 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001711 clusterResults = main.FALSE
1712 if numClusters == 1:
1713 clusterResults = main.TRUE
1714 utilities.assert_equals(
1715 expect=1,
1716 actual=numClusters,
1717 onpass="ONOS shows 1 SCC",
1718 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1719
1720 main.step( "Comparing ONOS topology to MN" )
1721 devicesResults = main.TRUE
1722 linksResults = main.TRUE
1723 hostsResults = main.TRUE
1724 mnSwitches = main.Mininet1.getSwitches()
1725 mnLinks = main.Mininet1.getLinks()
1726 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001727 for controller in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001728 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001729 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001730 "Error" not in devices[ controller ] and\
1731 "Error" not in ports[ controller ]:
1732 currentDevicesResult = main.Mininet1.compareSwitches(
1733 mnSwitches,
1734 json.loads( devices[ controller ] ),
1735 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001736 else:
1737 currentDevicesResult = main.FALSE
1738 utilities.assert_equals( expect=main.TRUE,
1739 actual=currentDevicesResult,
1740 onpass="ONOS" + controllerStr +
1741 " Switches view is correct",
1742 onfail="ONOS" + controllerStr +
1743 " Switches view is incorrect" )
1744 if links[ controller ] and "Error" not in links[ controller ]:
1745 currentLinksResult = main.Mininet1.compareLinks(
1746 mnSwitches, mnLinks,
1747 json.loads( links[ controller ] ) )
1748 else:
1749 currentLinksResult = main.FALSE
1750 utilities.assert_equals( expect=main.TRUE,
1751 actual=currentLinksResult,
1752 onpass="ONOS" + controllerStr +
1753 " links view is correct",
1754 onfail="ONOS" + controllerStr +
1755 " links view is incorrect" )
1756
Jon Hall657cdf62015-12-17 14:40:51 -08001757 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001758 currentHostsResult = main.Mininet1.compareHosts(
1759 mnHosts,
1760 hosts[ controller ] )
1761 else:
1762 currentHostsResult = main.FALSE
1763 utilities.assert_equals( expect=main.TRUE,
1764 actual=currentHostsResult,
1765 onpass="ONOS" + controllerStr +
1766 " hosts exist in Mininet",
1767 onfail="ONOS" + controllerStr +
1768 " hosts don't match Mininet" )
1769
1770 devicesResults = devicesResults and currentDevicesResult
1771 linksResults = linksResults and currentLinksResult
1772 hostsResults = hostsResults and currentHostsResult
1773
1774 main.step( "Device information is correct" )
1775 utilities.assert_equals(
1776 expect=main.TRUE,
1777 actual=devicesResults,
1778 onpass="Device information is correct",
1779 onfail="Device information is incorrect" )
1780
1781 main.step( "Links are correct" )
1782 utilities.assert_equals(
1783 expect=main.TRUE,
1784 actual=linksResults,
1785 onpass="Link are correct",
1786 onfail="Links are incorrect" )
1787
1788 main.step( "Hosts are correct" )
1789 utilities.assert_equals(
1790 expect=main.TRUE,
1791 actual=hostsResults,
1792 onpass="Hosts are correct",
1793 onfail="Hosts are incorrect" )
1794
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001795 def CASE61( self, main ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001796 """
1797 The Failure case.
1798 """
Jon Halle1a3b752015-07-22 13:02:46 -07001799 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001800 assert main, "main not defined"
1801 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001802 assert main.CLIs, "main.CLIs not defined"
1803 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001804 main.case( "Kill minority of ONOS nodes" )
Jon Hall96091e62015-09-21 17:34:17 -07001805
1806 main.step( "Checking ONOS Logs for errors" )
1807 for node in main.nodes:
1808 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1809 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1810
Jon Hall3b489db2015-10-05 14:38:37 -07001811 n = len( main.nodes ) # Number of nodes
1812 p = ( ( n + 1 ) / 2 ) + 1 # Number of partitions
1813 main.kill = [ 0 ] # ONOS node to kill, listed by index in main.nodes
1814 if n > 3:
1815 main.kill.append( p - 1 )
1816 # NOTE: This only works for cluster sizes of 3,5, or 7.
1817
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001818 main.step( "Kill " + str( len( main.kill ) ) + " ONOS nodes" )
Jon Hall3b489db2015-10-05 14:38:37 -07001819 killResults = main.TRUE
1820 for i in main.kill:
1821 killResults = killResults and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001822 main.ONOSbench.onosKill( main.nodes[ i ].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001823 main.activeNodes.remove( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001824 utilities.assert_equals( expect=main.TRUE, actual=killResults,
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001825 onpass="ONOS nodes killed successfully",
1826 onfail="ONOS nodes NOT successfully killed" )
1827
Jon Halld2871c22016-07-26 11:01:14 -07001828 main.step( "Checking ONOS nodes" )
1829 nodeResults = utilities.retry( main.HA.nodesCheck,
1830 False,
Jon Hallf37d44d2017-05-24 10:37:30 -07001831 args=[ main.activeNodes ],
Jon Halld2871c22016-07-26 11:01:14 -07001832 sleep=15,
1833 attempts=5 )
1834
1835 utilities.assert_equals( expect=True, actual=nodeResults,
1836 onpass="Nodes check successful",
1837 onfail="Nodes check NOT successful" )
1838
1839 if not nodeResults:
1840 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001841 cli = main.CLIs[ i ]
Jon Halld2871c22016-07-26 11:01:14 -07001842 main.log.debug( "{} components not ACTIVE: \n{}".format(
1843 cli.name,
1844 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
1845 main.log.error( "Failed to start ONOS, stopping test" )
1846 main.cleanup()
1847 main.exit()
1848
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001849 def CASE62( self, main ):
1850 """
1851 The bring up stopped nodes
1852 """
1853 import time
1854 assert main.numCtrls, "main.numCtrls not defined"
1855 assert main, "main not defined"
1856 assert utilities.assert_equals, "utilities.assert_equals not defined"
1857 assert main.CLIs, "main.CLIs not defined"
1858 assert main.nodes, "main.nodes not defined"
1859 assert main.kill, "main.kill not defined"
1860 main.case( "Restart minority of ONOS nodes" )
1861
1862 main.step( "Restarting " + str( len( main.kill ) ) + " ONOS nodes" )
1863 startResults = main.TRUE
1864 restartTime = time.time()
1865 for i in main.kill:
1866 startResults = startResults and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001867 main.ONOSbench.onosStart( main.nodes[ i ].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001868 utilities.assert_equals( expect=main.TRUE, actual=startResults,
1869 onpass="ONOS nodes started successfully",
1870 onfail="ONOS nodes NOT successfully started" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001871
1872 main.step( "Checking if ONOS is up yet" )
1873 count = 0
1874 onosIsupResult = main.FALSE
1875 while onosIsupResult == main.FALSE and count < 10:
Jon Hall3b489db2015-10-05 14:38:37 -07001876 onosIsupResult = main.TRUE
1877 for i in main.kill:
1878 onosIsupResult = onosIsupResult and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001879 main.ONOSbench.isup( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07001880 count = count + 1
Jon Hall5cf14d52015-07-16 12:15:19 -07001881 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1882 onpass="ONOS restarted successfully",
1883 onfail="ONOS restart NOT successful" )
1884
Jon Halle1a3b752015-07-22 13:02:46 -07001885 main.step( "Restarting ONOS main.CLIs" )
Jon Hall3b489db2015-10-05 14:38:37 -07001886 cliResults = main.TRUE
1887 for i in main.kill:
1888 cliResults = cliResults and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001889 main.CLIs[ i ].startOnosCli( main.nodes[ i ].ip_address )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001890 main.activeNodes.append( i )
Jon Hall5cf14d52015-07-16 12:15:19 -07001891 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1892 onpass="ONOS cli restarted",
1893 onfail="ONOS cli did not restart" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001894 main.activeNodes.sort()
1895 try:
1896 assert list( set( main.activeNodes ) ) == main.activeNodes,\
1897 "List of active nodes has duplicates, this likely indicates something was run out of order"
1898 except AssertionError:
1899 main.log.exception( "" )
1900 main.cleanup()
1901 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -07001902
1903 # Grab the time of restart so we chan check how long the gossip
1904 # protocol has had time to work
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001905 main.restartTime = time.time() - restartTime
Jon Hall5cf14d52015-07-16 12:15:19 -07001906 main.log.debug( "Restart time: " + str( main.restartTime ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001907 # TODO: MAke this configurable. Also, we are breaking the above timer
Jon Halld2871c22016-07-26 11:01:14 -07001908 main.step( "Checking ONOS nodes" )
1909 nodeResults = utilities.retry( main.HA.nodesCheck,
1910 False,
Jon Hallf37d44d2017-05-24 10:37:30 -07001911 args=[ main.activeNodes ],
Jon Halld2871c22016-07-26 11:01:14 -07001912 sleep=15,
1913 attempts=5 )
1914
1915 utilities.assert_equals( expect=True, actual=nodeResults,
1916 onpass="Nodes check successful",
1917 onfail="Nodes check NOT successful" )
1918
1919 if not nodeResults:
1920 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001921 cli = main.CLIs[ i ]
Jon Halld2871c22016-07-26 11:01:14 -07001922 main.log.debug( "{} components not ACTIVE: \n{}".format(
1923 cli.name,
1924 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
1925 main.log.error( "Failed to start ONOS, stopping test" )
1926 main.cleanup()
1927 main.exit()
1928
Jon Hallf37d44d2017-05-24 10:37:30 -07001929 node = main.activeNodes[ 0 ]
1930 main.log.debug( main.CLIs[ node ].nodes( jsonFormat=False ) )
1931 main.log.debug( main.CLIs[ node ].leaders( jsonFormat=False ) )
1932 main.log.debug( main.CLIs[ node ].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001933
Jon Halla440e872016-03-31 15:15:50 -07001934 main.step( "Rerun for election on the node(s) that were killed" )
1935 runResults = main.TRUE
1936 for i in main.kill:
1937 runResults = runResults and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001938 main.CLIs[ i ].electionTestRun()
Jon Halla440e872016-03-31 15:15:50 -07001939 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1940 onpass="ONOS nodes reran for election topic",
1941 onfail="Errror rerunning for election" )
1942
Jon Hall5cf14d52015-07-16 12:15:19 -07001943 def CASE7( self, main ):
1944 """
1945 Check state after ONOS failure
1946 """
1947 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001948 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001949 assert main, "main not defined"
1950 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001951 assert main.CLIs, "main.CLIs not defined"
1952 assert main.nodes, "main.nodes not defined"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001953 try:
1954 main.kill
1955 except AttributeError:
1956 main.kill = []
1957
Jon Hall5cf14d52015-07-16 12:15:19 -07001958 main.case( "Running ONOS Constant State Tests" )
1959
1960 main.step( "Check that each switch has a master" )
1961 # Assert that each device has a master
1962 rolesNotNull = main.TRUE
1963 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001964 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001965 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001966 name="rolesNotNull-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001967 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001968 threads.append( t )
1969 t.start()
1970
1971 for t in threads:
1972 t.join()
1973 rolesNotNull = rolesNotNull and t.result
1974 utilities.assert_equals(
1975 expect=main.TRUE,
1976 actual=rolesNotNull,
1977 onpass="Each device has a master",
1978 onfail="Some devices don't have a master assigned" )
1979
1980 main.step( "Read device roles from ONOS" )
1981 ONOSMastership = []
Jon Halla440e872016-03-31 15:15:50 -07001982 mastershipCheck = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001983 consistentMastership = True
1984 rolesResults = True
1985 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001986 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001987 t = main.Thread( target=main.CLIs[ i ].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001988 name="roles-" + str( i ),
1989 args=[] )
1990 threads.append( t )
1991 t.start()
1992
1993 for t in threads:
1994 t.join()
1995 ONOSMastership.append( t.result )
1996
Jon Hallb3ed8ed2015-10-28 16:43:55 -07001997 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001998 node = str( main.activeNodes[ i ] + 1 )
1999 if not ONOSMastership[ i ] or "Error" in ONOSMastership[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002000 main.log.error( "Error in getting ONOS" + node + " roles" )
2001 main.log.warn( "ONOS" + node + " mastership response: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07002002 repr( ONOSMastership[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002003 rolesResults = False
2004 utilities.assert_equals(
2005 expect=True,
2006 actual=rolesResults,
2007 onpass="No error in reading roles output",
2008 onfail="Error in reading roles from ONOS" )
2009
2010 main.step( "Check for consistency in roles from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002011 if all( [ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002012 main.log.info(
2013 "Switch roles are consistent across all ONOS nodes" )
2014 else:
2015 consistentMastership = False
2016 utilities.assert_equals(
2017 expect=True,
2018 actual=consistentMastership,
2019 onpass="Switch roles are consistent across all ONOS nodes",
2020 onfail="ONOS nodes have different views of switch roles" )
2021
2022 if rolesResults and not consistentMastership:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002023 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002024 node = str( main.activeNodes[ i ] + 1 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002025 main.log.warn( "ONOS" + node + " roles: ",
2026 json.dumps( json.loads( ONOSMastership[ i ] ),
2027 sort_keys=True,
2028 indent=4,
2029 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07002030 elif rolesResults and consistentMastership:
2031 mastershipCheck = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002032
2033 # NOTE: we expect mastership to change on controller failure
Jon Hall5cf14d52015-07-16 12:15:19 -07002034
2035 main.step( "Get the intents and compare across all nodes" )
2036 ONOSIntents = []
2037 intentCheck = main.FALSE
2038 consistentIntents = True
2039 intentsResults = True
2040 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002041 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002042 t = main.Thread( target=main.CLIs[ i ].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07002043 name="intents-" + str( i ),
2044 args=[],
2045 kwargs={ 'jsonFormat': True } )
2046 threads.append( t )
2047 t.start()
2048
2049 for t in threads:
2050 t.join()
2051 ONOSIntents.append( t.result )
2052
Jon Hallf37d44d2017-05-24 10:37:30 -07002053 for i in range( len( ONOSIntents ) ):
2054 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002055 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002056 main.log.error( "Error in getting ONOS" + node + " intents" )
2057 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002058 repr( ONOSIntents[ i ] ) )
2059 intentsResults = False
2060 utilities.assert_equals(
2061 expect=True,
2062 actual=intentsResults,
2063 onpass="No error in reading intents output",
2064 onfail="Error in reading intents from ONOS" )
2065
2066 main.step( "Check for consistency in Intents from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002067 if all( [ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002068 main.log.info( "Intents are consistent across all ONOS " +
2069 "nodes" )
2070 else:
2071 consistentIntents = False
2072
2073 # Try to make it easy to figure out what is happening
2074 #
2075 # Intent ONOS1 ONOS2 ...
2076 # 0x01 INSTALLED INSTALLING
2077 # ... ... ...
2078 # ... ... ...
2079 title = " ID"
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002080 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002081 title += " " * 10 + "ONOS" + str( n + 1 )
2082 main.log.warn( title )
2083 # get all intent keys in the cluster
2084 keys = []
2085 for nodeStr in ONOSIntents:
2086 node = json.loads( nodeStr )
2087 for intent in node:
2088 keys.append( intent.get( 'id' ) )
2089 keys = set( keys )
2090 for key in keys:
2091 row = "%-13s" % key
2092 for nodeStr in ONOSIntents:
2093 node = json.loads( nodeStr )
2094 for intent in node:
2095 if intent.get( 'id' ) == key:
2096 row += "%-15s" % intent.get( 'state' )
2097 main.log.warn( row )
2098 # End table view
2099
2100 utilities.assert_equals(
2101 expect=True,
2102 actual=consistentIntents,
2103 onpass="Intents are consistent across all ONOS nodes",
2104 onfail="ONOS nodes have different views of intents" )
2105 intentStates = []
2106 for node in ONOSIntents: # Iter through ONOS nodes
2107 nodeStates = []
2108 # Iter through intents of a node
2109 try:
2110 for intent in json.loads( node ):
2111 nodeStates.append( intent[ 'state' ] )
2112 except ( ValueError, TypeError ):
2113 main.log.exception( "Error in parsing intents" )
2114 main.log.error( repr( node ) )
2115 intentStates.append( nodeStates )
Jon Hallf37d44d2017-05-24 10:37:30 -07002116 out = [ ( i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002117 main.log.info( dict( out ) )
2118
2119 if intentsResults and not consistentIntents:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002120 for i in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002121 node = str( main.activeNodes[ i ] + 1 )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002122 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002123 main.log.warn( json.dumps(
2124 json.loads( ONOSIntents[ i ] ),
2125 sort_keys=True,
2126 indent=4,
2127 separators=( ',', ': ' ) ) )
2128 elif intentsResults and consistentIntents:
2129 intentCheck = main.TRUE
2130
2131 # NOTE: Store has no durability, so intents are lost across system
2132 # restarts
2133 main.step( "Compare current intents with intents before the failure" )
2134 # NOTE: this requires case 5 to pass for intentState to be set.
2135 # maybe we should stop the test if that fails?
2136 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002137 try:
2138 intentState
2139 except NameError:
2140 main.log.warn( "No previous intent state was saved" )
2141 else:
2142 if intentState and intentState == ONOSIntents[ 0 ]:
2143 sameIntents = main.TRUE
2144 main.log.info( "Intents are consistent with before failure" )
2145 # TODO: possibly the states have changed? we may need to figure out
2146 # what the acceptable states are
2147 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2148 sameIntents = main.TRUE
2149 try:
2150 before = json.loads( intentState )
2151 after = json.loads( ONOSIntents[ 0 ] )
2152 for intent in before:
2153 if intent not in after:
2154 sameIntents = main.FALSE
2155 main.log.debug( "Intent is not currently in ONOS " +
2156 "(at least in the same form):" )
2157 main.log.debug( json.dumps( intent ) )
2158 except ( ValueError, TypeError ):
2159 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002160 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002161 main.log.debug( repr( intentState ) )
2162 if sameIntents == main.FALSE:
2163 try:
2164 main.log.debug( "ONOS intents before: " )
2165 main.log.debug( json.dumps( json.loads( intentState ),
2166 sort_keys=True, indent=4,
2167 separators=( ',', ': ' ) ) )
2168 main.log.debug( "Current ONOS intents: " )
2169 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2170 sort_keys=True, indent=4,
2171 separators=( ',', ': ' ) ) )
2172 except ( ValueError, TypeError ):
2173 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002174 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002175 main.log.debug( repr( intentState ) )
2176 utilities.assert_equals(
2177 expect=main.TRUE,
2178 actual=sameIntents,
2179 onpass="Intents are consistent with before failure",
2180 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002181 intentCheck = intentCheck and sameIntents
2182
2183 main.step( "Get the OF Table entries and compare to before " +
2184 "component failure" )
2185 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002186 for i in range( 28 ):
2187 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002188 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hallf37d44d2017-05-24 10:37:30 -07002189 curSwitch = main.Mininet1.flowTableComp( flows[ i ], tmpFlows )
Jon Hall41d39f12016-04-11 22:54:35 -07002190 FlowTables = FlowTables and curSwitch
2191 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002192 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002193 utilities.assert_equals(
2194 expect=main.TRUE,
2195 actual=FlowTables,
2196 onpass="No changes were found in the flow tables",
2197 onfail="Changes were found in the flow tables" )
2198
2199 main.Mininet2.pingLongKill()
Jon Hallf37d44d2017-05-24 10:37:30 -07002200 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002201 main.step( "Check the continuous pings to ensure that no packets " +
2202 "were dropped during component failure" )
2203 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2204 main.params[ 'TESTONIP' ] )
2205 LossInPings = main.FALSE
2206 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2207 for i in range( 8, 18 ):
2208 main.log.info(
2209 "Checking for a loss in pings along flow from s" +
2210 str( i ) )
2211 LossInPings = main.Mininet2.checkForLoss(
2212 "/tmp/ping.h" +
2213 str( i ) ) or LossInPings
2214 if LossInPings == main.TRUE:
2215 main.log.info( "Loss in ping detected" )
2216 elif LossInPings == main.ERROR:
2217 main.log.info( "There are multiple mininet process running" )
2218 elif LossInPings == main.FALSE:
2219 main.log.info( "No Loss in the pings" )
2220 main.log.info( "No loss of dataplane connectivity" )
2221 utilities.assert_equals(
2222 expect=main.FALSE,
2223 actual=LossInPings,
2224 onpass="No Loss of connectivity",
2225 onfail="Loss of dataplane connectivity detected" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002226 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002227 main.step( "Leadership Election is still functional" )
2228 # Test of LeadershipElection
2229 leaderList = []
Jon Hall5cf14d52015-07-16 12:15:19 -07002230
Jon Hall3b489db2015-10-05 14:38:37 -07002231 restarted = []
2232 for i in main.kill:
Jon Hallf37d44d2017-05-24 10:37:30 -07002233 restarted.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -07002234 leaderResult = main.TRUE
Jon Hall3b489db2015-10-05 14:38:37 -07002235
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002236 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002237 cli = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002238 leaderN = cli.electionTestLeader()
2239 leaderList.append( leaderN )
2240 if leaderN == main.FALSE:
2241 # error in response
2242 main.log.error( "Something is wrong with " +
2243 "electionTestLeader function, check the" +
2244 " error logs" )
2245 leaderResult = main.FALSE
2246 elif leaderN is None:
2247 main.log.error( cli.name +
2248 " shows no leader for the election-app was" +
2249 " elected after the old one died" )
2250 leaderResult = main.FALSE
2251 elif leaderN in restarted:
2252 main.log.error( cli.name + " shows " + str( leaderN ) +
2253 " as leader for the election-app, but it " +
2254 "was restarted" )
2255 leaderResult = main.FALSE
2256 if len( set( leaderList ) ) != 1:
2257 leaderResult = main.FALSE
2258 main.log.error(
2259 "Inconsistent view of leader for the election test app" )
2260 # TODO: print the list
2261 utilities.assert_equals(
2262 expect=main.TRUE,
2263 actual=leaderResult,
2264 onpass="Leadership election passed",
2265 onfail="Something went wrong with Leadership election" )
2266
2267 def CASE8( self, main ):
2268 """
2269 Compare topo
2270 """
2271 import json
2272 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002273 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002274 assert main, "main not defined"
2275 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002276 assert main.CLIs, "main.CLIs not defined"
2277 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002278
2279 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002280 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002281 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002282 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002283 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002284 elapsed = 0
2285 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002286 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002287 startTime = time.time()
2288 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002289 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002290 devicesResults = main.TRUE
2291 linksResults = main.TRUE
2292 hostsResults = main.TRUE
2293 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002294 count += 1
2295 cliStart = time.time()
2296 devices = []
2297 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002298 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002299 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002300 name="devices-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002301 args=[ main.CLIs[ i ].devices, [ None ] ],
2302 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002303 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002304 threads.append( t )
2305 t.start()
2306
2307 for t in threads:
2308 t.join()
2309 devices.append( t.result )
2310 hosts = []
2311 ipResult = main.TRUE
2312 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002313 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002314 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002315 name="hosts-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002316 args=[ main.CLIs[ i ].hosts, [ None ] ],
2317 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Halld8f6de82015-12-17 17:04:34 -08002318 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002319 threads.append( t )
2320 t.start()
2321
2322 for t in threads:
2323 t.join()
2324 try:
2325 hosts.append( json.loads( t.result ) )
2326 except ( ValueError, TypeError ):
2327 main.log.exception( "Error parsing hosts results" )
2328 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002329 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002330 for controller in range( 0, len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002331 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002332 if hosts[ controller ]:
2333 for host in hosts[ controller ]:
2334 if host is None or host.get( 'ipAddresses', [] ) == []:
2335 main.log.error(
2336 "Error with host ipAddresses on controller" +
2337 controllerStr + ": " + str( host ) )
2338 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002339 ports = []
2340 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002341 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002342 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002343 name="ports-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002344 args=[ main.CLIs[ i ].ports, [ None ] ],
2345 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002346 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002347 threads.append( t )
2348 t.start()
2349
2350 for t in threads:
2351 t.join()
2352 ports.append( t.result )
2353 links = []
2354 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002355 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002356 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002357 name="links-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002358 args=[ main.CLIs[ i ].links, [ None ] ],
2359 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002360 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002361 threads.append( t )
2362 t.start()
2363
2364 for t in threads:
2365 t.join()
2366 links.append( t.result )
2367 clusters = []
2368 threads = []
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002369 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002370 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002371 name="clusters-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002372 args=[ main.CLIs[ i ].clusters, [ None ] ],
2373 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002374 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002375 threads.append( t )
2376 t.start()
2377
2378 for t in threads:
2379 t.join()
2380 clusters.append( t.result )
2381
2382 elapsed = time.time() - startTime
2383 cliTime = time.time() - cliStart
2384 print "Elapsed time: " + str( elapsed )
2385 print "CLI time: " + str( cliTime )
2386
Jon Hall6e709752016-02-01 13:38:46 -08002387 if all( e is None for e in devices ) and\
2388 all( e is None for e in hosts ) and\
2389 all( e is None for e in ports ) and\
2390 all( e is None for e in links ) and\
2391 all( e is None for e in clusters ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002392 topoFailMsg = "Could not get topology from ONOS"
2393 main.log.error( topoFailMsg )
2394 continue # Try again, No use trying to compare
Jon Hall6e709752016-02-01 13:38:46 -08002395
Jon Hall5cf14d52015-07-16 12:15:19 -07002396 mnSwitches = main.Mininet1.getSwitches()
2397 mnLinks = main.Mininet1.getLinks()
2398 mnHosts = main.Mininet1.getHosts()
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002399 for controller in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002400 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002401 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07002402 "Error" not in devices[ controller ] and\
2403 "Error" not in ports[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002404
Jon Hallc6793552016-01-19 14:18:37 -08002405 try:
2406 currentDevicesResult = main.Mininet1.compareSwitches(
2407 mnSwitches,
2408 json.loads( devices[ controller ] ),
2409 json.loads( ports[ controller ] ) )
2410 except ( TypeError, ValueError ) as e:
2411 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2412 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002413 else:
2414 currentDevicesResult = main.FALSE
2415 utilities.assert_equals( expect=main.TRUE,
2416 actual=currentDevicesResult,
2417 onpass="ONOS" + controllerStr +
2418 " Switches view is correct",
2419 onfail="ONOS" + controllerStr +
2420 " Switches view is incorrect" )
2421
2422 if links[ controller ] and "Error" not in links[ controller ]:
2423 currentLinksResult = main.Mininet1.compareLinks(
2424 mnSwitches, mnLinks,
2425 json.loads( links[ controller ] ) )
2426 else:
2427 currentLinksResult = main.FALSE
2428 utilities.assert_equals( expect=main.TRUE,
2429 actual=currentLinksResult,
2430 onpass="ONOS" + controllerStr +
2431 " links view is correct",
2432 onfail="ONOS" + controllerStr +
2433 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002434 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002435 currentHostsResult = main.Mininet1.compareHosts(
2436 mnHosts,
2437 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002438 elif hosts[ controller ] == []:
2439 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002440 else:
2441 currentHostsResult = main.FALSE
2442 utilities.assert_equals( expect=main.TRUE,
2443 actual=currentHostsResult,
2444 onpass="ONOS" + controllerStr +
2445 " hosts exist in Mininet",
2446 onfail="ONOS" + controllerStr +
2447 " hosts don't match Mininet" )
2448 # CHECKING HOST ATTACHMENT POINTS
2449 hostAttachment = True
2450 zeroHosts = False
2451 # FIXME: topo-HA/obelisk specific mappings:
2452 # key is mac and value is dpid
2453 mappings = {}
2454 for i in range( 1, 29 ): # hosts 1 through 28
2455 # set up correct variables:
Jon Hallf37d44d2017-05-24 10:37:30 -07002456 macId = "00:" * 5 + hex( i ).split( "0x" )[ 1 ].upper().zfill( 2 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002457 if i == 1:
Jon Hallf37d44d2017-05-24 10:37:30 -07002458 deviceId = "1000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002459 elif i == 2:
Jon Hallf37d44d2017-05-24 10:37:30 -07002460 deviceId = "2000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002461 elif i == 3:
Jon Hallf37d44d2017-05-24 10:37:30 -07002462 deviceId = "3000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002463 elif i == 4:
Jon Hallf37d44d2017-05-24 10:37:30 -07002464 deviceId = "3004".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002465 elif i == 5:
Jon Hallf37d44d2017-05-24 10:37:30 -07002466 deviceId = "5000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002467 elif i == 6:
Jon Hallf37d44d2017-05-24 10:37:30 -07002468 deviceId = "6000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002469 elif i == 7:
Jon Hallf37d44d2017-05-24 10:37:30 -07002470 deviceId = "6007".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002471 elif i >= 8 and i <= 17:
2472 dpid = '3' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07002473 deviceId = dpid.zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002474 elif i >= 18 and i <= 27:
2475 dpid = '6' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07002476 deviceId = dpid.zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002477 elif i == 28:
Jon Hallf37d44d2017-05-24 10:37:30 -07002478 deviceId = "2800".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002479 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002480 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002481 if hosts[ controller ] == []:
2482 main.log.warn( "There are no hosts discovered" )
2483 zeroHosts = True
2484 else:
2485 for host in hosts[ controller ]:
2486 mac = None
2487 location = None
2488 device = None
2489 port = None
2490 try:
2491 mac = host.get( 'mac' )
2492 assert mac, "mac field could not be found for this host object"
2493
Jeremy Ronquillo0e538bc2017-06-13 15:16:09 -07002494 location = host.get( 'locations' )[ 0 ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002495 assert location, "location field could not be found for this host object"
2496
2497 # Trim the protocol identifier off deviceId
Jon Hallf37d44d2017-05-24 10:37:30 -07002498 device = str( location.get( 'elementId' ) ).split( ':' )[ 1 ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002499 assert device, "elementId field could not be found for this host location object"
2500
2501 port = location.get( 'port' )
2502 assert port, "port field could not be found for this host location object"
2503
2504 # Now check if this matches where they should be
2505 if mac and device and port:
2506 if str( port ) != "1":
2507 main.log.error( "The attachment port is incorrect for " +
2508 "host " + str( mac ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07002509 ". Expected: 1 Actual: " + str( port ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002510 hostAttachment = False
2511 if device != mappings[ str( mac ) ]:
2512 main.log.error( "The attachment device is incorrect for " +
2513 "host " + str( mac ) +
2514 ". Expected: " + mappings[ str( mac ) ] +
2515 " Actual: " + device )
2516 hostAttachment = False
2517 else:
2518 hostAttachment = False
2519 except AssertionError:
2520 main.log.exception( "Json object not as expected" )
2521 main.log.error( repr( host ) )
2522 hostAttachment = False
2523 else:
2524 main.log.error( "No hosts json output or \"Error\"" +
2525 " in output. hosts = " +
2526 repr( hosts[ controller ] ) )
2527 if zeroHosts is False:
2528 hostAttachment = True
2529
2530 # END CHECKING HOST ATTACHMENT POINTS
2531 devicesResults = devicesResults and currentDevicesResult
2532 linksResults = linksResults and currentLinksResult
2533 hostsResults = hostsResults and currentHostsResult
2534 hostAttachmentResults = hostAttachmentResults and\
2535 hostAttachment
Jon Halla440e872016-03-31 15:15:50 -07002536 topoResult = ( devicesResults and linksResults
2537 and hostsResults and ipResult and
2538 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002539 utilities.assert_equals( expect=True,
2540 actual=topoResult,
2541 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002542 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002543 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002544
2545 # Compare json objects for hosts and dataplane clusters
2546
2547 # hosts
2548 main.step( "Hosts view is consistent across all ONOS nodes" )
2549 consistentHostsResult = main.TRUE
2550 for controller in range( len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002551 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002552 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002553 if hosts[ controller ] == hosts[ 0 ]:
2554 continue
2555 else: # hosts not consistent
2556 main.log.error( "hosts from ONOS" + controllerStr +
2557 " is inconsistent with ONOS1" )
2558 main.log.warn( repr( hosts[ controller ] ) )
2559 consistentHostsResult = main.FALSE
2560
2561 else:
2562 main.log.error( "Error in getting ONOS hosts from ONOS" +
2563 controllerStr )
2564 consistentHostsResult = main.FALSE
2565 main.log.warn( "ONOS" + controllerStr +
2566 " hosts response: " +
2567 repr( hosts[ controller ] ) )
2568 utilities.assert_equals(
2569 expect=main.TRUE,
2570 actual=consistentHostsResult,
2571 onpass="Hosts view is consistent across all ONOS nodes",
2572 onfail="ONOS nodes have different views of hosts" )
2573
2574 main.step( "Hosts information is correct" )
2575 hostsResults = hostsResults and ipResult
2576 utilities.assert_equals(
2577 expect=main.TRUE,
2578 actual=hostsResults,
2579 onpass="Host information is correct",
2580 onfail="Host information is incorrect" )
2581
2582 main.step( "Host attachment points to the network" )
2583 utilities.assert_equals(
2584 expect=True,
2585 actual=hostAttachmentResults,
2586 onpass="Hosts are correctly attached to the network",
2587 onfail="ONOS did not correctly attach hosts to the network" )
2588
2589 # Strongly connected clusters of devices
2590 main.step( "Clusters view is consistent across all ONOS nodes" )
2591 consistentClustersResult = main.TRUE
2592 for controller in range( len( clusters ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002593 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002594 if "Error" not in clusters[ controller ]:
2595 if clusters[ controller ] == clusters[ 0 ]:
2596 continue
2597 else: # clusters not consistent
2598 main.log.error( "clusters from ONOS" +
2599 controllerStr +
2600 " is inconsistent with ONOS1" )
2601 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002602 else:
2603 main.log.error( "Error in getting dataplane clusters " +
2604 "from ONOS" + controllerStr )
2605 consistentClustersResult = main.FALSE
2606 main.log.warn( "ONOS" + controllerStr +
2607 " clusters response: " +
2608 repr( clusters[ controller ] ) )
2609 utilities.assert_equals(
2610 expect=main.TRUE,
2611 actual=consistentClustersResult,
2612 onpass="Clusters view is consistent across all ONOS nodes",
2613 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002614 if not consistentClustersResult:
2615 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002616
2617 main.step( "There is only one SCC" )
2618 # there should always only be one cluster
2619 try:
2620 numClusters = len( json.loads( clusters[ 0 ] ) )
2621 except ( ValueError, TypeError ):
2622 main.log.exception( "Error parsing clusters[0]: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07002623 repr( clusters[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002624 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002625 clusterResults = main.FALSE
2626 if numClusters == 1:
2627 clusterResults = main.TRUE
2628 utilities.assert_equals(
2629 expect=1,
2630 actual=numClusters,
2631 onpass="ONOS shows 1 SCC",
2632 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2633
2634 topoResult = ( devicesResults and linksResults
2635 and hostsResults and consistentHostsResult
2636 and consistentClustersResult and clusterResults
2637 and ipResult and hostAttachmentResults )
2638
2639 topoResult = topoResult and int( count <= 2 )
2640 note = "note it takes about " + str( int( cliTime ) ) + \
2641 " seconds for the test to make all the cli calls to fetch " +\
2642 "the topology from each ONOS instance"
2643 main.log.info(
2644 "Very crass estimate for topology discovery/convergence( " +
2645 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2646 str( count ) + " tries" )
2647
2648 main.step( "Device information is correct" )
2649 utilities.assert_equals(
2650 expect=main.TRUE,
2651 actual=devicesResults,
2652 onpass="Device information is correct",
2653 onfail="Device information is incorrect" )
2654
2655 main.step( "Links are correct" )
2656 utilities.assert_equals(
2657 expect=main.TRUE,
2658 actual=linksResults,
2659 onpass="Link are correct",
2660 onfail="Links are incorrect" )
2661
Jon Halla440e872016-03-31 15:15:50 -07002662 main.step( "Hosts are correct" )
2663 utilities.assert_equals(
2664 expect=main.TRUE,
2665 actual=hostsResults,
2666 onpass="Hosts are correct",
2667 onfail="Hosts are incorrect" )
2668
Jon Hall5cf14d52015-07-16 12:15:19 -07002669 # FIXME: move this to an ONOS state case
2670 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002671 nodeResults = utilities.retry( main.HA.nodesCheck,
2672 False,
Jon Hallf37d44d2017-05-24 10:37:30 -07002673 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -07002674 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002675
Jon Hall41d39f12016-04-11 22:54:35 -07002676 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002677 onpass="Nodes check successful",
2678 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002679 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002680 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002681 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hallf37d44d2017-05-24 10:37:30 -07002682 main.CLIs[ i ].name,
2683 main.CLIs[ i ].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002684
Jon Halld2871c22016-07-26 11:01:14 -07002685 if not topoResult:
2686 main.cleanup()
2687 main.exit()
2688
Jon Hall5cf14d52015-07-16 12:15:19 -07002689 def CASE9( self, main ):
2690 """
2691 Link s3-s28 down
2692 """
2693 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002694 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002695 assert main, "main not defined"
2696 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002697 assert main.CLIs, "main.CLIs not defined"
2698 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002699 # NOTE: You should probably run a topology check after this
2700
2701 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2702
2703 description = "Turn off a link to ensure that Link Discovery " +\
2704 "is working properly"
2705 main.case( description )
2706
2707 main.step( "Kill Link between s3 and s28" )
2708 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2709 main.log.info( "Waiting " + str( linkSleep ) +
2710 " seconds for link down to be discovered" )
2711 time.sleep( linkSleep )
2712 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2713 onpass="Link down successful",
2714 onfail="Failed to bring link down" )
2715 # TODO do some sort of check here
2716
2717 def CASE10( self, main ):
2718 """
2719 Link s3-s28 up
2720 """
2721 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002722 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002723 assert main, "main not defined"
2724 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002725 assert main.CLIs, "main.CLIs not defined"
2726 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002727 # NOTE: You should probably run a topology check after this
2728
2729 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2730
2731 description = "Restore a link to ensure that Link Discovery is " + \
2732 "working properly"
2733 main.case( description )
2734
2735 main.step( "Bring link between s3 and s28 back up" )
2736 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2737 main.log.info( "Waiting " + str( linkSleep ) +
2738 " seconds for link up to be discovered" )
2739 time.sleep( linkSleep )
2740 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2741 onpass="Link up successful",
2742 onfail="Failed to bring link up" )
2743 # TODO do some sort of check here
2744
2745 def CASE11( self, main ):
2746 """
2747 Switch Down
2748 """
2749 # NOTE: You should probably run a topology check after this
2750 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002751 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002752 assert main, "main not defined"
2753 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002754 assert main.CLIs, "main.CLIs not defined"
2755 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002756
2757 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2758
2759 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallf37d44d2017-05-24 10:37:30 -07002760 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002761 main.case( description )
2762 switch = main.params[ 'kill' ][ 'switch' ]
2763 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2764
2765 # TODO: Make this switch parameterizable
2766 main.step( "Kill " + switch )
2767 main.log.info( "Deleting " + switch )
2768 main.Mininet1.delSwitch( switch )
2769 main.log.info( "Waiting " + str( switchSleep ) +
2770 " seconds for switch down to be discovered" )
2771 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002772 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002773 # Peek at the deleted switch
2774 main.log.warn( str( device ) )
2775 result = main.FALSE
2776 if device and device[ 'available' ] is False:
2777 result = main.TRUE
2778 utilities.assert_equals( expect=main.TRUE, actual=result,
2779 onpass="Kill switch successful",
2780 onfail="Failed to kill switch?" )
2781
2782 def CASE12( self, main ):
2783 """
2784 Switch Up
2785 """
2786 # NOTE: You should probably run a topology check after this
2787 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002788 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002789 assert main, "main not defined"
2790 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002791 assert main.CLIs, "main.CLIs not defined"
2792 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002793 assert ONOS1Port, "ONOS1Port not defined"
2794 assert ONOS2Port, "ONOS2Port not defined"
2795 assert ONOS3Port, "ONOS3Port not defined"
2796 assert ONOS4Port, "ONOS4Port not defined"
2797 assert ONOS5Port, "ONOS5Port not defined"
2798 assert ONOS6Port, "ONOS6Port not defined"
2799 assert ONOS7Port, "ONOS7Port not defined"
2800
2801 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2802 switch = main.params[ 'kill' ][ 'switch' ]
2803 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2804 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallf37d44d2017-05-24 10:37:30 -07002805 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002806 description = "Adding a switch to ensure it is discovered correctly"
2807 main.case( description )
2808
2809 main.step( "Add back " + switch )
2810 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2811 for peer in links:
2812 main.Mininet1.addLink( switch, peer )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002813 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002814 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2815 main.log.info( "Waiting " + str( switchSleep ) +
2816 " seconds for switch up to be discovered" )
2817 time.sleep( switchSleep )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002818 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002819 # Peek at the deleted switch
2820 main.log.warn( str( device ) )
2821 result = main.FALSE
2822 if device and device[ 'available' ]:
2823 result = main.TRUE
2824 utilities.assert_equals( expect=main.TRUE, actual=result,
2825 onpass="add switch successful",
2826 onfail="Failed to add switch?" )
2827
2828 def CASE13( self, main ):
2829 """
2830 Clean up
2831 """
2832 import os
2833 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002834 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002835 assert main, "main not defined"
2836 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002837 assert main.CLIs, "main.CLIs not defined"
2838 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002839
2840 # printing colors to terminal
2841 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2842 'blue': '\033[94m', 'green': '\033[92m',
2843 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2844 main.case( "Test Cleanup" )
2845 main.step( "Killing tcpdumps" )
2846 main.Mininet2.stopTcpdump()
2847
2848 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002849 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002850 main.step( "Copying MN pcap and ONOS log files to test station" )
2851 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2852 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002853 # NOTE: MN Pcap file is being saved to logdir.
2854 # We scp this file as MN and TestON aren't necessarily the same vm
2855
2856 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002857 # TODO: Load these from params
2858 # NOTE: must end in /
2859 logFolder = "/opt/onos/log/"
2860 logFiles = [ "karaf.log", "karaf.log.1" ]
2861 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002862 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002863 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002864 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002865 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2866 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002867 # std*.log's
2868 # NOTE: must end in /
2869 logFolder = "/opt/onos/var/"
2870 logFiles = [ "stderr.log", "stdout.log" ]
2871 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002872 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002873 for node in main.nodes:
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002874 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002875 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2876 logFolder + f, dstName )
2877 else:
2878 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002879
2880 main.step( "Stopping Mininet" )
2881 mnResult = main.Mininet1.stopNet()
2882 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2883 onpass="Mininet stopped",
2884 onfail="MN cleanup NOT successful" )
2885
2886 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002887 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002888 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2889 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002890
2891 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07002892 timerLog = open( main.logdir + "/Timers.csv", 'w' )
Jon Hall5cf14d52015-07-16 12:15:19 -07002893 # Overwrite with empty line and close
2894 labels = "Gossip Intents, Restart"
2895 data = str( gossipTime ) + ", " + str( main.restartTime )
2896 timerLog.write( labels + "\n" + data )
2897 timerLog.close()
Jon Hallf37d44d2017-05-24 10:37:30 -07002898 except NameError as e:
2899 main.log.exception( e )
Jon Hall5cf14d52015-07-16 12:15:19 -07002900
2901 def CASE14( self, main ):
2902 """
2903 start election app on all onos nodes
2904 """
Jon Halle1a3b752015-07-22 13:02:46 -07002905 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002906 assert main, "main not defined"
2907 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002908 assert main.CLIs, "main.CLIs not defined"
2909 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002910
Jon Hallf37d44d2017-05-24 10:37:30 -07002911 main.case( "Start Leadership Election app" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002912 main.step( "Install leadership election app" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002913 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002914 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002915 utilities.assert_equals(
2916 expect=main.TRUE,
2917 actual=appResult,
2918 onpass="Election app installed",
2919 onfail="Something went wrong with installing Leadership election" )
2920
2921 main.step( "Run for election on each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002922 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002923 main.CLIs[ i ].electionTestRun()
2924 time.sleep( 5 )
2925 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall25463a82016-04-13 14:03:52 -07002926 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002927 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002928 expect=True,
2929 actual=sameResult,
2930 onpass="All nodes see the same leaderboards",
2931 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002932
Jon Hall25463a82016-04-13 14:03:52 -07002933 if sameResult:
2934 leader = leaders[ 0 ][ 0 ]
Jon Hallf37d44d2017-05-24 10:37:30 -07002935 if main.nodes[ main.activeNodes[ 0 ] ].ip_address in leader:
Jon Hall25463a82016-04-13 14:03:52 -07002936 correctLeader = True
2937 else:
2938 correctLeader = False
2939 main.step( "First node was elected leader" )
2940 utilities.assert_equals(
2941 expect=True,
2942 actual=correctLeader,
2943 onpass="Correct leader was elected",
2944 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002945
2946 def CASE15( self, main ):
2947 """
2948 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002949 15.1 Run election on each node
2950 15.2 Check that each node has the same leaders and candidates
2951 15.3 Find current leader and withdraw
2952 15.4 Check that a new node was elected leader
2953 15.5 Check that that new leader was the candidate of old leader
2954 15.6 Run for election on old leader
2955 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2956 15.8 Make sure that the old leader was added to the candidate list
2957
2958 old and new variable prefixes refer to data from before vs after
2959 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002960 """
2961 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002962 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002963 assert main, "main not defined"
2964 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002965 assert main.CLIs, "main.CLIs not defined"
2966 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002967
Jon Hall5cf14d52015-07-16 12:15:19 -07002968 description = "Check that Leadership Election is still functional"
2969 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002970 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002971
Jon Halla440e872016-03-31 15:15:50 -07002972 oldLeaders = [] # list of lists of each nodes' candidates before
2973 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002974 oldLeader = '' # the old leader from oldLeaders, None if not same
2975 newLeader = '' # the new leaders fron newLoeaders, None if not same
2976 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2977 expectNoLeader = False # True when there is only one leader
2978 if main.numCtrls == 1:
2979 expectNoLeader = True
2980
2981 main.step( "Run for election on each node" )
2982 electionResult = main.TRUE
2983
Jon Hallb3ed8ed2015-10-28 16:43:55 -07002984 for i in main.activeNodes: # run test election on each node
Jon Hallf37d44d2017-05-24 10:37:30 -07002985 if main.CLIs[ i ].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002986 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002987 utilities.assert_equals(
2988 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002989 actual=electionResult,
2990 onpass="All nodes successfully ran for leadership",
2991 onfail="At least one node failed to run for leadership" )
2992
acsmars3a72bde2015-09-02 14:16:22 -07002993 if electionResult == main.FALSE:
2994 main.log.error(
2995 "Skipping Test Case because Election Test App isn't loaded" )
2996 main.skipCase()
2997
acsmars71adceb2015-08-31 15:09:26 -07002998 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002999 failMessage = "Nodes have different leaderboards"
Jon Hallf37d44d2017-05-24 10:37:30 -07003000 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07003001 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07003002 if sameResult:
3003 oldLeader = oldLeaders[ 0 ][ 0 ]
3004 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07003005 else:
Jon Halla440e872016-03-31 15:15:50 -07003006 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07003007 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003008 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003009 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07003010 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07003011 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07003012
3013 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07003014 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07003015 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07003016 if oldLeader is None:
3017 main.log.error( "Leadership isn't consistent." )
3018 withdrawResult = main.FALSE
3019 # Get the CLI of the oldLeader
Jon Hallb3ed8ed2015-10-28 16:43:55 -07003020 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07003021 if oldLeader == main.nodes[ i ].ip_address:
3022 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07003023 break
3024 else: # FOR/ELSE statement
3025 main.log.error( "Leader election, could not find current leader" )
3026 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07003027 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07003028 utilities.assert_equals(
3029 expect=main.TRUE,
3030 actual=withdrawResult,
3031 onpass="Node was withdrawn from election",
3032 onfail="Node was not withdrawn from election" )
3033
acsmars71adceb2015-08-31 15:09:26 -07003034 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07003035 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07003036 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07003037 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07003038 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07003039 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07003040 if newLeaders[ 0 ][ 0 ] == 'none':
3041 main.log.error( "No leader was elected on at least 1 node" )
3042 if not expectNoLeader:
3043 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07003044 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07003045
3046 # Check that the new leader is not the older leader, which was withdrawn
3047 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07003048 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08003049 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07003050 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003051 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003052 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003053 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003054 onpass="Leadership election passed",
3055 onfail="Something went wrong with Leadership election" )
3056
Jon Halla440e872016-03-31 15:15:50 -07003057 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08003058 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07003059 correctCandidateResult = main.TRUE
3060 if expectNoLeader:
3061 if newLeader == 'none':
3062 main.log.info( "No leader expected. None found. Pass" )
3063 correctCandidateResult = main.TRUE
3064 else:
3065 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3066 correctCandidateResult = main.FALSE
Jon Hallf37d44d2017-05-24 10:37:30 -07003067 elif len( oldLeaders[ 0 ] ) >= 3:
Jon Halla440e872016-03-31 15:15:50 -07003068 if newLeader == oldLeaders[ 0 ][ 2 ]:
3069 # correct leader was elected
3070 correctCandidateResult = main.TRUE
3071 else:
3072 correctCandidateResult = main.FALSE
3073 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3074 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003075 else:
3076 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003077 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003078 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07003079 utilities.assert_equals(
3080 expect=main.TRUE,
3081 actual=correctCandidateResult,
3082 onpass="Correct Candidate Elected",
3083 onfail="Incorrect Candidate Elected" )
3084
Jon Hall5cf14d52015-07-16 12:15:19 -07003085 main.step( "Run for election on old leader( just so everyone " +
3086 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07003087 if oldLeaderCLI is not None:
3088 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003089 else:
acsmars71adceb2015-08-31 15:09:26 -07003090 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003091 runResult = main.FALSE
3092 utilities.assert_equals(
3093 expect=main.TRUE,
3094 actual=runResult,
3095 onpass="App re-ran for election",
3096 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003097
acsmars71adceb2015-08-31 15:09:26 -07003098 main.step(
3099 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003100 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003101 # Get new leaders and candidates
3102 reRunLeaders = []
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003103 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003104 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003105
3106 # Check that the re-elected node is last on the candidate List
Jon Hallf37d44d2017-05-24 10:37:30 -07003107 if not reRunLeaders[ 0 ]:
Jon Hall3a7843a2016-04-12 03:01:09 -07003108 positionResult = main.FALSE
3109 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07003110 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader ),
Jon Halla440e872016-03-31 15:15:50 -07003111 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07003112 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003113 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003114 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07003115 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003116 onpass="Old leader successfully re-ran for election",
3117 onfail="Something went wrong with Leadership election after " +
3118 "the old leader re-ran for election" )
3119
3120 def CASE16( self, main ):
3121 """
3122 Install Distributed Primitives app
3123 """
3124 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003125 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003126 assert main, "main not defined"
3127 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003128 assert main.CLIs, "main.CLIs not defined"
3129 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003130
3131 # Variables for the distributed primitives tests
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003132 main.pCounterName = "TestON-Partitions"
3133 main.pCounterValue = 0
Jon Hallf37d44d2017-05-24 10:37:30 -07003134 main.onosSet = set( [] )
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003135 main.onosSetName = "TestON-set"
Jon Hall5cf14d52015-07-16 12:15:19 -07003136
3137 description = "Install Primitives app"
3138 main.case( description )
3139 main.step( "Install Primitives app" )
3140 appName = "org.onosproject.distributedprimitives"
Jon Hallf37d44d2017-05-24 10:37:30 -07003141 node = main.activeNodes[ 0 ]
3142 appResults = main.CLIs[ node ].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003143 utilities.assert_equals( expect=main.TRUE,
3144 actual=appResults,
3145 onpass="Primitives app activated",
3146 onfail="Primitives app not activated" )
3147 time.sleep( 5 ) # To allow all nodes to activate
3148
3149 def CASE17( self, main ):
3150 """
3151 Check for basic functionality with distributed primitives
3152 """
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003153 main.HA.CASE17( main )