blob: 73ec921b1b313d54e959669e1127bc36f290238e [file] [log] [blame]
Jon Hall85794ff2015-07-08 14:12:30 -07001"""
2Description: This test is to determine if a single
3 instance ONOS 'cluster' can handle a restart
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
Jon Hall85794ff2015-07-08 14:12:30 -070025class HAsingleInstanceRestart:
26
27 def __init__( self ):
28 self.default = ''
29
30 def CASE1( self, main ):
31 """
32 CASE1 is to compile ONOS and push it to the test machines
33
34 Startup sequence:
35 cell <name>
36 onos-verify-cell
37 NOTE: temporary - onos-remove-raft-logs
38 onos-uninstall
39 start mininet
40 git pull
41 mvn clean install
42 onos-package
43 onos-install -f
44 onos-wait-for-start
45 start cli sessions
46 start tcpdump
47 """
Jon Halle1a3b752015-07-22 13:02:46 -070048 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080049 import time
Jon Halla440e872016-03-31 15:15:50 -070050 import json
Jon Hall85794ff2015-07-08 14:12:30 -070051 main.log.info( "ONOS Single node cluster restart " +
52 "HA test - initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall85794ff2015-07-08 14:12:30 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
Jon Hall85794ff2015-07-08 14:12:30 -070057
58 # load some variables from the params file
59 PULLCODE = False
60 if main.params[ 'Git' ] == 'True':
61 PULLCODE = True
62 gitBranch = main.params[ 'branch' ]
63 cellName = main.params[ 'ENV' ][ 'cellName' ]
64
Jon Halle1a3b752015-07-22 13:02:46 -070065 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070066 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070067 if main.ONOSbench.maxNodes < main.numCtrls:
68 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall85794ff2015-07-08 14:12:30 -070069
Jon Halle1a3b752015-07-22 13:02:46 -070070 try:
Jon Hall53c5e662016-04-13 16:06:56 -070071 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070072 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070073 except Exception as e:
74 main.log.exception( e )
75 main.cleanup()
76 main.exit()
77
78 main.CLIs = []
79 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -070080 ipList = []
81 for i in range( 1, int( main.ONOSbench.maxNodes ) + 1 ):
82 try:
Jon Halle1a3b752015-07-22 13:02:46 -070083 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
84 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
85 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -070086 except AttributeError:
87 break
Jon Hall85794ff2015-07-08 14:12:30 -070088
Jon Hall5cf14d52015-07-16 12:15:19 -070089 main.step( "Create cell file" )
90 cellAppString = main.params[ 'ENV' ][ 'appString' ]
91 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
92 main.Mininet1.ip_address,
Devin Limdc78e202017-06-09 18:30:07 -070093 cellAppString, ipList, main.ONOScli1.karafUser )
Jon Hall85794ff2015-07-08 14:12:30 -070094 main.step( "Applying cell variable to environment" )
95 cellResult = main.ONOSbench.setCell( cellName )
96 verifyResult = main.ONOSbench.verifyCell()
97
98 # FIXME:this is short term fix
99 main.log.info( "Removing raft logs" )
100 main.ONOSbench.onosRemoveRaftLogs()
101
102 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700103 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700104 main.ONOSbench.onosUninstall( node.ip_address )
105
106 # Make sure ONOS is DEAD
107 main.log.info( "Killing any ONOS processes" )
108 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700109 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700110 killed = main.ONOSbench.onosKill( node.ip_address )
111 killResults = killResults and killed
112
Jon Hall85794ff2015-07-08 14:12:30 -0700113 gitPullResult = main.TRUE
114
115 main.step( "Starting Mininet" )
116 # scp topo file to mininet
117 # TODO: move to params?
118 topoName = "obelisk.py"
119 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700120 main.ONOSbench.scp( main.Mininet1,
121 filePath + topoName,
122 main.Mininet1.home,
123 direction="to" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700124 mnResult = main.Mininet1.startNet()
Jon Hall85794ff2015-07-08 14:12:30 -0700125 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
126 onpass="Mininet Started",
127 onfail="Error starting Mininet" )
128
129 main.step( "Git checkout and pull " + gitBranch )
130 if PULLCODE:
131 main.ONOSbench.gitCheckout( gitBranch )
132 gitPullResult = main.ONOSbench.gitPull()
133 # values of 1 or 3 are good
134 utilities.assert_lesser( expect=0, actual=gitPullResult,
135 onpass="Git pull successful",
136 onfail="Git pull failed" )
137 main.ONOSbench.getVersion( report=True )
138
Jon Hall85794ff2015-07-08 14:12:30 -0700139 # GRAPHS
140 # NOTE: important params here:
141 # job = name of Jenkins job
142 # Plot Name = Plot-HA, only can be used if multiple plots
143 # index = The number of the graph under plot name
Jon Hall5cf14d52015-07-16 12:15:19 -0700144 job = "HAsingleInstanceRestart"
Jon Hall85794ff2015-07-08 14:12:30 -0700145 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700146 index = "2"
Jon Hall85794ff2015-07-08 14:12:30 -0700147 graphs = '<ac:structured-macro ac:name="html">\n'
148 graphs += '<ac:plain-text-body><![CDATA[\n'
149 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800150 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall85794ff2015-07-08 14:12:30 -0700151 '&width=500&height=300"' +\
152 'noborder="0" width="500" height="300" scrolling="yes" ' +\
153 'seamless="seamless"></iframe>\n'
154 graphs += ']]></ac:plain-text-body>\n'
155 graphs += '</ac:structured-macro>\n'
Jon Hallf37d44d2017-05-24 10:37:30 -0700156 main.log.wiki( graphs )
Jon Hall85794ff2015-07-08 14:12:30 -0700157
Jon Halle1a3b752015-07-22 13:02:46 -0700158 main.CLIs = []
159 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700160 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700161 for i in range( 1, main.numCtrls + 1 ):
162 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
163 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
164 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700165
166 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, "SingleHA",
167 main.Mininet1.ip_address,
Devin Limdc78e202017-06-09 18:30:07 -0700168 cellAppString, ipList[ 0 ], main.ONOScli1.karafUser )
Jon Hall85794ff2015-07-08 14:12:30 -0700169 cellResult = main.ONOSbench.setCell( "SingleHA" )
170 verifyResult = main.ONOSbench.verifyCell()
171 main.step( "Creating ONOS package" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700172 packageResult = main.ONOSbench.buckBuild()
Jon Hall85794ff2015-07-08 14:12:30 -0700173 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
174 onpass="ONOS package successful",
175 onfail="ONOS package failed" )
176
177 main.step( "Installing ONOS package" )
Jon Halla440e872016-03-31 15:15:50 -0700178 onosInstallResult = main.TRUE
179 for node in main.nodes:
180 tmpResult = main.ONOSbench.onosInstall( options="-f",
181 node=node.ip_address )
182 onosInstallResult = onosInstallResult and tmpResult
Jon Hall85794ff2015-07-08 14:12:30 -0700183 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
184 onpass="ONOS install successful",
185 onfail="ONOS install failed" )
186
You Wangf5de25b2017-01-06 15:13:01 -0800187 main.step( "Set up ONOS secure SSH" )
188 secureSshResult = main.TRUE
189 for node in main.nodes:
190 secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=node.ip_address )
191 utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
192 onpass="Test step PASS",
193 onfail="Test step FAIL" )
194
Jon Hall85794ff2015-07-08 14:12:30 -0700195 main.step( "Checking if ONOS is up yet" )
196 for i in range( 2 ):
Jon Halla440e872016-03-31 15:15:50 -0700197 onosIsupResult = main.TRUE
198 for node in main.nodes:
199 started = main.ONOSbench.isup( node.ip_address )
200 if not started:
201 main.log.error( node.name + " hasn't started" )
202 onosIsupResult = onosIsupResult and started
203 if onosIsupResult == main.TRUE:
Jon Hall85794ff2015-07-08 14:12:30 -0700204 break
Jon Halla440e872016-03-31 15:15:50 -0700205 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
Jon Hall85794ff2015-07-08 14:12:30 -0700206 onpass="ONOS startup successful",
207 onfail="ONOS startup failed" )
208
Jon Hall6509dbf2016-06-21 17:01:17 -0700209 main.step( "Starting ONOS CLI sessions" )
Jon Halla440e872016-03-31 15:15:50 -0700210 cliResults = main.TRUE
211 threads = []
212 for i in range( main.numCtrls ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700213 t = main.Thread( target=main.CLIs[ i ].startOnosCli,
Jon Halla440e872016-03-31 15:15:50 -0700214 name="startOnosCli-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -0700215 args=[ main.nodes[ i ].ip_address ] )
Jon Halla440e872016-03-31 15:15:50 -0700216 threads.append( t )
217 t.start()
218
219 for t in threads:
220 t.join()
221 cliResults = cliResults and t.result
Jon Hall85794ff2015-07-08 14:12:30 -0700222 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
223 onpass="ONOS cli startup successful",
224 onfail="ONOS cli startup failed" )
225
Jon Halla440e872016-03-31 15:15:50 -0700226 # Create a list of active nodes for use when some nodes are stopped
227 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
228
Jon Hall85794ff2015-07-08 14:12:30 -0700229 if main.params[ 'tcpdump' ].lower() == "true":
230 main.step( "Start Packet Capture MN" )
231 main.Mininet2.startTcpdump(
232 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
233 + "-MN.pcap",
234 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
235 port=main.params[ 'MNtcpdump' ][ 'port' ] )
236
Jon Halla440e872016-03-31 15:15:50 -0700237 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700238 nodeResults = utilities.retry( main.HA.nodesCheck,
239 False,
Jon Hallf37d44d2017-05-24 10:37:30 -0700240 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -0700241 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700242
Jon Hall41d39f12016-04-11 22:54:35 -0700243 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700244 onpass="Nodes check successful",
245 onfail="Nodes check NOT successful" )
246
247 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700248 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700249 cli = main.CLIs[ i ]
Jon Halla440e872016-03-31 15:15:50 -0700250 main.log.debug( "{} components not ACTIVE: \n{}".format(
251 cli.name,
252 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700253 main.log.error( "Failed to start ONOS, stopping test" )
254 main.cleanup()
255 main.exit()
256
Jon Hall172b7ba2016-04-07 18:12:20 -0700257 main.step( "Activate apps defined in the params file" )
258 # get data from the params
259 apps = main.params.get( 'apps' )
260 if apps:
Jon Hallf37d44d2017-05-24 10:37:30 -0700261 apps = apps.split( ',' )
Jon Hall172b7ba2016-04-07 18:12:20 -0700262 main.log.warn( apps )
263 activateResult = True
264 for app in apps:
265 main.CLIs[ 0 ].app( app, "Activate" )
266 # TODO: check this worked
267 time.sleep( 10 ) # wait for apps to activate
268 for app in apps:
269 state = main.CLIs[ 0 ].appStatus( app )
270 if state == "ACTIVE":
Jon Hall937bc812017-01-31 16:44:10 -0800271 activateResult = activateResult and True
Jon Hall172b7ba2016-04-07 18:12:20 -0700272 else:
273 main.log.error( "{} is in {} state".format( app, state ) )
Jon Hall937bc812017-01-31 16:44:10 -0800274 activateResult = False
Jon Hall172b7ba2016-04-07 18:12:20 -0700275 utilities.assert_equals( expect=True,
276 actual=activateResult,
277 onpass="Successfully activated apps",
278 onfail="Failed to activate apps" )
279 else:
280 main.log.warn( "No apps were specified to be loaded after startup" )
281
282 main.step( "Set ONOS configurations" )
283 config = main.params.get( 'ONOS_Configuration' )
284 if config:
285 main.log.debug( config )
286 checkResult = main.TRUE
287 for component in config:
Jon Hallf37d44d2017-05-24 10:37:30 -0700288 for setting in config[ component ]:
289 value = config[ component ][ setting ]
Jon Hall172b7ba2016-04-07 18:12:20 -0700290 check = main.CLIs[ 0 ].setCfg( component, setting, value )
291 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
292 checkResult = check and checkResult
293 utilities.assert_equals( expect=main.TRUE,
294 actual=checkResult,
295 onpass="Successfully set config",
296 onfail="Failed to set config" )
297 else:
298 main.log.warn( "No configurations were specified to be changed after startup" )
299
Jon Hall9d2dcad2016-04-08 10:15:20 -0700300 main.step( "App Ids check" )
301 appCheck = main.TRUE
302 threads = []
303 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700304 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall9d2dcad2016-04-08 10:15:20 -0700305 name="appToIDCheck-" + str( i ),
306 args=[] )
307 threads.append( t )
308 t.start()
309
310 for t in threads:
311 t.join()
312 appCheck = appCheck and t.result
313 if appCheck != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700314 node = main.activeNodes[ 0 ]
315 main.log.warn( main.CLIs[ node ].apps() )
316 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700317 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
318 onpass="App Ids seem to be correct",
319 onfail="Something is wrong with app Ids" )
320
Jon Hall85794ff2015-07-08 14:12:30 -0700321 def CASE2( self, main ):
322 """
323 Assign devices to controllers
324 """
325 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700326 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700327 assert main, "main not defined"
328 assert utilities.assert_equals, "utilities.assert_equals not defined"
329
330 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700331 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700332 "and check that an ONOS node becomes the " +\
333 "master of the device."
334 main.step( "Assign switches to controllers" )
335
336 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700337 for i in range( main.numCtrls ):
338 ipList.append( main.nodes[ i ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700339 swList = []
340 for i in range( 1, 29 ):
341 swList.append( "s" + str( i ) )
342 main.Mininet1.assignSwController( sw=swList, ip=ipList )
343
344 mastershipCheck = main.TRUE
345 for i in range( 1, 29 ):
346 response = main.Mininet1.getSwController( "s" + str( i ) )
347 try:
348 main.log.info( str( response ) )
349 except Exception:
350 main.log.info( repr( response ) )
Jon Halla440e872016-03-31 15:15:50 -0700351 for node in main.nodes:
352 if re.search( "tcp:" + node.ip_address, response ):
353 mastershipCheck = mastershipCheck and main.TRUE
354 else:
355 main.log.error( "Error, node " + node.ip_address + " is " +
356 "not in the list of controllers s" +
357 str( i ) + " is connecting to." )
358 mastershipCheck = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700359 utilities.assert_equals(
360 expect=main.TRUE,
361 actual=mastershipCheck,
362 onpass="Switch mastership assigned correctly",
363 onfail="Switches not assigned correctly to controllers" )
364
365 def CASE21( self, main ):
366 """
367 Assign mastership to controllers
368 """
Jon Halle1a3b752015-07-22 13:02:46 -0700369 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700370 assert main, "main not defined"
371 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700372 assert main.CLIs, "main.CLIs not defined"
373 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700374
375 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700376 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700377 "device. Then manually assign" +\
378 " mastership to specific ONOS nodes using" +\
379 " 'device-role'"
380 main.step( "Assign mastership of switches to specific controllers" )
Jon Halla440e872016-03-31 15:15:50 -0700381 # Manually assign mastership to the controller we want
Jon Hall85794ff2015-07-08 14:12:30 -0700382 roleCall = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700383
Jon Hallf37d44d2017-05-24 10:37:30 -0700384 ipList = []
Jon Halla440e872016-03-31 15:15:50 -0700385 deviceList = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700386 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall85794ff2015-07-08 14:12:30 -0700387 try:
Jon Halla440e872016-03-31 15:15:50 -0700388 # Assign mastership to specific controllers. This assignment was
389 # determined for a 7 node cluser, but will work with any sized
390 # cluster
Jon Hall85794ff2015-07-08 14:12:30 -0700391 for i in range( 1, 29 ): # switches 1 through 28
Jon Hall85794ff2015-07-08 14:12:30 -0700392 # set up correct variables:
393 if i == 1:
Jon Halla440e872016-03-31 15:15:50 -0700394 c = 0
395 ip = main.nodes[ c ].ip_address # ONOS1
396 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700397 elif i == 2:
Jon Halla440e872016-03-31 15:15:50 -0700398 c = 1 % main.numCtrls
399 ip = main.nodes[ c ].ip_address # ONOS2
400 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700401 elif i == 3:
Jon Halla440e872016-03-31 15:15:50 -0700402 c = 1 % main.numCtrls
403 ip = main.nodes[ c ].ip_address # ONOS2
404 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700405 elif i == 4:
Jon Halla440e872016-03-31 15:15:50 -0700406 c = 3 % main.numCtrls
407 ip = main.nodes[ c ].ip_address # ONOS4
408 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700409 elif i == 5:
Jon Halla440e872016-03-31 15:15:50 -0700410 c = 2 % main.numCtrls
411 ip = main.nodes[ c ].ip_address # ONOS3
412 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700413 elif i == 6:
Jon Halla440e872016-03-31 15:15:50 -0700414 c = 2 % main.numCtrls
415 ip = main.nodes[ c ].ip_address # ONOS3
416 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700417 elif i == 7:
Jon Halla440e872016-03-31 15:15:50 -0700418 c = 5 % main.numCtrls
419 ip = main.nodes[ c ].ip_address # ONOS6
420 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700421 elif i >= 8 and i <= 17:
Jon Halla440e872016-03-31 15:15:50 -0700422 c = 4 % main.numCtrls
423 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall85794ff2015-07-08 14:12:30 -0700424 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700425 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700426 elif i >= 18 and i <= 27:
Jon Halla440e872016-03-31 15:15:50 -0700427 c = 6 % main.numCtrls
428 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall85794ff2015-07-08 14:12:30 -0700429 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700430 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700431 elif i == 28:
Jon Halla440e872016-03-31 15:15:50 -0700432 c = 0
433 ip = main.nodes[ c ].ip_address # ONOS1
434 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700435 else:
436 main.log.error( "You didn't write an else statement for " +
437 "switch s" + str( i ) )
Jon Halla440e872016-03-31 15:15:50 -0700438 roleCall = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700439 # Assign switch
440 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
441 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700442 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
443 ipList.append( ip )
444 deviceList.append( deviceId )
Jon Hall85794ff2015-07-08 14:12:30 -0700445 except ( AttributeError, AssertionError ):
446 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700447 main.log.info( onosCli.devices() )
Jon Hall85794ff2015-07-08 14:12:30 -0700448 utilities.assert_equals(
449 expect=main.TRUE,
450 actual=roleCall,
451 onpass="Re-assigned switch mastership to designated controller",
452 onfail="Something wrong with deviceRole calls" )
453
454 main.step( "Check mastership was correctly assigned" )
Jon Halla440e872016-03-31 15:15:50 -0700455 roleCheck = main.TRUE
456 # NOTE: This is due to the fact that device mastership change is not
457 # atomic and is actually a multi step process
458 time.sleep( 5 )
459 for i in range( len( ipList ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700460 ip = ipList[ i ]
461 deviceId = deviceList[ i ]
Jon Halla440e872016-03-31 15:15:50 -0700462 # Check assignment
463 master = onosCli.getRole( deviceId ).get( 'master' )
464 if ip in master:
465 roleCheck = roleCheck and main.TRUE
466 else:
467 roleCheck = roleCheck and main.FALSE
468 main.log.error( "Error, controller " + ip + " is not" +
469 " master " + "of device " +
470 str( deviceId ) + ". Master is " +
471 repr( master ) + "." )
Jon Hall85794ff2015-07-08 14:12:30 -0700472 utilities.assert_equals(
473 expect=main.TRUE,
474 actual=roleCheck,
475 onpass="Switches were successfully reassigned to designated " +
476 "controller",
477 onfail="Switches were not successfully reassigned" )
478
479 def CASE3( self, main ):
480 """
481 Assign intents
482 """
483 import time
484 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700485 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700486 assert main, "main not defined"
487 assert utilities.assert_equals, "utilities.assert_equals not defined"
488 # NOTE: we must reinstall intents until we have a persistant intent
489 # datastore!
490 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700491 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700492 "assign predetermined host-to-host intents." +\
493 " After installation, check that the intent" +\
494 " is distributed to all nodes and the state" +\
495 " is INSTALLED"
496
497 # install onos-app-fwd
498 main.step( "Install reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700499 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Halla440e872016-03-31 15:15:50 -0700500 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700501 utilities.assert_equals( expect=main.TRUE, actual=installResults,
502 onpass="Install fwd successful",
503 onfail="Install fwd failed" )
504
505 main.step( "Check app ids" )
Jon Halla440e872016-03-31 15:15:50 -0700506 appCheck = main.TRUE
507 threads = []
508 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700509 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Halla440e872016-03-31 15:15:50 -0700510 name="appToIDCheck-" + str( i ),
511 args=[] )
512 threads.append( t )
513 t.start()
514
515 for t in threads:
516 t.join()
517 appCheck = appCheck and t.result
Jon Hall85794ff2015-07-08 14:12:30 -0700518 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700519 main.log.warn( onosCli.apps() )
520 main.log.warn( onosCli.appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700521 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
522 onpass="App Ids seem to be correct",
523 onfail="Something is wrong with app Ids" )
524
525 main.step( "Discovering Hosts( Via pingall for now )" )
526 # FIXME: Once we have a host discovery mechanism, use that instead
527 # REACTIVE FWD test
528 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700529 passMsg = "Reactive Pingall test passed"
530 time1 = time.time()
531 pingResult = main.Mininet1.pingall()
532 time2 = time.time()
533 if not pingResult:
Jon Hallf37d44d2017-05-24 10:37:30 -0700534 main.log.warn( "First pingall failed. Trying again..." )
Jon Hall85794ff2015-07-08 14:12:30 -0700535 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700536 passMsg += " on the second try"
537 utilities.assert_equals(
538 expect=main.TRUE,
539 actual=pingResult,
Jon Hallf37d44d2017-05-24 10:37:30 -0700540 onpass=passMsg,
Jon Hall96091e62015-09-21 17:34:17 -0700541 onfail="Reactive Pingall failed, " +
542 "one or more ping pairs failed" )
543 main.log.info( "Time for pingall: %2f seconds" %
544 ( time2 - time1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700545 # timeout for fwd flows
546 time.sleep( 11 )
547 # uninstall onos-app-fwd
548 main.step( "Uninstall reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700549 node = main.activeNodes[ 0 ]
550 uninstallResult = main.CLIs[ node ].deactivateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700551 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
552 onpass="Uninstall fwd successful",
553 onfail="Uninstall fwd failed" )
554
555 main.step( "Check app ids" )
Jon Halla440e872016-03-31 15:15:50 -0700556 threads = []
557 appCheck2 = main.TRUE
558 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700559 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Halla440e872016-03-31 15:15:50 -0700560 name="appToIDCheck-" + str( i ),
561 args=[] )
562 threads.append( t )
563 t.start()
564
565 for t in threads:
566 t.join()
567 appCheck2 = appCheck2 and t.result
Jon Hall85794ff2015-07-08 14:12:30 -0700568 if appCheck2 != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700569 node = main.activeNodes[ 0 ]
570 main.log.warn( main.CLIs[ node ].apps() )
571 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700572 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
573 onpass="App Ids seem to be correct",
574 onfail="Something is wrong with app Ids" )
575
576 main.step( "Add host intents via cli" )
577 intentIds = []
Jon Halla440e872016-03-31 15:15:50 -0700578 # TODO: move the host numbers to params
579 # Maybe look at all the paths we ping?
Jon Hall85794ff2015-07-08 14:12:30 -0700580 intentAddResult = True
581 hostResult = main.TRUE
582 for i in range( 8, 18 ):
583 main.log.info( "Adding host intent between h" + str( i ) +
584 " and h" + str( i + 10 ) )
585 host1 = "00:00:00:00:00:" + \
586 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
587 host2 = "00:00:00:00:00:" + \
588 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
589 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700590 host1Dict = onosCli.getHost( host1 )
591 host2Dict = onosCli.getHost( host2 )
Jon Hall85794ff2015-07-08 14:12:30 -0700592 host1Id = None
593 host2Id = None
594 if host1Dict and host2Dict:
595 host1Id = host1Dict.get( 'id', None )
596 host2Id = host2Dict.get( 'id', None )
597 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700598 nodeNum = ( i % len( main.activeNodes ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700599 node = main.activeNodes[ nodeNum ]
600 tmpId = main.CLIs[ node ].addHostIntent( host1Id, host2Id )
Jon Hall85794ff2015-07-08 14:12:30 -0700601 if tmpId:
602 main.log.info( "Added intent with id: " + tmpId )
603 intentIds.append( tmpId )
604 else:
605 main.log.error( "addHostIntent returned: " +
606 repr( tmpId ) )
607 else:
608 main.log.error( "Error, getHost() failed for h" + str( i ) +
609 " and/or h" + str( i + 10 ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700610 node = main.activeNodes[ 0 ]
611 hosts = main.CLIs[ node ].hosts()
Jon Hall85794ff2015-07-08 14:12:30 -0700612 main.log.warn( "Hosts output: " )
613 try:
614 main.log.warn( json.dumps( json.loads( hosts ),
615 sort_keys=True,
616 indent=4,
617 separators=( ',', ': ' ) ) )
618 except ( ValueError, TypeError ):
619 main.log.warn( repr( hosts ) )
620 hostResult = main.FALSE
621 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
622 onpass="Found a host id for each host",
623 onfail="Error looking up host ids" )
624
625 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700626 onosIds = onosCli.getAllIntentsId()
Jon Hall85794ff2015-07-08 14:12:30 -0700627 main.log.info( "Submitted intents: " + str( intentIds ) )
628 main.log.info( "Intents in ONOS: " + str( onosIds ) )
629 for intent in intentIds:
630 if intent in onosIds:
631 pass # intent submitted is in onos
632 else:
633 intentAddResult = False
634 if intentAddResult:
635 intentStop = time.time()
636 else:
637 intentStop = None
638 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700639 intents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -0700640 intentStates = []
641 installedCheck = True
642 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
643 count = 0
644 try:
645 for intent in json.loads( intents ):
646 state = intent.get( 'state', None )
647 if "INSTALLED" not in state:
648 installedCheck = False
649 intentId = intent.get( 'id', None )
650 intentStates.append( ( intentId, state ) )
651 except ( ValueError, TypeError ):
652 main.log.exception( "Error parsing intents" )
653 # add submitted intents not in the store
654 tmplist = [ i for i, s in intentStates ]
655 missingIntents = False
656 for i in intentIds:
657 if i not in tmplist:
658 intentStates.append( ( i, " - " ) )
659 missingIntents = True
660 intentStates.sort()
661 for i, s in intentStates:
662 count += 1
663 main.log.info( "%-6s%-15s%-15s" %
664 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700665 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -0700666 try:
667 missing = False
668 if leaders:
669 parsedLeaders = json.loads( leaders )
670 main.log.warn( json.dumps( parsedLeaders,
671 sort_keys=True,
672 indent=4,
673 separators=( ',', ': ' ) ) )
674 # check for all intent partitions
675 topics = []
676 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700677 topics.append( "work-partition-" + str( i ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700678 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700679 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall85794ff2015-07-08 14:12:30 -0700680 for topic in topics:
681 if topic not in ONOStopics:
682 main.log.error( "Error: " + topic +
683 " not in leaders" )
684 missing = True
685 else:
686 main.log.error( "leaders() returned None" )
687 except ( ValueError, TypeError ):
688 main.log.exception( "Error parsing leaders" )
689 main.log.error( repr( leaders ) )
690 # Check all nodes
691 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700692 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700693 response = main.CLIs[ i ].leaders( jsonFormat=False )
694 main.log.warn( str( main.CLIs[ i ].name ) + " leaders output: \n" +
Jon Halla440e872016-03-31 15:15:50 -0700695 str( response ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700696
Jon Halla440e872016-03-31 15:15:50 -0700697 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -0700698 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700699 if partitions:
Jon Hall85794ff2015-07-08 14:12:30 -0700700 parsedPartitions = json.loads( partitions )
701 main.log.warn( json.dumps( parsedPartitions,
702 sort_keys=True,
703 indent=4,
704 separators=( ',', ': ' ) ) )
705 # TODO check for a leader in all paritions
706 # TODO check for consistency among nodes
707 else:
708 main.log.error( "partitions() returned None" )
709 except ( ValueError, TypeError ):
710 main.log.exception( "Error parsing partitions" )
711 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700712 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -0700713 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700714 if pendingMap:
Jon Hall85794ff2015-07-08 14:12:30 -0700715 parsedPending = json.loads( pendingMap )
716 main.log.warn( json.dumps( parsedPending,
717 sort_keys=True,
718 indent=4,
719 separators=( ',', ': ' ) ) )
720 # TODO check something here?
721 else:
722 main.log.error( "pendingMap() returned None" )
723 except ( ValueError, TypeError ):
724 main.log.exception( "Error parsing pending map" )
725 main.log.error( repr( pendingMap ) )
726
727 intentAddResult = bool( intentAddResult and not missingIntents and
728 installedCheck )
729 if not intentAddResult:
730 main.log.error( "Error in pushing host intents to ONOS" )
731
732 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700733 for j in range( 100 ):
Jon Hall85794ff2015-07-08 14:12:30 -0700734 correct = True
735 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700736 for i in main.activeNodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700737 onosIds = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700738 ids = main.CLIs[ i ].getAllIntentsId()
Jon Hall85794ff2015-07-08 14:12:30 -0700739 onosIds.append( ids )
Jon Hallf37d44d2017-05-24 10:37:30 -0700740 main.log.debug( "Intents in " + main.CLIs[ i ].name + ": " +
Jon Hall85794ff2015-07-08 14:12:30 -0700741 str( sorted( onosIds ) ) )
742 if sorted( ids ) != sorted( intentIds ):
743 main.log.warn( "Set of intent IDs doesn't match" )
744 correct = False
745 break
746 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700747 intents = json.loads( main.CLIs[ i ].intents() )
Jon Hall85794ff2015-07-08 14:12:30 -0700748 for intent in intents:
749 if intent[ 'state' ] != "INSTALLED":
750 main.log.warn( "Intent " + intent[ 'id' ] +
751 " is " + intent[ 'state' ] )
752 correct = False
753 break
754 if correct:
755 break
756 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700757 time.sleep( 1 )
Jon Hall85794ff2015-07-08 14:12:30 -0700758 if not intentStop:
759 intentStop = time.time()
760 global gossipTime
761 gossipTime = intentStop - intentStart
762 main.log.info( "It took about " + str( gossipTime ) +
763 " seconds for all intents to appear in each node" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700764 gossipPeriod = int( main.params[ 'timers' ][ 'gossip' ] )
Jon Halla440e872016-03-31 15:15:50 -0700765 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall85794ff2015-07-08 14:12:30 -0700766 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700767 expect=maxGossipTime, actual=gossipTime,
Jon Hall85794ff2015-07-08 14:12:30 -0700768 onpass="ECM anti-entropy for intents worked within " +
769 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700770 onfail="Intent ECM anti-entropy took too long. " +
771 "Expected time:{}, Actual time:{}".format( maxGossipTime,
772 gossipTime ) )
773 if gossipTime <= maxGossipTime:
Jon Hall85794ff2015-07-08 14:12:30 -0700774 intentAddResult = True
775
776 if not intentAddResult or "key" in pendingMap:
777 import time
778 installedCheck = True
779 main.log.info( "Sleeping 60 seconds to see if intents are found" )
780 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700781 onosIds = onosCli.getAllIntentsId()
Jon Hall85794ff2015-07-08 14:12:30 -0700782 main.log.info( "Submitted intents: " + str( intentIds ) )
783 main.log.info( "Intents in ONOS: " + str( onosIds ) )
784 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700785 intents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -0700786 intentStates = []
787 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
788 count = 0
789 try:
790 for intent in json.loads( intents ):
791 # Iter through intents of a node
792 state = intent.get( 'state', None )
793 if "INSTALLED" not in state:
794 installedCheck = False
795 intentId = intent.get( 'id', None )
796 intentStates.append( ( intentId, state ) )
797 except ( ValueError, TypeError ):
798 main.log.exception( "Error parsing intents" )
799 # add submitted intents not in the store
800 tmplist = [ i for i, s in intentStates ]
801 for i in intentIds:
802 if i not in tmplist:
803 intentStates.append( ( i, " - " ) )
804 intentStates.sort()
805 for i, s in intentStates:
806 count += 1
807 main.log.info( "%-6s%-15s%-15s" %
808 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700809 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -0700810 try:
811 missing = False
812 if leaders:
813 parsedLeaders = json.loads( leaders )
814 main.log.warn( json.dumps( parsedLeaders,
815 sort_keys=True,
816 indent=4,
817 separators=( ',', ': ' ) ) )
818 # check for all intent partitions
819 # check for election
820 topics = []
821 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700822 topics.append( "work-partition-" + str( i ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700823 # FIXME: this should only be after we start the app
824 topics.append( "org.onosproject.election" )
825 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700826 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall85794ff2015-07-08 14:12:30 -0700827 for topic in topics:
828 if topic not in ONOStopics:
829 main.log.error( "Error: " + topic +
830 " not in leaders" )
831 missing = True
832 else:
833 main.log.error( "leaders() returned None" )
834 except ( ValueError, TypeError ):
835 main.log.exception( "Error parsing leaders" )
836 main.log.error( repr( leaders ) )
837 # Check all nodes
838 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700839 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700840 node = main.CLIs[ i ]
841 response = node.leaders( jsonFormat=False )
Jon Halla440e872016-03-31 15:15:50 -0700842 main.log.warn( str( node.name ) + " leaders output: \n" +
843 str( response ) )
844
845 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -0700846 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700847 if partitions:
Jon Hall85794ff2015-07-08 14:12:30 -0700848 parsedPartitions = json.loads( partitions )
849 main.log.warn( json.dumps( parsedPartitions,
850 sort_keys=True,
851 indent=4,
852 separators=( ',', ': ' ) ) )
853 # TODO check for a leader in all paritions
854 # TODO check for consistency among nodes
855 else:
856 main.log.error( "partitions() returned None" )
857 except ( ValueError, TypeError ):
858 main.log.exception( "Error parsing partitions" )
859 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700860 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -0700861 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700862 if pendingMap:
Jon Hall85794ff2015-07-08 14:12:30 -0700863 parsedPending = json.loads( pendingMap )
864 main.log.warn( json.dumps( parsedPending,
865 sort_keys=True,
866 indent=4,
867 separators=( ',', ': ' ) ) )
868 # TODO check something here?
869 else:
870 main.log.error( "pendingMap() returned None" )
871 except ( ValueError, TypeError ):
872 main.log.exception( "Error parsing pending map" )
873 main.log.error( repr( pendingMap ) )
874
875 def CASE4( self, main ):
876 """
877 Ping across added host intents
878 """
879 import json
880 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700881 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700882 assert main, "main not defined"
883 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halla440e872016-03-31 15:15:50 -0700884 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700885 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700886 "functionality and check the state of " +\
887 "the intent"
Jon Hall9d2dcad2016-04-08 10:15:20 -0700888
Jon Hallf37d44d2017-05-24 10:37:30 -0700889 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall9d2dcad2016-04-08 10:15:20 -0700890 main.step( "Check Intent state" )
891 installedCheck = True
892 # Print the intent states
893 intents = main.ONOScli1.intents()
894 intentStates = []
895 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
896 count = 0
897 # Iter through intents of a node
898 try:
899 for intent in json.loads( intents ):
900 state = intent.get( 'state', None )
901 if "INSTALLED" not in state:
902 installedCheck = False
903 intentId = intent.get( 'id', None )
904 intentStates.append( ( intentId, state ) )
905 except ( ValueError, TypeError ):
906 main.log.exception( "Error parsing intents." )
907 # Print states
908 intentStates.sort()
909 for i, s in intentStates:
910 count += 1
911 main.log.info( "%-6s%-15s%-15s" %
912 ( str( count ), str( i ), str( s ) ) )
913 utilities.assert_equals( expect=True, actual=installedCheck,
914 onpass="Intents are all INSTALLED",
915 onfail="Intents are not all in " +
916 "INSTALLED state" )
917
Jon Hall85794ff2015-07-08 14:12:30 -0700918 main.step( "Ping across added host intents" )
919 PingResult = main.TRUE
920 for i in range( 8, 18 ):
921 ping = main.Mininet1.pingHost( src="h" + str( i ),
922 target="h" + str( i + 10 ) )
923 PingResult = PingResult and ping
924 if ping == main.FALSE:
925 main.log.warn( "Ping failed between h" + str( i ) +
926 " and h" + str( i + 10 ) )
927 elif ping == main.TRUE:
928 main.log.info( "Ping test passed!" )
929 # Don't set PingResult or you'd override failures
930 if PingResult == main.FALSE:
931 main.log.error(
932 "Intents have not been installed correctly, pings failed." )
933 # TODO: pretty print
934 main.log.warn( "ONOS1 intents: " )
935 try:
Jon Halla440e872016-03-31 15:15:50 -0700936 tmpIntents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -0700937 main.log.warn( json.dumps( json.loads( tmpIntents ),
938 sort_keys=True,
939 indent=4,
940 separators=( ',', ': ' ) ) )
941 except ( ValueError, TypeError ):
942 main.log.warn( repr( tmpIntents ) )
943 utilities.assert_equals(
944 expect=main.TRUE,
945 actual=PingResult,
946 onpass="Intents have been installed correctly and pings work",
947 onfail="Intents have not been installed correctly, pings failed." )
948
Jon Hall85794ff2015-07-08 14:12:30 -0700949 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -0700950 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -0700951 topicCheck = main.TRUE
952 try:
953 if leaders:
954 parsedLeaders = json.loads( leaders )
955 main.log.warn( json.dumps( parsedLeaders,
956 sort_keys=True,
957 indent=4,
958 separators=( ',', ': ' ) ) )
959 # check for all intent partitions
960 # check for election
961 # TODO: Look at Devices as topics now that it uses this system
962 topics = []
963 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700964 topics.append( "work-partition-" + str( i ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700965 # FIXME: this should only be after we start the app
966 # FIXME: topics.append( "org.onosproject.election" )
967 # Print leaders output
968 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700969 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall85794ff2015-07-08 14:12:30 -0700970 for topic in topics:
971 if topic not in ONOStopics:
972 main.log.error( "Error: " + topic +
973 " not in leaders" )
974 topicCheck = main.FALSE
975 else:
976 main.log.error( "leaders() returned None" )
977 topicCheck = main.FALSE
978 except ( ValueError, TypeError ):
979 topicCheck = main.FALSE
980 main.log.exception( "Error parsing leaders" )
981 main.log.error( repr( leaders ) )
982 # TODO: Check for a leader of these topics
Jon Halla440e872016-03-31 15:15:50 -0700983 # Check all nodes
984 if topicCheck:
985 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700986 node = main.CLIs[ i ]
987 response = node.leaders( jsonFormat=False )
Jon Halla440e872016-03-31 15:15:50 -0700988 main.log.warn( str( node.name ) + " leaders output: \n" +
989 str( response ) )
990
Jon Hall85794ff2015-07-08 14:12:30 -0700991 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
992 onpass="intent Partitions is in leaders",
993 onfail="Some topics were lost " )
994 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -0700995 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -0700996 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700997 if partitions:
Jon Hall85794ff2015-07-08 14:12:30 -0700998 parsedPartitions = json.loads( partitions )
999 main.log.warn( json.dumps( parsedPartitions,
1000 sort_keys=True,
1001 indent=4,
1002 separators=( ',', ': ' ) ) )
1003 # TODO check for a leader in all paritions
1004 # TODO check for consistency among nodes
1005 else:
1006 main.log.error( "partitions() returned None" )
1007 except ( ValueError, TypeError ):
1008 main.log.exception( "Error parsing partitions" )
1009 main.log.error( repr( partitions ) )
1010 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001011 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -07001012 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001013 if pendingMap:
Jon Hall85794ff2015-07-08 14:12:30 -07001014 parsedPending = json.loads( pendingMap )
1015 main.log.warn( json.dumps( parsedPending,
1016 sort_keys=True,
1017 indent=4,
1018 separators=( ',', ': ' ) ) )
1019 # TODO check something here?
1020 else:
1021 main.log.error( "pendingMap() returned None" )
1022 except ( ValueError, TypeError ):
1023 main.log.exception( "Error parsing pending map" )
1024 main.log.error( repr( pendingMap ) )
1025
1026 if not installedCheck:
1027 main.log.info( "Waiting 60 seconds to see if the state of " +
1028 "intents change" )
1029 time.sleep( 60 )
1030 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001031 intents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -07001032 intentStates = []
1033 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1034 count = 0
1035 # Iter through intents of a node
1036 try:
1037 for intent in json.loads( intents ):
1038 state = intent.get( 'state', None )
1039 if "INSTALLED" not in state:
1040 installedCheck = False
1041 intentId = intent.get( 'id', None )
1042 intentStates.append( ( intentId, state ) )
1043 except ( ValueError, TypeError ):
1044 main.log.exception( "Error parsing intents." )
1045 intentStates.sort()
1046 for i, s in intentStates:
1047 count += 1
1048 main.log.info( "%-6s%-15s%-15s" %
1049 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001050 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -07001051 try:
1052 missing = False
1053 if leaders:
1054 parsedLeaders = json.loads( leaders )
1055 main.log.warn( json.dumps( parsedLeaders,
1056 sort_keys=True,
1057 indent=4,
1058 separators=( ',', ': ' ) ) )
1059 # check for all intent partitions
1060 # check for election
1061 topics = []
1062 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001063 topics.append( "work-partition-" + str( i ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001064 # FIXME: this should only be after we start the app
1065 topics.append( "org.onosproject.election" )
1066 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -07001067 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall85794ff2015-07-08 14:12:30 -07001068 for topic in topics:
1069 if topic not in ONOStopics:
1070 main.log.error( "Error: " + topic +
1071 " not in leaders" )
1072 missing = True
1073 else:
1074 main.log.error( "leaders() returned None" )
1075 except ( ValueError, TypeError ):
1076 main.log.exception( "Error parsing leaders" )
1077 main.log.error( repr( leaders ) )
1078 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001079 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001080 node = main.CLIs[ i ]
1081 response = node.leaders( jsonFormat=False )
Jon Halla440e872016-03-31 15:15:50 -07001082 main.log.warn( str( node.name ) + " leaders output: \n" +
1083 str( response ) )
1084
1085 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -07001086 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001087 if partitions:
Jon Hall85794ff2015-07-08 14:12:30 -07001088 parsedPartitions = json.loads( partitions )
1089 main.log.warn( json.dumps( parsedPartitions,
1090 sort_keys=True,
1091 indent=4,
1092 separators=( ',', ': ' ) ) )
1093 # TODO check for a leader in all paritions
1094 # TODO check for consistency among nodes
1095 else:
1096 main.log.error( "partitions() returned None" )
1097 except ( ValueError, TypeError ):
1098 main.log.exception( "Error parsing partitions" )
1099 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001100 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -07001101 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001102 if pendingMap:
Jon Hall85794ff2015-07-08 14:12:30 -07001103 parsedPending = json.loads( pendingMap )
1104 main.log.warn( json.dumps( parsedPending,
1105 sort_keys=True,
1106 indent=4,
1107 separators=( ',', ': ' ) ) )
1108 # TODO check something here?
1109 else:
1110 main.log.error( "pendingMap() returned None" )
1111 except ( ValueError, TypeError ):
1112 main.log.exception( "Error parsing pending map" )
1113 main.log.error( repr( pendingMap ) )
1114 # Print flowrules
Jon Hallf37d44d2017-05-24 10:37:30 -07001115 node = main.activeNodes[ 0 ]
1116 main.log.debug( main.CLIs[ node ].flows( jsonFormat=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001117 main.step( "Wait a minute then ping again" )
1118 # the wait is above
1119 PingResult = main.TRUE
1120 for i in range( 8, 18 ):
1121 ping = main.Mininet1.pingHost( src="h" + str( i ),
1122 target="h" + str( i + 10 ) )
1123 PingResult = PingResult and ping
1124 if ping == main.FALSE:
1125 main.log.warn( "Ping failed between h" + str( i ) +
1126 " and h" + str( i + 10 ) )
1127 elif ping == main.TRUE:
1128 main.log.info( "Ping test passed!" )
1129 # Don't set PingResult or you'd override failures
1130 if PingResult == main.FALSE:
1131 main.log.error(
1132 "Intents have not been installed correctly, pings failed." )
1133 # TODO: pretty print
1134 main.log.warn( "ONOS1 intents: " )
1135 try:
Jon Halla440e872016-03-31 15:15:50 -07001136 tmpIntents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -07001137 main.log.warn( json.dumps( json.loads( tmpIntents ),
1138 sort_keys=True,
1139 indent=4,
1140 separators=( ',', ': ' ) ) )
1141 except ( ValueError, TypeError ):
1142 main.log.warn( repr( tmpIntents ) )
1143 utilities.assert_equals(
1144 expect=main.TRUE,
1145 actual=PingResult,
1146 onpass="Intents have been installed correctly and pings work",
1147 onfail="Intents have not been installed correctly, pings failed." )
1148
1149 def CASE5( self, main ):
1150 """
1151 Reading state of ONOS
1152 """
1153 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001154 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001155 assert main, "main not defined"
1156 assert utilities.assert_equals, "utilities.assert_equals not defined"
1157
1158 main.case( "Setting up and gathering data for current state" )
1159 # The general idea for this test case is to pull the state of
1160 # ( intents,flows, topology,... ) from each ONOS node
1161 # We can then compare them with each other and also with past states
1162
1163 main.step( "Check that each switch has a master" )
1164 global mastershipState
1165 mastershipState = '[]'
1166
1167 # Assert that each device has a master
Jon Halla440e872016-03-31 15:15:50 -07001168 rolesNotNull = main.TRUE
1169 threads = []
1170 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001171 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Halla440e872016-03-31 15:15:50 -07001172 name="rolesNotNull-" + str( i ),
1173 args=[] )
1174 threads.append( t )
1175 t.start()
1176
1177 for t in threads:
1178 t.join()
1179 rolesNotNull = rolesNotNull and t.result
Jon Hall85794ff2015-07-08 14:12:30 -07001180 utilities.assert_equals(
1181 expect=main.TRUE,
1182 actual=rolesNotNull,
1183 onpass="Each device has a master",
1184 onfail="Some devices don't have a master assigned" )
1185
1186 main.step( "Get the Mastership of each switch" )
1187 ONOS1Mastership = main.ONOScli1.roles()
1188 # TODO: Make this a meaningful check
1189 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1190 main.log.error( "Error in getting ONOS roles" )
1191 main.log.warn(
1192 "ONOS1 mastership response: " +
1193 repr( ONOS1Mastership ) )
1194 consistentMastership = main.FALSE
1195 else:
1196 mastershipState = ONOS1Mastership
1197 consistentMastership = main.TRUE
1198
1199 main.step( "Get the intents from each controller" )
1200 global intentState
1201 intentState = []
1202 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1203 intentCheck = main.FALSE
1204 if "Error" in ONOS1Intents or not ONOS1Intents:
1205 main.log.error( "Error in getting ONOS intents" )
1206 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1207 else:
1208 intentCheck = main.TRUE
1209
1210 main.step( "Get the flows from each controller" )
1211 global flowState
1212 flowState = []
1213 flowCheck = main.FALSE
1214 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
1215 if "Error" in ONOS1Flows or not ONOS1Flows:
1216 main.log.error( "Error in getting ONOS flows" )
1217 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
1218 else:
1219 # TODO: Do a better check, maybe compare flows on switches?
1220 flowState = ONOS1Flows
1221 flowCheck = main.TRUE
1222
1223 main.step( "Get the OF Table entries" )
1224 global flows
1225 flows = []
1226 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001227 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001228 if flowCheck == main.FALSE:
1229 for table in flows:
1230 main.log.warn( table )
1231 # TODO: Compare switch flow tables with ONOS flow tables
1232
1233 main.step( "Collecting topology information from ONOS" )
1234 devices = []
1235 devices.append( main.ONOScli1.devices() )
1236 hosts = []
1237 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1238 ports = []
1239 ports.append( main.ONOScli1.ports() )
1240 links = []
1241 links.append( main.ONOScli1.links() )
1242 clusters = []
1243 clusters.append( main.ONOScli1.clusters() )
1244
1245 main.step( "Each host has an IP address" )
1246 ipResult = main.TRUE
1247 for controller in range( 0, len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001248 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Halla440e872016-03-31 15:15:50 -07001249 if hosts[ controller ]:
1250 for host in hosts[ controller ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07001251 if not host.get( 'ipAddresses', [] ):
Jon Halla440e872016-03-31 15:15:50 -07001252 main.log.error( "Error with host ips on controller" +
1253 controllerStr + ": " + str( host ) )
1254 ipResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07001255 utilities.assert_equals(
1256 expect=main.TRUE,
1257 actual=ipResult,
1258 onpass="The ips of the hosts aren't empty",
1259 onfail="The ip of at least one host is missing" )
1260
1261 # there should always only be one cluster
1262 main.step( "There is only one dataplane cluster" )
1263 try:
1264 numClusters = len( json.loads( clusters[ 0 ] ) )
1265 except ( ValueError, TypeError ):
1266 main.log.exception( "Error parsing clusters[0]: " +
1267 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001268 numClusters = "ERROR"
Jon Hall85794ff2015-07-08 14:12:30 -07001269 clusterResults = main.FALSE
1270 if numClusters == 1:
1271 clusterResults = main.TRUE
1272 utilities.assert_equals(
1273 expect=1,
1274 actual=numClusters,
1275 onpass="ONOS shows 1 SCC",
1276 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1277
1278 main.step( "Comparing ONOS topology to MN" )
1279 devicesResults = main.TRUE
1280 linksResults = main.TRUE
1281 hostsResults = main.TRUE
1282 mnSwitches = main.Mininet1.getSwitches()
1283 mnLinks = main.Mininet1.getLinks()
1284 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001285 for controller in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001286 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07001287 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001288 "Error" not in devices[ controller ] and\
1289 "Error" not in ports[ controller ]:
1290 currentDevicesResult = main.Mininet1.compareSwitches(
1291 mnSwitches,
1292 json.loads( devices[ controller ] ),
1293 json.loads( ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001294 else:
1295 currentDevicesResult = main.FALSE
1296 utilities.assert_equals( expect=main.TRUE,
1297 actual=currentDevicesResult,
1298 onpass="ONOS" + controllerStr +
1299 " Switches view is correct",
1300 onfail="ONOS" + controllerStr +
1301 " Switches view is incorrect" )
1302 if links[ controller ] and "Error" not in links[ controller ]:
1303 currentLinksResult = main.Mininet1.compareLinks(
1304 mnSwitches, mnLinks,
1305 json.loads( links[ controller ] ) )
1306 else:
1307 currentLinksResult = main.FALSE
1308 utilities.assert_equals( expect=main.TRUE,
1309 actual=currentLinksResult,
1310 onpass="ONOS" + controllerStr +
1311 " links view is correct",
1312 onfail="ONOS" + controllerStr +
1313 " links view is incorrect" )
1314
Jon Halla440e872016-03-31 15:15:50 -07001315 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall85794ff2015-07-08 14:12:30 -07001316 currentHostsResult = main.Mininet1.compareHosts(
1317 mnHosts,
1318 hosts[ controller ] )
1319 else:
1320 currentHostsResult = main.FALSE
1321 utilities.assert_equals( expect=main.TRUE,
1322 actual=currentHostsResult,
1323 onpass="ONOS" + controllerStr +
1324 " hosts exist in Mininet",
1325 onfail="ONOS" + controllerStr +
1326 " hosts don't match Mininet" )
1327
1328 devicesResults = devicesResults and currentDevicesResult
1329 linksResults = linksResults and currentLinksResult
1330 hostsResults = hostsResults and currentHostsResult
1331
1332 main.step( "Device information is correct" )
1333 utilities.assert_equals(
1334 expect=main.TRUE,
1335 actual=devicesResults,
1336 onpass="Device information is correct",
1337 onfail="Device information is incorrect" )
1338
1339 main.step( "Links are correct" )
1340 utilities.assert_equals(
1341 expect=main.TRUE,
1342 actual=linksResults,
1343 onpass="Link are correct",
1344 onfail="Links are incorrect" )
1345
1346 main.step( "Hosts are correct" )
1347 utilities.assert_equals(
1348 expect=main.TRUE,
1349 actual=hostsResults,
1350 onpass="Hosts are correct",
1351 onfail="Hosts are incorrect" )
1352
1353 def CASE6( self, main ):
1354 """
1355 The Failure case.
1356 """
1357 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001358 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001359 assert main, "main not defined"
1360 assert utilities.assert_equals, "utilities.assert_equals not defined"
1361
1362 # Reset non-persistent variables
1363 try:
1364 iCounterValue = 0
1365 except NameError:
1366 main.log.error( "iCounterValue not defined, setting to 0" )
1367 iCounterValue = 0
1368
1369 main.case( "Restart ONOS node" )
Jon Hall783bbf92015-07-23 14:33:19 -07001370 main.caseExplanation = "Killing ONOS process and restart cli " +\
Jon Hall85794ff2015-07-08 14:12:30 -07001371 "sessions once onos is up."
Jon Hall96091e62015-09-21 17:34:17 -07001372
1373 main.step( "Checking ONOS Logs for errors" )
1374 for node in main.nodes:
1375 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1376 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1377
Jon Hall85794ff2015-07-08 14:12:30 -07001378 main.step( "Killing ONOS processes" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001379 killResult = main.ONOSbench.onosKill( main.nodes[ 0 ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001380 start = time.time()
1381 utilities.assert_equals( expect=main.TRUE, actual=killResult,
1382 onpass="ONOS Killed",
1383 onfail="Error killing ONOS" )
1384
1385 main.step( "Checking if ONOS is up yet" )
1386 count = 0
1387 while count < 10:
Jon Hallf37d44d2017-05-24 10:37:30 -07001388 onos1Isup = main.ONOSbench.isup( main.nodes[ 0 ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001389 if onos1Isup == main.TRUE:
1390 elapsed = time.time() - start
1391 break
1392 else:
1393 count = count + 1
1394 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
1395 onpass="ONOS is back up",
1396 onfail="ONOS failed to start" )
1397
Jon Hall6509dbf2016-06-21 17:01:17 -07001398 main.step( "Starting ONOS CLI sessions" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001399 cliResults = main.ONOScli1.startOnosCli( main.nodes[ 0 ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001400 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1401 onpass="ONOS cli startup successful",
1402 onfail="ONOS cli startup failed" )
1403
1404 if elapsed:
1405 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
1406 str( elapsed ) )
1407 main.restartTime = elapsed
1408 else:
1409 main.restartTime = -1
1410 time.sleep( 5 )
1411 # rerun on election apps
1412 main.ONOScli1.electionTestRun()
1413
1414 def CASE7( self, main ):
1415 """
1416 Check state after ONOS failure
1417 """
1418 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001419 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001420 assert main, "main not defined"
1421 assert utilities.assert_equals, "utilities.assert_equals not defined"
1422 main.case( "Running ONOS Constant State Tests" )
Jon Hall6e709752016-02-01 13:38:46 -08001423
Jon Hall85794ff2015-07-08 14:12:30 -07001424 main.step( "Check that each switch has a master" )
1425 # Assert that each device has a master
Jon Halla440e872016-03-31 15:15:50 -07001426 rolesNotNull = main.TRUE
1427 threads = []
1428 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001429 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Halla440e872016-03-31 15:15:50 -07001430 name="rolesNotNull-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001431 args=[] )
Jon Halla440e872016-03-31 15:15:50 -07001432 threads.append( t )
1433 t.start()
1434
1435 for t in threads:
1436 t.join()
1437 rolesNotNull = rolesNotNull and t.result
Jon Hall85794ff2015-07-08 14:12:30 -07001438 utilities.assert_equals(
1439 expect=main.TRUE,
1440 actual=rolesNotNull,
1441 onpass="Each device has a master",
1442 onfail="Some devices don't have a master assigned" )
1443
1444 main.step( "Check if switch roles are consistent across all nodes" )
1445 ONOS1Mastership = main.ONOScli1.roles()
1446 # FIXME: Refactor this whole case for single instance
1447 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1448 main.log.error( "Error in getting ONOS mastership" )
1449 main.log.warn( "ONOS1 mastership response: " +
1450 repr( ONOS1Mastership ) )
1451 consistentMastership = main.FALSE
1452 else:
1453 consistentMastership = main.TRUE
1454 utilities.assert_equals(
1455 expect=main.TRUE,
1456 actual=consistentMastership,
1457 onpass="Switch roles are consistent across all ONOS nodes",
1458 onfail="ONOS nodes have different views of switch roles" )
1459
1460 description2 = "Compare switch roles from before failure"
1461 main.step( description2 )
1462
1463 currentJson = json.loads( ONOS1Mastership )
1464 oldJson = json.loads( mastershipState )
1465 mastershipCheck = main.TRUE
1466 for i in range( 1, 29 ):
1467 switchDPID = str(
1468 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1469
1470 current = [ switch[ 'master' ] for switch in currentJson
1471 if switchDPID in switch[ 'id' ] ]
1472 old = [ switch[ 'master' ] for switch in oldJson
1473 if switchDPID in switch[ 'id' ] ]
1474 if current == old:
1475 mastershipCheck = mastershipCheck and main.TRUE
1476 else:
1477 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1478 mastershipCheck = main.FALSE
1479 utilities.assert_equals(
1480 expect=main.TRUE,
1481 actual=mastershipCheck,
1482 onpass="Mastership of Switches was not changed",
1483 onfail="Mastership of some switches changed" )
1484 mastershipCheck = mastershipCheck and consistentMastership
1485
1486 main.step( "Get the intents and compare across all nodes" )
1487 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1488 intentCheck = main.FALSE
1489 if "Error" in ONOS1Intents or not ONOS1Intents:
1490 main.log.error( "Error in getting ONOS intents" )
1491 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1492 else:
1493 intentCheck = main.TRUE
1494 utilities.assert_equals(
1495 expect=main.TRUE,
1496 actual=intentCheck,
1497 onpass="Intents are consistent across all ONOS nodes",
1498 onfail="ONOS nodes have different views of intents" )
1499 # Print the intent states
1500 intents = []
1501 intents.append( ONOS1Intents )
1502 intentStates = []
1503 for node in intents: # Iter through ONOS nodes
1504 nodeStates = []
1505 # Iter through intents of a node
1506 for intent in json.loads( node ):
1507 nodeStates.append( intent[ 'state' ] )
1508 intentStates.append( nodeStates )
Jon Hallf37d44d2017-05-24 10:37:30 -07001509 out = [ ( i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
Jon Hall85794ff2015-07-08 14:12:30 -07001510 main.log.info( dict( out ) )
1511
1512 # NOTE: Store has no durability, so intents are lost across system
1513 # restarts
1514 """
1515 main.step( "Compare current intents with intents before the failure" )
1516 # NOTE: this requires case 5 to pass for intentState to be set.
1517 # maybe we should stop the test if that fails?
1518 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07001519 try:
1520 intentState
1521 except NameError:
1522 main.log.warn( "No previous intent state was saved" )
1523 else:
1524 if intentState and intentState == ONOSIntents[ 0 ]:
1525 sameIntents = main.TRUE
1526 main.log.info( "Intents are consistent with before failure" )
1527 # TODO: possibly the states have changed? we may need to figure out
1528 # what the acceptable states are
1529 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1530 sameIntents = main.TRUE
1531 try:
1532 before = json.loads( intentState )
1533 after = json.loads( ONOSIntents[ 0 ] )
1534 for intent in before:
1535 if intent not in after:
1536 sameIntents = main.FALSE
1537 main.log.debug( "Intent is not currently in ONOS " +
1538 "(at least in the same form):" )
1539 main.log.debug( json.dumps( intent ) )
1540 except ( ValueError, TypeError ):
1541 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001542 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07001543 main.log.debug( repr( intentState ) )
1544 if sameIntents == main.FALSE:
1545 try:
1546 main.log.debug( "ONOS intents before: " )
1547 main.log.debug( json.dumps( json.loads( intentState ),
1548 sort_keys=True, indent=4,
1549 separators=( ',', ': ' ) ) )
1550 main.log.debug( "Current ONOS intents: " )
1551 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1552 sort_keys=True, indent=4,
1553 separators=( ',', ': ' ) ) )
1554 except ( ValueError, TypeError ):
1555 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001556 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07001557 main.log.debug( repr( intentState ) )
1558 utilities.assert_equals(
1559 expect=main.TRUE,
1560 actual=sameIntents,
1561 onpass="Intents are consistent with before failure",
1562 onfail="The Intents changed during failure" )
Jon Hall85794ff2015-07-08 14:12:30 -07001563 intentCheck = intentCheck and sameIntents
1564 """
1565 main.step( "Get the OF Table entries and compare to before " +
1566 "component failure" )
1567 FlowTables = main.TRUE
Jon Hall85794ff2015-07-08 14:12:30 -07001568 for i in range( 28 ):
1569 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08001570 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hallf37d44d2017-05-24 10:37:30 -07001571 curSwitch = main.Mininet1.flowTableComp( flows[ i ], tmpFlows )
Jon Hall41d39f12016-04-11 22:54:35 -07001572 FlowTables = FlowTables and curSwitch
1573 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08001574 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001575 utilities.assert_equals(
1576 expect=main.TRUE,
1577 actual=FlowTables,
1578 onpass="No changes were found in the flow tables",
1579 onfail="Changes were found in the flow tables" )
1580
1581 main.step( "Leadership Election is still functional" )
1582 # Test of LeadershipElection
1583
Jon Halla440e872016-03-31 15:15:50 -07001584 leader = main.nodes[ main.activeNodes[ 0 ] ].ip_address
Jon Hall85794ff2015-07-08 14:12:30 -07001585 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001586 for controller in range( 1, main.numCtrls + 1 ):
Jon Hall85794ff2015-07-08 14:12:30 -07001587 # loop through ONOScli handlers
1588 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
1589 leaderN = node.electionTestLeader()
1590 # verify leader is ONOS1
1591 # NOTE even though we restarted ONOS, it is the only one so onos 1
1592 # must be leader
1593 if leaderN == leader:
1594 # all is well
1595 pass
1596 elif leaderN == main.FALSE:
1597 # error in response
1598 main.log.error( "Something is wrong with " +
1599 "electionTestLeader function, check the" +
1600 " error logs" )
1601 leaderResult = main.FALSE
1602 elif leader != leaderN:
1603 leaderResult = main.FALSE
1604 main.log.error( "ONOS" + str( controller ) + " sees " +
1605 str( leaderN ) +
1606 " as the leader of the election app. " +
1607 "Leader should be " + str( leader ) )
1608 utilities.assert_equals(
1609 expect=main.TRUE,
1610 actual=leaderResult,
1611 onpass="Leadership election passed",
1612 onfail="Something went wrong with Leadership election" )
1613
1614 def CASE8( self, main ):
1615 """
1616 Compare topo
1617 """
1618 import json
1619 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001620 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001621 assert main, "main not defined"
1622 assert utilities.assert_equals, "utilities.assert_equals not defined"
1623
1624 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07001625 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall85794ff2015-07-08 14:12:30 -07001626 " and ONOS"
Jon Hall85794ff2015-07-08 14:12:30 -07001627 topoResult = main.FALSE
1628 elapsed = 0
1629 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08001630 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall85794ff2015-07-08 14:12:30 -07001631 startTime = time.time()
1632 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08001633 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07001634 devicesResults = main.TRUE
1635 linksResults = main.TRUE
1636 hostsResults = main.TRUE
1637 hostAttachmentResults = True
Jon Hall85794ff2015-07-08 14:12:30 -07001638 count += 1
1639 cliStart = time.time()
1640 devices = []
1641 devices.append( main.ONOScli1.devices() )
1642 hosts = []
1643 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1644 ipResult = main.TRUE
1645 for controller in range( 0, len( hosts ) ):
1646 controllerStr = str( controller + 1 )
1647 for host in hosts[ controller ]:
1648 if host is None or host.get( 'ipAddresses', [] ) == []:
1649 main.log.error(
1650 "DEBUG:Error with host ips on controller" +
1651 controllerStr + ": " + str( host ) )
1652 ipResult = main.FALSE
1653 ports = []
1654 ports.append( main.ONOScli1.ports() )
1655 links = []
1656 links.append( main.ONOScli1.links() )
1657 clusters = []
1658 clusters.append( main.ONOScli1.clusters() )
1659
1660 elapsed = time.time() - startTime
1661 cliTime = time.time() - cliStart
1662 print "CLI time: " + str( cliTime )
1663
1664 mnSwitches = main.Mininet1.getSwitches()
1665 mnLinks = main.Mininet1.getLinks()
1666 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001667 for controller in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07001668 controllerStr = str( controller + 1 )
1669 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001670 "Error" not in devices[ controller ] and\
1671 "Error" not in ports[ controller ]:
Jon Hall85794ff2015-07-08 14:12:30 -07001672
Jon Hallc6793552016-01-19 14:18:37 -08001673 try:
1674 currentDevicesResult = main.Mininet1.compareSwitches(
1675 mnSwitches,
1676 json.loads( devices[ controller ] ),
1677 json.loads( ports[ controller ] ) )
1678 except ( TypeError, ValueError ) as e:
1679 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
1680 devices[ controller ], ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001681 else:
1682 currentDevicesResult = main.FALSE
1683 utilities.assert_equals( expect=main.TRUE,
1684 actual=currentDevicesResult,
1685 onpass="ONOS" + controllerStr +
1686 " Switches view is correct",
1687 onfail="ONOS" + controllerStr +
1688 " Switches view is incorrect" )
1689
1690 if links[ controller ] and "Error" not in links[ controller ]:
1691 currentLinksResult = main.Mininet1.compareLinks(
1692 mnSwitches, mnLinks,
1693 json.loads( links[ controller ] ) )
1694 else:
1695 currentLinksResult = main.FALSE
1696 utilities.assert_equals( expect=main.TRUE,
1697 actual=currentLinksResult,
1698 onpass="ONOS" + controllerStr +
1699 " links view is correct",
1700 onfail="ONOS" + controllerStr +
1701 " links view is incorrect" )
1702
1703 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1704 currentHostsResult = main.Mininet1.compareHosts(
1705 mnHosts,
1706 hosts[ controller ] )
1707 else:
1708 currentHostsResult = main.FALSE
1709 utilities.assert_equals( expect=main.TRUE,
1710 actual=currentHostsResult,
1711 onpass="ONOS" + controllerStr +
1712 " hosts exist in Mininet",
1713 onfail="ONOS" + controllerStr +
1714 " hosts don't match Mininet" )
1715 # CHECKING HOST ATTACHMENT POINTS
1716 hostAttachment = True
1717 zeroHosts = False
1718 # FIXME: topo-HA/obelisk specific mappings:
1719 # key is mac and value is dpid
1720 mappings = {}
1721 for i in range( 1, 29 ): # hosts 1 through 28
1722 # set up correct variables:
Jon Hallf37d44d2017-05-24 10:37:30 -07001723 macId = "00:" * 5 + hex( i ).split( "0x" )[ 1 ].upper().zfill( 2 )
Jon Hall85794ff2015-07-08 14:12:30 -07001724 if i == 1:
Jon Hallf37d44d2017-05-24 10:37:30 -07001725 deviceId = "1000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001726 elif i == 2:
Jon Hallf37d44d2017-05-24 10:37:30 -07001727 deviceId = "2000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001728 elif i == 3:
Jon Hallf37d44d2017-05-24 10:37:30 -07001729 deviceId = "3000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001730 elif i == 4:
Jon Hallf37d44d2017-05-24 10:37:30 -07001731 deviceId = "3004".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001732 elif i == 5:
Jon Hallf37d44d2017-05-24 10:37:30 -07001733 deviceId = "5000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001734 elif i == 6:
Jon Hallf37d44d2017-05-24 10:37:30 -07001735 deviceId = "6000".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001736 elif i == 7:
Jon Hallf37d44d2017-05-24 10:37:30 -07001737 deviceId = "6007".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001738 elif i >= 8 and i <= 17:
1739 dpid = '3' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07001740 deviceId = dpid.zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001741 elif i >= 18 and i <= 27:
1742 dpid = '6' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07001743 deviceId = dpid.zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001744 elif i == 28:
Jon Hallf37d44d2017-05-24 10:37:30 -07001745 deviceId = "2800".zfill( 16 )
Jon Hall85794ff2015-07-08 14:12:30 -07001746 mappings[ macId ] = deviceId
1747 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1748 if hosts[ controller ] == []:
1749 main.log.warn( "There are no hosts discovered" )
1750 zeroHosts = True
1751 else:
1752 for host in hosts[ controller ]:
1753 mac = None
1754 location = None
1755 device = None
1756 port = None
1757 try:
1758 mac = host.get( 'mac' )
1759 assert mac, "mac field could not be found for this host object"
1760
Jeremy Ronquillo0e538bc2017-06-13 15:16:09 -07001761 location = host.get( 'locations' )[ 0 ]
Jon Hall85794ff2015-07-08 14:12:30 -07001762 assert location, "location field could not be found for this host object"
1763
1764 # Trim the protocol identifier off deviceId
Jon Hallf37d44d2017-05-24 10:37:30 -07001765 device = str( location.get( 'elementId' ) ).split( ':' )[ 1 ]
Jon Hall85794ff2015-07-08 14:12:30 -07001766 assert device, "elementId field could not be found for this host location object"
1767
1768 port = location.get( 'port' )
1769 assert port, "port field could not be found for this host location object"
1770
1771 # Now check if this matches where they should be
1772 if mac and device and port:
1773 if str( port ) != "1":
1774 main.log.error( "The attachment port is incorrect for " +
1775 "host " + str( mac ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07001776 ". Expected: 1 Actual: " + str( port ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001777 hostAttachment = False
1778 if device != mappings[ str( mac ) ]:
1779 main.log.error( "The attachment device is incorrect for " +
1780 "host " + str( mac ) +
1781 ". Expected: " + mappings[ str( mac ) ] +
1782 " Actual: " + device )
1783 hostAttachment = False
1784 else:
1785 hostAttachment = False
1786 except AssertionError:
1787 main.log.exception( "Json object not as expected" )
1788 main.log.error( repr( host ) )
1789 hostAttachment = False
1790 else:
1791 main.log.error( "No hosts json output or \"Error\"" +
1792 " in output. hosts = " +
1793 repr( hosts[ controller ] ) )
1794 if zeroHosts is False:
1795 hostAttachment = True
1796
Jon Hall85794ff2015-07-08 14:12:30 -07001797 devicesResults = devicesResults and currentDevicesResult
1798 linksResults = linksResults and currentLinksResult
1799 hostsResults = hostsResults and currentHostsResult
1800 hostAttachmentResults = hostAttachmentResults and\
1801 hostAttachment
1802
Jon Halla440e872016-03-31 15:15:50 -07001803 # "consistent" results don't make sense for single instance
Jon Hall85794ff2015-07-08 14:12:30 -07001804 # there should always only be one cluster
Jon Hall85794ff2015-07-08 14:12:30 -07001805 clusterResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07001806 try:
1807 numClusters = len( json.loads( clusters[ 0 ] ) )
1808 except ( ValueError, TypeError ):
1809 main.log.exception( "Error parsing clusters[0]: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001810 repr( clusters[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07001811 numClusters = "ERROR"
1812 clusterResults = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07001813 if numClusters == 1:
1814 clusterResults = main.TRUE
1815 utilities.assert_equals(
1816 expect=1,
1817 actual=numClusters,
1818 onpass="ONOS shows 1 SCC",
1819 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1820
1821 topoResult = ( devicesResults and linksResults
1822 and hostsResults and ipResult and clusterResults and
1823 hostAttachmentResults )
1824
1825 topoResult = topoResult and int( count <= 2 )
1826 note = "note it takes about " + str( int( cliTime ) ) + \
1827 " seconds for the test to make all the cli calls to fetch " +\
1828 "the topology from each ONOS instance"
1829 main.log.info(
1830 "Very crass estimate for topology discovery/convergence( " +
1831 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1832 str( count ) + " tries" )
1833 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1834 onpass="Topology Check Test successful",
1835 onfail="Topology Check Test NOT successful" )
Jon Hall41d39f12016-04-11 22:54:35 -07001836 main.step( "Checking ONOS nodes" )
1837 nodeResults = utilities.retry( main.HA.nodesCheck,
1838 False,
Jon Hallf37d44d2017-05-24 10:37:30 -07001839 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -07001840 attempts=5 )
1841
1842 utilities.assert_equals( expect=True, actual=nodeResults,
1843 onpass="Nodes check successful",
1844 onfail="Nodes check NOT successful" )
1845 if not nodeResults:
1846 for i in main.activeNodes:
1847 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hallf37d44d2017-05-24 10:37:30 -07001848 main.CLIs[ i ].name,
1849 main.CLIs[ i ].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001850
Jon Halld2871c22016-07-26 11:01:14 -07001851 if not topoResult:
1852 main.cleanup()
1853 main.exit()
1854
Jon Hall85794ff2015-07-08 14:12:30 -07001855 def CASE9( self, main ):
1856 """
1857 Link s3-s28 down
1858 """
1859 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001860 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001861 assert main, "main not defined"
1862 assert utilities.assert_equals, "utilities.assert_equals not defined"
1863 # NOTE: You should probably run a topology check after this
1864
1865 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1866
1867 description = "Turn off a link to ensure that Link Discovery " +\
1868 "is working properly"
1869 main.case( description )
1870
1871 main.step( "Kill Link between s3 and s28" )
1872 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
1873 main.log.info( "Waiting " + str( linkSleep ) +
1874 " seconds for link down to be discovered" )
1875 time.sleep( linkSleep )
1876 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
1877 onpass="Link down successful",
1878 onfail="Failed to bring link down" )
1879 # TODO do some sort of check here
1880
1881 def CASE10( self, main ):
1882 """
1883 Link s3-s28 up
1884 """
1885 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001886 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001887 assert main, "main not defined"
1888 assert utilities.assert_equals, "utilities.assert_equals not defined"
1889 # NOTE: You should probably run a topology check after this
1890
1891 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1892
1893 description = "Restore a link to ensure that Link Discovery is " + \
1894 "working properly"
1895 main.case( description )
1896
1897 main.step( "Bring link between s3 and s28 back up" )
1898 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
1899 main.log.info( "Waiting " + str( linkSleep ) +
1900 " seconds for link up to be discovered" )
1901 time.sleep( linkSleep )
1902 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
1903 onpass="Link up successful",
1904 onfail="Failed to bring link up" )
1905 # TODO do some sort of check here
1906
1907 def CASE11( self, main ):
1908 """
1909 Switch Down
1910 """
1911 # NOTE: You should probably run a topology check after this
1912 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001913 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001914 assert main, "main not defined"
1915 assert utilities.assert_equals, "utilities.assert_equals not defined"
1916
1917 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1918
1919 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallf37d44d2017-05-24 10:37:30 -07001920 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall85794ff2015-07-08 14:12:30 -07001921 main.case( description )
1922 switch = main.params[ 'kill' ][ 'switch' ]
1923 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1924
1925 # TODO: Make this switch parameterizable
1926 main.step( "Kill " + switch )
1927 main.log.info( "Deleting " + switch )
1928 main.Mininet1.delSwitch( switch )
1929 main.log.info( "Waiting " + str( switchSleep ) +
1930 " seconds for switch down to be discovered" )
1931 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07001932 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall85794ff2015-07-08 14:12:30 -07001933 # Peek at the deleted switch
1934 main.log.warn( str( device ) )
1935 result = main.FALSE
1936 if device and device[ 'available' ] is False:
1937 result = main.TRUE
1938 utilities.assert_equals( expect=main.TRUE, actual=result,
1939 onpass="Kill switch successful",
1940 onfail="Failed to kill switch?" )
1941
1942 def CASE12( self, main ):
1943 """
1944 Switch Up
1945 """
1946 # NOTE: You should probably run a topology check after this
1947 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001948 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001949 assert main, "main not defined"
1950 assert utilities.assert_equals, "utilities.assert_equals not defined"
1951
1952 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1953 switch = main.params[ 'kill' ][ 'switch' ]
1954 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1955 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallf37d44d2017-05-24 10:37:30 -07001956 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall85794ff2015-07-08 14:12:30 -07001957 description = "Adding a switch to ensure it is discovered correctly"
1958 main.case( description )
1959
1960 main.step( "Add back " + switch )
1961 main.Mininet1.addSwitch( switch, dpid=switchDPID )
1962 for peer in links:
1963 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07001964 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall85794ff2015-07-08 14:12:30 -07001965 main.Mininet1.assignSwController( sw=switch, ip=ipList )
1966 main.log.info( "Waiting " + str( switchSleep ) +
1967 " seconds for switch up to be discovered" )
1968 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07001969 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall85794ff2015-07-08 14:12:30 -07001970 # Peek at the deleted switch
1971 main.log.warn( str( device ) )
1972 result = main.FALSE
1973 if device and device[ 'available' ]:
1974 result = main.TRUE
1975 utilities.assert_equals( expect=main.TRUE, actual=result,
1976 onpass="add switch successful",
1977 onfail="Failed to add switch?" )
1978
1979 def CASE13( self, main ):
1980 """
1981 Clean up
1982 """
1983 import os
1984 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001985 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001986 assert main, "main not defined"
1987 assert utilities.assert_equals, "utilities.assert_equals not defined"
1988 # printing colors to terminal
1989 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
1990 'blue': '\033[94m', 'green': '\033[92m',
1991 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
1992 main.case( "Test Cleanup" )
1993 main.step( "Killing tcpdumps" )
1994 main.Mininet2.stopTcpdump()
1995
1996 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07001997 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall85794ff2015-07-08 14:12:30 -07001998 main.step( "Copying MN pcap and ONOS log files to test station" )
1999 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2000 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002001 # NOTE: MN Pcap file is being saved to logdir.
2002 # We scp this file as MN and TestON aren't necessarily the same vm
2003
2004 # FIXME: To be replaced with a Jenkin's post script
Jon Hall85794ff2015-07-08 14:12:30 -07002005 # TODO: Load these from params
2006 # NOTE: must end in /
2007 logFolder = "/opt/onos/log/"
2008 logFiles = [ "karaf.log", "karaf.log.1" ]
2009 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07002010 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07002011 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002012 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002013 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2014 logFolder + f, dstName )
Jon Hall85794ff2015-07-08 14:12:30 -07002015 # std*.log's
2016 # NOTE: must end in /
2017 logFolder = "/opt/onos/var/"
2018 logFiles = [ "stderr.log", "stdout.log" ]
2019 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07002020 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07002021 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002022 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002023 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2024 logFolder + f, dstName )
2025 else:
2026 main.log.debug( "skipping saving log files" )
Jon Hall85794ff2015-07-08 14:12:30 -07002027
Jon Hall85794ff2015-07-08 14:12:30 -07002028 main.step( "Stopping Mininet" )
2029 mnResult = main.Mininet1.stopNet()
2030 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2031 onpass="Mininet stopped",
2032 onfail="MN cleanup NOT successful" )
2033
2034 main.step( "Checking ONOS Logs for errors" )
Jon Hall96091e62015-09-21 17:34:17 -07002035 for node in main.nodes:
2036 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2037 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall85794ff2015-07-08 14:12:30 -07002038
2039 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07002040 timerLog = open( main.logdir + "/Timers.csv", 'w' )
Jon Hall85794ff2015-07-08 14:12:30 -07002041 # Overwrite with empty line and close
2042 labels = "Gossip Intents, Restart"
2043 data = str( gossipTime ) + ", " + str( main.restartTime )
2044 timerLog.write( labels + "\n" + data )
2045 timerLog.close()
Jon Hallf37d44d2017-05-24 10:37:30 -07002046 except NameError as e:
2047 main.log.exception( e )
Jon Hall85794ff2015-07-08 14:12:30 -07002048
2049 def CASE14( self, main ):
2050 """
2051 start election app on all onos nodes
2052 """
Jon Halle1a3b752015-07-22 13:02:46 -07002053 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002054 assert main, "main not defined"
2055 assert utilities.assert_equals, "utilities.assert_equals not defined"
2056
Jon Hallf37d44d2017-05-24 10:37:30 -07002057 main.case( "Start Leadership Election app" )
Jon Hall85794ff2015-07-08 14:12:30 -07002058 main.step( "Install leadership election app" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002059 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Halla440e872016-03-31 15:15:50 -07002060 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall85794ff2015-07-08 14:12:30 -07002061 utilities.assert_equals(
2062 expect=main.TRUE,
2063 actual=appResult,
2064 onpass="Election app installed",
2065 onfail="Something went wrong with installing Leadership election" )
2066
2067 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002068 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002069 main.CLIs[ i ].electionTestRun()
2070 time.sleep( 5 )
2071 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall25463a82016-04-13 14:03:52 -07002072 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall85794ff2015-07-08 14:12:30 -07002073 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002074 expect=True,
2075 actual=sameResult,
2076 onpass="All nodes see the same leaderboards",
2077 onfail="Inconsistent leaderboards" )
2078
2079 if sameResult:
2080 leader = leaders[ 0 ][ 0 ]
Jon Hallf37d44d2017-05-24 10:37:30 -07002081 if main.nodes[ main.activeNodes[ 0 ] ].ip_address in leader:
Jon Hall25463a82016-04-13 14:03:52 -07002082 correctLeader = True
2083 else:
2084 correctLeader = False
2085 main.step( "First node was elected leader" )
2086 utilities.assert_equals(
2087 expect=True,
2088 actual=correctLeader,
2089 onpass="Correct leader was elected",
2090 onfail="Incorrect leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07002091
2092 def CASE15( self, main ):
2093 """
2094 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002095 15.1 Run election on each node
2096 15.2 Check that each node has the same leaders and candidates
2097 15.3 Find current leader and withdraw
2098 15.4 Check that a new node was elected leader
2099 15.5 Check that that new leader was the candidate of old leader
2100 15.6 Run for election on old leader
2101 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2102 15.8 Make sure that the old leader was added to the candidate list
2103
2104 old and new variable prefixes refer to data from before vs after
2105 withdrawl and later before withdrawl vs after re-election
Jon Hall85794ff2015-07-08 14:12:30 -07002106 """
acsmars71adceb2015-08-31 15:09:26 -07002107 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002108 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002109 assert main, "main not defined"
2110 assert utilities.assert_equals, "utilities.assert_equals not defined"
acsmars71adceb2015-08-31 15:09:26 -07002111 assert main.CLIs, "main.CLIs not defined"
2112 assert main.nodes, "main.nodes not defined"
2113
Jon Hall85794ff2015-07-08 14:12:30 -07002114 description = "Check that Leadership Election is still functional"
2115 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002116 # NOTE: Need to re-run after restarts since being a canidate is not persistant
acsmars2c2fcdd2015-08-25 17:14:13 -07002117
Jon Halla440e872016-03-31 15:15:50 -07002118 oldLeaders = [] # list of lists of each nodes' candidates before
2119 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002120 oldLeader = '' # the old leader from oldLeaders, None if not same
2121 newLeader = '' # the new leaders fron newLoeaders, None if not same
2122 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2123 expectNoLeader = False # True when there is only one leader
2124 if main.numCtrls == 1:
2125 expectNoLeader = True
acsmars2c2fcdd2015-08-25 17:14:13 -07002126
acsmars71adceb2015-08-31 15:09:26 -07002127 main.step( "Run for election on each node" )
2128 electionResult = main.TRUE
2129
Jon Halla440e872016-03-31 15:15:50 -07002130 for i in main.activeNodes: # run test election on each node
Jon Hallf37d44d2017-05-24 10:37:30 -07002131 if main.CLIs[ i ].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002132 electionResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002133 utilities.assert_equals(
2134 expect=main.TRUE,
2135 actual=electionResult,
2136 onpass="All nodes successfully ran for leadership",
2137 onfail="At least one node failed to run for leadership" )
2138
acsmars3a72bde2015-09-02 14:16:22 -07002139 if electionResult == main.FALSE:
2140 main.log.error(
2141 "Skipping Test Case because Election Test App isn't loaded" )
2142 main.skipCase()
2143
acsmars71adceb2015-08-31 15:09:26 -07002144 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002145 failMessage = "Nodes have different leaderboards"
Jon Hallf37d44d2017-05-24 10:37:30 -07002146 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002147 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002148 if sameResult:
2149 oldLeader = oldLeaders[ 0 ][ 0 ]
2150 main.log.warn( oldLeader )
Jon Hall85794ff2015-07-08 14:12:30 -07002151 else:
Jon Halla440e872016-03-31 15:15:50 -07002152 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002153 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002154 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002155 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002156 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002157 onfail=failMessage )
2158
2159 main.step( "Find current leader and withdraw" )
2160 withdrawResult = main.TRUE
2161 # do some sanity checking on leader before using it
2162 if oldLeader is None:
2163 main.log.error( "Leadership isn't consistent." )
2164 withdrawResult = main.FALSE
2165 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002166 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002167 if oldLeader == main.nodes[ i ].ip_address:
2168 oldLeaderCLI = main.CLIs[ i ]
2169 break
2170 else: # FOR/ELSE statement
2171 main.log.error( "Leader election, could not find current leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07002172 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002173 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall85794ff2015-07-08 14:12:30 -07002174 utilities.assert_equals(
2175 expect=main.TRUE,
2176 actual=withdrawResult,
2177 onpass="Node was withdrawn from election",
2178 onfail="Node was not withdrawn from election" )
2179
acsmars71adceb2015-08-31 15:09:26 -07002180 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002181 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07002182 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002183 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002184 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002185 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002186 if newLeaders[ 0 ][ 0 ] == 'none':
2187 main.log.error( "No leader was elected on at least 1 node" )
2188 if not expectNoLeader:
2189 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002190 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002191
2192 # Check that the new leader is not the older leader, which was withdrawn
2193 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002194 newLeaderResult = False
2195 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07002196 " as the current leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07002197 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002198 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002199 actual=newLeaderResult,
2200 onpass="Leadership election passed",
2201 onfail="Something went wrong with Leadership election" )
Jon Hall85794ff2015-07-08 14:12:30 -07002202
Jon Halla440e872016-03-31 15:15:50 -07002203 main.step( "Check that that new leader was the candidate of old leader" )
2204 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07002205 correctCandidateResult = main.TRUE
2206 if expectNoLeader:
2207 if newLeader == 'none':
2208 main.log.info( "No leader expected. None found. Pass" )
2209 correctCandidateResult = main.TRUE
2210 else:
2211 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2212 correctCandidateResult = main.FALSE
Jon Hallf37d44d2017-05-24 10:37:30 -07002213 elif len( oldLeaders[ 0 ] ) >= 3:
Jon Halla440e872016-03-31 15:15:50 -07002214 if newLeader == oldLeaders[ 0 ][ 2 ]:
2215 # correct leader was elected
2216 correctCandidateResult = main.TRUE
2217 else:
2218 correctCandidateResult = main.FALSE
2219 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
2220 newLeader, oldLeaders[ 0 ][ 2 ] ) )
2221 else:
2222 main.log.warn( "Could not determine who should be the correct leader" )
2223 main.log.debug( oldLeaders[ 0 ] )
acsmars71adceb2015-08-31 15:09:26 -07002224 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002225 utilities.assert_equals(
2226 expect=main.TRUE,
2227 actual=correctCandidateResult,
2228 onpass="Correct Candidate Elected",
2229 onfail="Incorrect Candidate Elected" )
2230
Jon Hall85794ff2015-07-08 14:12:30 -07002231 main.step( "Run for election on old leader( just so everyone " +
2232 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002233 if oldLeaderCLI is not None:
2234 runResult = oldLeaderCLI.electionTestRun()
Jon Hall85794ff2015-07-08 14:12:30 -07002235 else:
acsmars71adceb2015-08-31 15:09:26 -07002236 main.log.error( "No old leader to re-elect" )
Jon Hall85794ff2015-07-08 14:12:30 -07002237 runResult = main.FALSE
2238 utilities.assert_equals(
2239 expect=main.TRUE,
2240 actual=runResult,
2241 onpass="App re-ran for election",
2242 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07002243
acsmars71adceb2015-08-31 15:09:26 -07002244 main.step(
2245 "Check that oldLeader is a candidate, and leader if only 1 node" )
2246 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07002247 # Get new leaders and candidates
2248 reRunLeaders = []
Jon Hall7a6ebfd2017-03-13 10:58:58 -07002249 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07002250 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07002251
2252 # Check that the re-elected node is last on the candidate List
Jon Hallf37d44d2017-05-24 10:37:30 -07002253 if not reRunLeaders[ 0 ]:
Jon Hall3a7843a2016-04-12 03:01:09 -07002254 positionResult = main.FALSE
2255 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07002256 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader ),
Jon Halla440e872016-03-31 15:15:50 -07002257 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07002258 positionResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07002259 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002260 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002261 actual=positionResult,
Jon Hall85794ff2015-07-08 14:12:30 -07002262 onpass="Old leader successfully re-ran for election",
2263 onfail="Something went wrong with Leadership election after " +
2264 "the old leader re-ran for election" )
2265
2266 def CASE16( self, main ):
2267 """
2268 Install Distributed Primitives app
2269 """
Jon Halla440e872016-03-31 15:15:50 -07002270 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002271 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002272 assert main, "main not defined"
2273 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002274 assert main.CLIs, "main.CLIs not defined"
2275 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002276
2277 # Variables for the distributed primitives tests
Jon Hall7a6ebfd2017-03-13 10:58:58 -07002278 main.pCounterName = "TestON-Partitions"
2279 main.pCounterValue = 0
Jon Hallf37d44d2017-05-24 10:37:30 -07002280 main.onosSet = set( [] )
Jon Hall7a6ebfd2017-03-13 10:58:58 -07002281 main.onosSetName = "TestON-set"
Jon Hall85794ff2015-07-08 14:12:30 -07002282
2283 description = "Install Primitives app"
2284 main.case( description )
2285 main.step( "Install Primitives app" )
2286 appName = "org.onosproject.distributedprimitives"
Jon Hallf37d44d2017-05-24 10:37:30 -07002287 node = main.activeNodes[ 0 ]
2288 appResults = main.CLIs[ node ].activateApp( appName )
Jon Hall85794ff2015-07-08 14:12:30 -07002289 utilities.assert_equals( expect=main.TRUE,
2290 actual=appResults,
2291 onpass="Primitives app activated",
2292 onfail="Primitives app not activated" )
Jon Hall7a6ebfd2017-03-13 10:58:58 -07002293 # TODO check on all nodes instead of sleeping
Jon Halla440e872016-03-31 15:15:50 -07002294 time.sleep( 5 ) # To allow all nodes to activate
Jon Hall85794ff2015-07-08 14:12:30 -07002295
2296 def CASE17( self, main ):
2297 """
2298 Check for basic functionality with distributed primitives
2299 """
Jon Hall7a6ebfd2017-03-13 10:58:58 -07002300 main.HA.CASE17( main )