blob: 3047df29aaed971e3f69e8252800484e581f5932 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if ONOS can handle
3 all of it's nodes restarting
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
Jon Hall5cf14d52015-07-16 12:15:19 -070025class HAclusterRestart:
26
27 def __init__( self ):
28 self.default = ''
29
30 def CASE1( self, main ):
31 """
32 CASE1 is to compile ONOS and push it to the test machines
33
34 Startup sequence:
35 cell <name>
36 onos-verify-cell
37 NOTE: temporary - onos-remove-raft-logs
38 onos-uninstall
39 start mininet
40 git pull
41 mvn clean install
42 onos-package
43 onos-install -f
44 onos-wait-for-start
45 start cli sessions
46 start tcpdump
47 """
Jon Halle1a3b752015-07-22 13:02:46 -070048 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080049 import time
Jon Halla440e872016-03-31 15:15:50 -070050 import json
Jon Hall5cf14d52015-07-16 12:15:19 -070051 main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
52 "initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070057
58 # load some variables from the params file
59 PULLCODE = False
60 if main.params[ 'Git' ] == 'True':
61 PULLCODE = True
62 gitBranch = main.params[ 'branch' ]
63 cellName = main.params[ 'ENV' ][ 'cellName' ]
64
Jon Halle1a3b752015-07-22 13:02:46 -070065 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070066 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070067 if main.ONOSbench.maxNodes < main.numCtrls:
68 main.numCtrls = int( main.ONOSbench.maxNodes )
69 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070070 global ONOS1Port
71 global ONOS2Port
72 global ONOS3Port
73 global ONOS4Port
74 global ONOS5Port
75 global ONOS6Port
76 global ONOS7Port
77 # These are for csv plotting in jenkins
78 global labels
79 global data
80 labels = []
81 data = []
82
83 # FIXME: just get controller port from params?
84 # TODO: do we really need all these?
85 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
86 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
87 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
88 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
89 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
90 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
91 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
92
Jon Halle1a3b752015-07-22 13:02:46 -070093 try:
Jon Hall53c5e662016-04-13 16:06:56 -070094 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070095 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070096 except Exception as e:
97 main.log.exception( e )
98 main.cleanup()
99 main.exit()
100
101 main.CLIs = []
102 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700103 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700104 for i in range( 1, main.numCtrls + 1 ):
105 try:
106 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
107 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
108 ipList.append( main.nodes[ -1 ].ip_address )
109 except AttributeError:
110 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700111
112 main.step( "Create cell file" )
113 cellAppString = main.params[ 'ENV' ][ 'appString' ]
114 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
115 main.Mininet1.ip_address,
Devin Limdc78e202017-06-09 18:30:07 -0700116 cellAppString, ipList, main.ONOScli1.karafUser )
Jon Hall5cf14d52015-07-16 12:15:19 -0700117 main.step( "Applying cell variable to environment" )
118 cellResult = main.ONOSbench.setCell( cellName )
119 verifyResult = main.ONOSbench.verifyCell()
120
121 # FIXME:this is short term fix
122 main.log.info( "Removing raft logs" )
123 main.ONOSbench.onosRemoveRaftLogs()
124
125 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700126 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700127 main.ONOSbench.onosUninstall( node.ip_address )
128
129 # Make sure ONOS is DEAD
130 main.log.info( "Killing any ONOS processes" )
131 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700132 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700133 killed = main.ONOSbench.onosKill( node.ip_address )
134 killResults = killResults and killed
135
Jon Hall5cf14d52015-07-16 12:15:19 -0700136 gitPullResult = main.TRUE
137
138 main.step( "Starting Mininet" )
139 # scp topo file to mininet
140 # TODO: move to params?
141 topoName = "obelisk.py"
142 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700143 main.ONOSbench.scp( main.Mininet1,
144 filePath + topoName,
145 main.Mininet1.home,
146 direction="to" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700147 mnResult = main.Mininet1.startNet()
Jon Hall5cf14d52015-07-16 12:15:19 -0700148 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
149 onpass="Mininet Started",
150 onfail="Error starting Mininet" )
151
152 main.step( "Git checkout and pull " + gitBranch )
153 if PULLCODE:
154 main.ONOSbench.gitCheckout( gitBranch )
155 gitPullResult = main.ONOSbench.gitPull()
156 # values of 1 or 3 are good
157 utilities.assert_lesser( expect=0, actual=gitPullResult,
158 onpass="Git pull successful",
159 onfail="Git pull failed" )
160 main.ONOSbench.getVersion( report=True )
161
Jon Hall5cf14d52015-07-16 12:15:19 -0700162 # GRAPHS
163 # NOTE: important params here:
164 # job = name of Jenkins job
165 # Plot Name = Plot-HA, only can be used if multiple plots
166 # index = The number of the graph under plot name
167 job = "HAclusterRestart"
168 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700169 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700170 graphs = '<ac:structured-macro ac:name="html">\n'
171 graphs += '<ac:plain-text-body><![CDATA[\n'
172 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800173 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700174 '&width=500&height=300"' +\
175 'noborder="0" width="500" height="300" scrolling="yes" ' +\
176 'seamless="seamless"></iframe>\n'
177 graphs += ']]></ac:plain-text-body>\n'
178 graphs += '</ac:structured-macro>\n'
Jon Hallf37d44d2017-05-24 10:37:30 -0700179 main.log.wiki( graphs )
Jon Hall5cf14d52015-07-16 12:15:19 -0700180
181 main.step( "Creating ONOS package" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700182 packageResult = main.ONOSbench.buckBuild()
Jon Hall5cf14d52015-07-16 12:15:19 -0700183 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
184 onpass="ONOS package successful",
185 onfail="ONOS package failed" )
186
187 main.step( "Installing ONOS package" )
188 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700189 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700190 tmpResult = main.ONOSbench.onosInstall( options="-f",
191 node=node.ip_address )
192 onosInstallResult = onosInstallResult and tmpResult
193 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
194 onpass="ONOS install successful",
195 onfail="ONOS install failed" )
196
You Wangf5de25b2017-01-06 15:13:01 -0800197 main.step( "Set up ONOS secure SSH" )
198 secureSshResult = main.TRUE
199 for node in main.nodes:
200 secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=node.ip_address )
201 utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
202 onpass="Test step PASS",
203 onfail="Test step FAIL" )
204
Jon Hall5cf14d52015-07-16 12:15:19 -0700205 main.step( "Checking if ONOS is up yet" )
206 for i in range( 2 ):
207 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700208 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700209 started = main.ONOSbench.isup( node.ip_address )
210 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800211 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700212 onosIsupResult = onosIsupResult and started
213 if onosIsupResult == main.TRUE:
214 break
215 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
216 onpass="ONOS startup successful",
217 onfail="ONOS startup failed" )
218
Jon Hall6509dbf2016-06-21 17:01:17 -0700219 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700220 cliResults = main.TRUE
221 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700222 for i in range( main.numCtrls ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700223 t = main.Thread( target=main.CLIs[ i ].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700224 name="startOnosCli-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -0700225 args=[ main.nodes[ i ].ip_address ] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700226 threads.append( t )
227 t.start()
228
229 for t in threads:
230 t.join()
231 cliResults = cliResults and t.result
232 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
233 onpass="ONOS cli startup successful",
234 onfail="ONOS cli startup failed" )
235
Jon Halla440e872016-03-31 15:15:50 -0700236 # Create a list of active nodes for use when some nodes are stopped
237 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
238
Jon Hall5cf14d52015-07-16 12:15:19 -0700239 if main.params[ 'tcpdump' ].lower() == "true":
240 main.step( "Start Packet Capture MN" )
241 main.Mininet2.startTcpdump(
242 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
243 + "-MN.pcap",
244 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
245 port=main.params[ 'MNtcpdump' ][ 'port' ] )
246
Jon Halla440e872016-03-31 15:15:50 -0700247 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700248 nodeResults = utilities.retry( main.HA.nodesCheck,
249 False,
Jon Hallf37d44d2017-05-24 10:37:30 -0700250 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -0700251 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700252
Jon Hall41d39f12016-04-11 22:54:35 -0700253 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700254 onpass="Nodes check successful",
255 onfail="Nodes check NOT successful" )
256
257 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700258 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700259 cli = main.CLIs[ i ]
Jon Halla440e872016-03-31 15:15:50 -0700260 main.log.debug( "{} components not ACTIVE: \n{}".format(
261 cli.name,
262 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700263 main.log.error( "Failed to start ONOS, stopping test" )
264 main.cleanup()
265 main.exit()
266
Jon Hall172b7ba2016-04-07 18:12:20 -0700267 main.step( "Activate apps defined in the params file" )
268 # get data from the params
269 apps = main.params.get( 'apps' )
270 if apps:
Jon Hallf37d44d2017-05-24 10:37:30 -0700271 apps = apps.split( ',' )
Jon Hall172b7ba2016-04-07 18:12:20 -0700272 main.log.warn( apps )
273 activateResult = True
274 for app in apps:
275 main.CLIs[ 0 ].app( app, "Activate" )
276 # TODO: check this worked
277 time.sleep( 10 ) # wait for apps to activate
278 for app in apps:
279 state = main.CLIs[ 0 ].appStatus( app )
280 if state == "ACTIVE":
Jon Hall937bc812017-01-31 16:44:10 -0800281 activateResult = activateResult and True
Jon Hall172b7ba2016-04-07 18:12:20 -0700282 else:
283 main.log.error( "{} is in {} state".format( app, state ) )
Jon Hall937bc812017-01-31 16:44:10 -0800284 activateResult = False
Jon Hall172b7ba2016-04-07 18:12:20 -0700285 utilities.assert_equals( expect=True,
286 actual=activateResult,
287 onpass="Successfully activated apps",
288 onfail="Failed to activate apps" )
289 else:
290 main.log.warn( "No apps were specified to be loaded after startup" )
291
292 main.step( "Set ONOS configurations" )
293 config = main.params.get( 'ONOS_Configuration' )
294 if config:
295 main.log.debug( config )
296 checkResult = main.TRUE
297 for component in config:
Jon Hallf37d44d2017-05-24 10:37:30 -0700298 for setting in config[ component ]:
299 value = config[ component ][ setting ]
Jon Hall172b7ba2016-04-07 18:12:20 -0700300 check = main.CLIs[ 0 ].setCfg( component, setting, value )
301 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
302 checkResult = check and checkResult
303 utilities.assert_equals( expect=main.TRUE,
304 actual=checkResult,
305 onpass="Successfully set config",
306 onfail="Failed to set config" )
307 else:
308 main.log.warn( "No configurations were specified to be changed after startup" )
309
Jon Hall9d2dcad2016-04-08 10:15:20 -0700310 main.step( "App Ids check" )
311 appCheck = main.TRUE
312 threads = []
313 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700314 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall9d2dcad2016-04-08 10:15:20 -0700315 name="appToIDCheck-" + str( i ),
316 args=[] )
317 threads.append( t )
318 t.start()
319
320 for t in threads:
321 t.join()
322 appCheck = appCheck and t.result
323 if appCheck != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700324 node = main.activeNodes[ 0 ]
325 main.log.warn( main.CLIs[ node ].apps() )
326 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700327 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
328 onpass="App Ids seem to be correct",
329 onfail="Something is wrong with app Ids" )
330
Jon Hall5cf14d52015-07-16 12:15:19 -0700331 def CASE2( self, main ):
332 """
333 Assign devices to controllers
334 """
335 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700336 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700337 assert main, "main not defined"
338 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700339 assert main.CLIs, "main.CLIs not defined"
340 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700341 assert ONOS1Port, "ONOS1Port not defined"
342 assert ONOS2Port, "ONOS2Port not defined"
343 assert ONOS3Port, "ONOS3Port not defined"
344 assert ONOS4Port, "ONOS4Port not defined"
345 assert ONOS5Port, "ONOS5Port not defined"
346 assert ONOS6Port, "ONOS6Port not defined"
347 assert ONOS7Port, "ONOS7Port not defined"
348
349 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700350 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700351 "and check that an ONOS node becomes the " +\
352 "master of the device."
353 main.step( "Assign switches to controllers" )
354
355 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700356 for i in range( main.numCtrls ):
357 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700358 swList = []
359 for i in range( 1, 29 ):
360 swList.append( "s" + str( i ) )
361 main.Mininet1.assignSwController( sw=swList, ip=ipList )
362
363 mastershipCheck = main.TRUE
364 for i in range( 1, 29 ):
365 response = main.Mininet1.getSwController( "s" + str( i ) )
366 try:
367 main.log.info( str( response ) )
368 except Exception:
369 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700370 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700371 if re.search( "tcp:" + node.ip_address, response ):
372 mastershipCheck = mastershipCheck and main.TRUE
373 else:
374 main.log.error( "Error, node " + node.ip_address + " is " +
375 "not in the list of controllers s" +
376 str( i ) + " is connecting to." )
377 mastershipCheck = main.FALSE
378 utilities.assert_equals(
379 expect=main.TRUE,
380 actual=mastershipCheck,
381 onpass="Switch mastership assigned correctly",
382 onfail="Switches not assigned correctly to controllers" )
383
384 def CASE21( self, main ):
385 """
386 Assign mastership to controllers
387 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700388 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700389 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700390 assert main, "main not defined"
391 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700392 assert main.CLIs, "main.CLIs not defined"
393 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700394 assert ONOS1Port, "ONOS1Port not defined"
395 assert ONOS2Port, "ONOS2Port not defined"
396 assert ONOS3Port, "ONOS3Port not defined"
397 assert ONOS4Port, "ONOS4Port not defined"
398 assert ONOS5Port, "ONOS5Port not defined"
399 assert ONOS6Port, "ONOS6Port not defined"
400 assert ONOS7Port, "ONOS7Port not defined"
401
402 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700403 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700404 "device. Then manually assign" +\
405 " mastership to specific ONOS nodes using" +\
406 " 'device-role'"
407 main.step( "Assign mastership of switches to specific controllers" )
408 # Manually assign mastership to the controller we want
409 roleCall = main.TRUE
410
Jon Hallf37d44d2017-05-24 10:37:30 -0700411 ipList = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700412 deviceList = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700413 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700414 try:
415 # Assign mastership to specific controllers. This assignment was
416 # determined for a 7 node cluser, but will work with any sized
417 # cluster
418 for i in range( 1, 29 ): # switches 1 through 28
419 # set up correct variables:
420 if i == 1:
421 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700422 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700423 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700424 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700425 c = 1 % main.numCtrls
426 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700427 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700428 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700429 c = 1 % main.numCtrls
430 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700431 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700432 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700433 c = 3 % main.numCtrls
434 ip = main.nodes[ c ].ip_address # ONOS4
Jon Halla440e872016-03-31 15:15:50 -0700435 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700436 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700437 c = 2 % main.numCtrls
438 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700439 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700440 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700441 c = 2 % main.numCtrls
442 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700443 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700444 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700445 c = 5 % main.numCtrls
446 ip = main.nodes[ c ].ip_address # ONOS6
Jon Halla440e872016-03-31 15:15:50 -0700447 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700448 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700449 c = 4 % main.numCtrls
450 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700451 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700452 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700453 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700454 c = 6 % main.numCtrls
455 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700456 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700457 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700458 elif i == 28:
459 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700460 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700461 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700462 else:
463 main.log.error( "You didn't write an else statement for " +
464 "switch s" + str( i ) )
465 roleCall = main.FALSE
466 # Assign switch
467 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
468 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700469 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700470 ipList.append( ip )
471 deviceList.append( deviceId )
472 except ( AttributeError, AssertionError ):
473 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700474 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700475 utilities.assert_equals(
476 expect=main.TRUE,
477 actual=roleCall,
478 onpass="Re-assigned switch mastership to designated controller",
479 onfail="Something wrong with deviceRole calls" )
480
481 main.step( "Check mastership was correctly assigned" )
482 roleCheck = main.TRUE
483 # NOTE: This is due to the fact that device mastership change is not
484 # atomic and is actually a multi step process
485 time.sleep( 5 )
486 for i in range( len( ipList ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700487 ip = ipList[ i ]
488 deviceId = deviceList[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700489 # Check assignment
Jon Halla440e872016-03-31 15:15:50 -0700490 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700491 if ip in master:
492 roleCheck = roleCheck and main.TRUE
493 else:
494 roleCheck = roleCheck and main.FALSE
495 main.log.error( "Error, controller " + ip + " is not" +
496 " master " + "of device " +
497 str( deviceId ) + ". Master is " +
498 repr( master ) + "." )
499 utilities.assert_equals(
500 expect=main.TRUE,
501 actual=roleCheck,
502 onpass="Switches were successfully reassigned to designated " +
503 "controller",
504 onfail="Switches were not successfully reassigned" )
505
506 def CASE3( self, main ):
507 """
508 Assign intents
509 """
510 import time
511 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700512 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700513 assert main, "main not defined"
514 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700515 assert main.CLIs, "main.CLIs not defined"
516 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700517 try:
518 labels
519 except NameError:
520 main.log.error( "labels not defined, setting to []" )
521 labels = []
522 try:
523 data
524 except NameError:
525 main.log.error( "data not defined, setting to []" )
526 data = []
527 # NOTE: we must reinstall intents until we have a persistant intent
528 # datastore!
529 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700530 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700531 "assign predetermined host-to-host intents." +\
532 " After installation, check that the intent" +\
533 " is distributed to all nodes and the state" +\
534 " is INSTALLED"
535
536 # install onos-app-fwd
537 main.step( "Install reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700538 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Halla440e872016-03-31 15:15:50 -0700539 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700540 utilities.assert_equals( expect=main.TRUE, actual=installResults,
541 onpass="Install fwd successful",
542 onfail="Install fwd failed" )
543
544 main.step( "Check app ids" )
545 appCheck = main.TRUE
546 threads = []
Jon Halla440e872016-03-31 15:15:50 -0700547 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700548 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700549 name="appToIDCheck-" + str( i ),
550 args=[] )
551 threads.append( t )
552 t.start()
553
554 for t in threads:
555 t.join()
556 appCheck = appCheck and t.result
557 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700558 main.log.warn( onosCli.apps() )
559 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700560 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
561 onpass="App Ids seem to be correct",
562 onfail="Something is wrong with app Ids" )
563
564 main.step( "Discovering Hosts( Via pingall for now )" )
565 # FIXME: Once we have a host discovery mechanism, use that instead
566 # REACTIVE FWD test
567 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700568 passMsg = "Reactive Pingall test passed"
569 time1 = time.time()
570 pingResult = main.Mininet1.pingall()
571 time2 = time.time()
572 if not pingResult:
Jon Hallf37d44d2017-05-24 10:37:30 -0700573 main.log.warn( "First pingall failed. Trying again..." )
Jon Hall5cf14d52015-07-16 12:15:19 -0700574 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700575 passMsg += " on the second try"
576 utilities.assert_equals(
577 expect=main.TRUE,
578 actual=pingResult,
Jon Hallf37d44d2017-05-24 10:37:30 -0700579 onpass=passMsg,
Jon Hall96091e62015-09-21 17:34:17 -0700580 onfail="Reactive Pingall failed, " +
581 "one or more ping pairs failed" )
582 main.log.info( "Time for pingall: %2f seconds" %
583 ( time2 - time1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700584 # timeout for fwd flows
585 time.sleep( 11 )
586 # uninstall onos-app-fwd
587 main.step( "Uninstall reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700588 node = main.activeNodes[ 0 ]
589 uninstallResult = main.CLIs[ node ].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700590 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
591 onpass="Uninstall fwd successful",
592 onfail="Uninstall fwd failed" )
593
594 main.step( "Check app ids" )
595 threads = []
596 appCheck2 = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700597 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700598 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700599 name="appToIDCheck-" + str( i ),
600 args=[] )
601 threads.append( t )
602 t.start()
603
604 for t in threads:
605 t.join()
606 appCheck2 = appCheck2 and t.result
607 if appCheck2 != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700608 node = main.activeNodes[ 0 ]
609 main.log.warn( main.CLIs[ node ].apps() )
610 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700611 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
612 onpass="App Ids seem to be correct",
613 onfail="Something is wrong with app Ids" )
614
615 main.step( "Add host intents via cli" )
616 intentIds = []
Jon Hall6e709752016-02-01 13:38:46 -0800617 # TODO: move the host numbers to params
618 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700619 intentAddResult = True
620 hostResult = main.TRUE
621 for i in range( 8, 18 ):
622 main.log.info( "Adding host intent between h" + str( i ) +
623 " and h" + str( i + 10 ) )
624 host1 = "00:00:00:00:00:" + \
625 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
626 host2 = "00:00:00:00:00:" + \
627 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
628 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700629 host1Dict = onosCli.getHost( host1 )
630 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700631 host1Id = None
632 host2Id = None
633 if host1Dict and host2Dict:
634 host1Id = host1Dict.get( 'id', None )
635 host2Id = host2Dict.get( 'id', None )
636 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700637 nodeNum = ( i % len( main.activeNodes ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700638 node = main.activeNodes[ nodeNum ]
639 tmpId = main.CLIs[ node ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700640 if tmpId:
641 main.log.info( "Added intent with id: " + tmpId )
642 intentIds.append( tmpId )
643 else:
644 main.log.error( "addHostIntent returned: " +
645 repr( tmpId ) )
646 else:
647 main.log.error( "Error, getHost() failed for h" + str( i ) +
648 " and/or h" + str( i + 10 ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700649 node = main.activeNodes[ 0 ]
650 hosts = main.CLIs[ node ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700651 main.log.warn( "Hosts output: " )
652 try:
653 main.log.warn( json.dumps( json.loads( hosts ),
654 sort_keys=True,
655 indent=4,
656 separators=( ',', ': ' ) ) )
657 except ( ValueError, TypeError ):
658 main.log.warn( repr( hosts ) )
659 hostResult = main.FALSE
660 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
661 onpass="Found a host id for each host",
662 onfail="Error looking up host ids" )
663
664 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700665 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700666 main.log.info( "Submitted intents: " + str( intentIds ) )
667 main.log.info( "Intents in ONOS: " + str( onosIds ) )
668 for intent in intentIds:
669 if intent in onosIds:
670 pass # intent submitted is in onos
671 else:
672 intentAddResult = False
673 if intentAddResult:
674 intentStop = time.time()
675 else:
676 intentStop = None
677 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700678 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700679 intentStates = []
680 installedCheck = True
681 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
682 count = 0
683 try:
684 for intent in json.loads( intents ):
685 state = intent.get( 'state', None )
686 if "INSTALLED" not in state:
687 installedCheck = False
688 intentId = intent.get( 'id', None )
689 intentStates.append( ( intentId, state ) )
690 except ( ValueError, TypeError ):
691 main.log.exception( "Error parsing intents" )
692 # add submitted intents not in the store
693 tmplist = [ i for i, s in intentStates ]
694 missingIntents = False
695 for i in intentIds:
696 if i not in tmplist:
697 intentStates.append( ( i, " - " ) )
698 missingIntents = True
699 intentStates.sort()
700 for i, s in intentStates:
701 count += 1
702 main.log.info( "%-6s%-15s%-15s" %
703 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700704 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700705 try:
706 missing = False
707 if leaders:
708 parsedLeaders = json.loads( leaders )
709 main.log.warn( json.dumps( parsedLeaders,
710 sort_keys=True,
711 indent=4,
712 separators=( ',', ': ' ) ) )
713 # check for all intent partitions
714 topics = []
715 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700716 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700717 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700718 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700719 for topic in topics:
720 if topic not in ONOStopics:
721 main.log.error( "Error: " + topic +
722 " not in leaders" )
723 missing = True
724 else:
725 main.log.error( "leaders() returned None" )
726 except ( ValueError, TypeError ):
727 main.log.exception( "Error parsing leaders" )
728 main.log.error( repr( leaders ) )
729 # Check all nodes
730 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700731 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700732 response = main.CLIs[ i ].leaders( jsonFormat=False )
733 main.log.warn( str( main.CLIs[ i ].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700734 str( response ) )
735
Jon Halla440e872016-03-31 15:15:50 -0700736 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700737 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700738 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -0700739 parsedPartitions = json.loads( partitions )
740 main.log.warn( json.dumps( parsedPartitions,
741 sort_keys=True,
742 indent=4,
743 separators=( ',', ': ' ) ) )
744 # TODO check for a leader in all paritions
745 # TODO check for consistency among nodes
746 else:
747 main.log.error( "partitions() returned None" )
748 except ( ValueError, TypeError ):
749 main.log.exception( "Error parsing partitions" )
750 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700751 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700752 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700753 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -0700754 parsedPending = json.loads( pendingMap )
755 main.log.warn( json.dumps( parsedPending,
756 sort_keys=True,
757 indent=4,
758 separators=( ',', ': ' ) ) )
759 # TODO check something here?
760 else:
761 main.log.error( "pendingMap() returned None" )
762 except ( ValueError, TypeError ):
763 main.log.exception( "Error parsing pending map" )
764 main.log.error( repr( pendingMap ) )
765
766 intentAddResult = bool( intentAddResult and not missingIntents and
767 installedCheck )
768 if not intentAddResult:
769 main.log.error( "Error in pushing host intents to ONOS" )
770
771 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700772 for j in range( 100 ):
Jon Hall5cf14d52015-07-16 12:15:19 -0700773 correct = True
774 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700775 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700776 onosIds = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700777 ids = main.CLIs[ i ].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700778 onosIds.append( ids )
Jon Hallf37d44d2017-05-24 10:37:30 -0700779 main.log.debug( "Intents in " + main.CLIs[ i ].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700780 str( sorted( onosIds ) ) )
781 if sorted( ids ) != sorted( intentIds ):
782 main.log.warn( "Set of intent IDs doesn't match" )
783 correct = False
784 break
785 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700786 intents = json.loads( main.CLIs[ i ].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700787 for intent in intents:
788 if intent[ 'state' ] != "INSTALLED":
789 main.log.warn( "Intent " + intent[ 'id' ] +
790 " is " + intent[ 'state' ] )
791 correct = False
792 break
793 if correct:
794 break
795 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700796 time.sleep( 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700797 if not intentStop:
798 intentStop = time.time()
799 global gossipTime
800 gossipTime = intentStop - intentStart
801 main.log.info( "It took about " + str( gossipTime ) +
802 " seconds for all intents to appear in each node" )
803 append = False
804 title = "Gossip Intents"
805 count = 1
806 while append is False:
807 curTitle = title + str( count )
808 if curTitle not in labels:
809 labels.append( curTitle )
810 data.append( str( gossipTime ) )
811 append = True
812 else:
813 count += 1
Jon Hallf37d44d2017-05-24 10:37:30 -0700814 gossipPeriod = int( main.params[ 'timers' ][ 'gossip' ] )
Jon Halla440e872016-03-31 15:15:50 -0700815 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700816 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700817 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700818 onpass="ECM anti-entropy for intents worked within " +
819 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700820 onfail="Intent ECM anti-entropy took too long. " +
821 "Expected time:{}, Actual time:{}".format( maxGossipTime,
822 gossipTime ) )
823 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700824 intentAddResult = True
825
826 if not intentAddResult or "key" in pendingMap:
827 import time
828 installedCheck = True
829 main.log.info( "Sleeping 60 seconds to see if intents are found" )
830 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700831 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700832 main.log.info( "Submitted intents: " + str( intentIds ) )
833 main.log.info( "Intents in ONOS: " + str( onosIds ) )
834 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700835 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700836 intentStates = []
837 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
838 count = 0
839 try:
840 for intent in json.loads( intents ):
841 # Iter through intents of a node
842 state = intent.get( 'state', None )
843 if "INSTALLED" not in state:
844 installedCheck = False
845 intentId = intent.get( 'id', None )
846 intentStates.append( ( intentId, state ) )
847 except ( ValueError, TypeError ):
848 main.log.exception( "Error parsing intents" )
849 # add submitted intents not in the store
850 tmplist = [ i for i, s in intentStates ]
851 for i in intentIds:
852 if i not in tmplist:
853 intentStates.append( ( i, " - " ) )
854 intentStates.sort()
855 for i, s in intentStates:
856 count += 1
857 main.log.info( "%-6s%-15s%-15s" %
858 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700859 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700860 try:
861 missing = False
862 if leaders:
863 parsedLeaders = json.loads( leaders )
864 main.log.warn( json.dumps( parsedLeaders,
865 sort_keys=True,
866 indent=4,
867 separators=( ',', ': ' ) ) )
868 # check for all intent partitions
869 # check for election
870 topics = []
871 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700872 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700873 # FIXME: this should only be after we start the app
874 topics.append( "org.onosproject.election" )
875 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700876 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700877 for topic in topics:
878 if topic not in ONOStopics:
879 main.log.error( "Error: " + topic +
880 " not in leaders" )
881 missing = True
882 else:
883 main.log.error( "leaders() returned None" )
884 except ( ValueError, TypeError ):
885 main.log.exception( "Error parsing leaders" )
886 main.log.error( repr( leaders ) )
887 # Check all nodes
888 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700889 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700890 node = main.CLIs[ i ]
891 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -0700892 main.log.warn( str( node.name ) + " leaders output: \n" +
893 str( response ) )
894
Jon Halla440e872016-03-31 15:15:50 -0700895 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700896 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700897 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -0700898 parsedPartitions = json.loads( partitions )
899 main.log.warn( json.dumps( parsedPartitions,
900 sort_keys=True,
901 indent=4,
902 separators=( ',', ': ' ) ) )
903 # TODO check for a leader in all paritions
904 # TODO check for consistency among nodes
905 else:
906 main.log.error( "partitions() returned None" )
907 except ( ValueError, TypeError ):
908 main.log.exception( "Error parsing partitions" )
909 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700910 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700911 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700912 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -0700913 parsedPending = json.loads( pendingMap )
914 main.log.warn( json.dumps( parsedPending,
915 sort_keys=True,
916 indent=4,
917 separators=( ',', ': ' ) ) )
918 # TODO check something here?
919 else:
920 main.log.error( "pendingMap() returned None" )
921 except ( ValueError, TypeError ):
922 main.log.exception( "Error parsing pending map" )
923 main.log.error( repr( pendingMap ) )
924
925 def CASE4( self, main ):
926 """
927 Ping across added host intents
928 """
929 import json
930 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700931 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700932 assert main, "main not defined"
933 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700934 assert main.CLIs, "main.CLIs not defined"
935 assert main.nodes, "main.nodes not defined"
Jon Hall6e709752016-02-01 13:38:46 -0800936 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700937 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700938 "functionality and check the state of " +\
939 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700940
Jon Hallf37d44d2017-05-24 10:37:30 -0700941 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700942 main.step( "Check Intent state" )
943 installedCheck = False
944 loopCount = 0
945 while not installedCheck and loopCount < 40:
946 installedCheck = True
947 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700948 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700949 intentStates = []
950 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700951 count = 0
Jon Hall5cf14d52015-07-16 12:15:19 -0700952 # Iter through intents of a node
953 try:
954 for intent in json.loads( intents ):
955 state = intent.get( 'state', None )
956 if "INSTALLED" not in state:
957 installedCheck = False
958 intentId = intent.get( 'id', None )
959 intentStates.append( ( intentId, state ) )
960 except ( ValueError, TypeError ):
961 main.log.exception( "Error parsing intents." )
962 # Print states
963 intentStates.sort()
964 for i, s in intentStates:
965 count += 1
966 main.log.info( "%-6s%-15s%-15s" %
967 ( str( count ), str( i ), str( s ) ) )
968 if not installedCheck:
969 time.sleep( 1 )
970 loopCount += 1
971 utilities.assert_equals( expect=True, actual=installedCheck,
972 onpass="Intents are all INSTALLED",
973 onfail="Intents are not all in " +
974 "INSTALLED state" )
975
Jon Hall9d2dcad2016-04-08 10:15:20 -0700976 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700977 PingResult = main.TRUE
978 for i in range( 8, 18 ):
979 ping = main.Mininet1.pingHost( src="h" + str( i ),
980 target="h" + str( i + 10 ) )
981 PingResult = PingResult and ping
982 if ping == main.FALSE:
983 main.log.warn( "Ping failed between h" + str( i ) +
984 " and h" + str( i + 10 ) )
985 elif ping == main.TRUE:
986 main.log.info( "Ping test passed!" )
987 # Don't set PingResult or you'd override failures
988 if PingResult == main.FALSE:
989 main.log.error(
990 "Intents have not been installed correctly, pings failed." )
991 # TODO: pretty print
992 main.log.warn( "ONOS1 intents: " )
993 try:
994 tmpIntents = onosCli.intents()
995 main.log.warn( json.dumps( json.loads( tmpIntents ),
996 sort_keys=True,
997 indent=4,
998 separators=( ',', ': ' ) ) )
999 except ( ValueError, TypeError ):
1000 main.log.warn( repr( tmpIntents ) )
1001 utilities.assert_equals(
1002 expect=main.TRUE,
1003 actual=PingResult,
1004 onpass="Intents have been installed correctly and pings work",
1005 onfail="Intents have not been installed correctly, pings failed." )
1006
Jon Hall5cf14d52015-07-16 12:15:19 -07001007 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -07001008 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001009 topicCheck = main.TRUE
1010 try:
1011 if leaders:
1012 parsedLeaders = json.loads( leaders )
1013 main.log.warn( json.dumps( parsedLeaders,
1014 sort_keys=True,
1015 indent=4,
1016 separators=( ',', ': ' ) ) )
1017 # check for all intent partitions
1018 # check for election
1019 # TODO: Look at Devices as topics now that it uses this system
1020 topics = []
1021 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001022 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001023 # FIXME: this should only be after we start the app
1024 # FIXME: topics.append( "org.onosproject.election" )
1025 # Print leaders output
1026 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -07001027 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -07001028 for topic in topics:
1029 if topic not in ONOStopics:
1030 main.log.error( "Error: " + topic +
1031 " not in leaders" )
1032 topicCheck = main.FALSE
1033 else:
1034 main.log.error( "leaders() returned None" )
1035 topicCheck = main.FALSE
1036 except ( ValueError, TypeError ):
1037 topicCheck = main.FALSE
1038 main.log.exception( "Error parsing leaders" )
1039 main.log.error( repr( leaders ) )
1040 # TODO: Check for a leader of these topics
1041 # Check all nodes
1042 if topicCheck:
Jon Halla440e872016-03-31 15:15:50 -07001043 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001044 node = main.CLIs[ i ]
1045 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -07001046 main.log.warn( str( node.name ) + " leaders output: \n" +
1047 str( response ) )
1048
1049 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1050 onpass="intent Partitions is in leaders",
1051 onfail="Some topics were lost " )
1052 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -07001053 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001054 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001055 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -07001056 parsedPartitions = json.loads( partitions )
1057 main.log.warn( json.dumps( parsedPartitions,
1058 sort_keys=True,
1059 indent=4,
1060 separators=( ',', ': ' ) ) )
1061 # TODO check for a leader in all paritions
1062 # TODO check for consistency among nodes
1063 else:
1064 main.log.error( "partitions() returned None" )
1065 except ( ValueError, TypeError ):
1066 main.log.exception( "Error parsing partitions" )
1067 main.log.error( repr( partitions ) )
1068 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001069 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001070 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001071 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -07001072 parsedPending = json.loads( pendingMap )
1073 main.log.warn( json.dumps( parsedPending,
1074 sort_keys=True,
1075 indent=4,
1076 separators=( ',', ': ' ) ) )
1077 # TODO check something here?
1078 else:
1079 main.log.error( "pendingMap() returned None" )
1080 except ( ValueError, TypeError ):
1081 main.log.exception( "Error parsing pending map" )
1082 main.log.error( repr( pendingMap ) )
1083
1084 if not installedCheck:
1085 main.log.info( "Waiting 60 seconds to see if the state of " +
1086 "intents change" )
1087 time.sleep( 60 )
1088 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001089 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001090 intentStates = []
1091 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1092 count = 0
1093 # Iter through intents of a node
1094 try:
1095 for intent in json.loads( intents ):
1096 state = intent.get( 'state', None )
1097 if "INSTALLED" not in state:
1098 installedCheck = False
1099 intentId = intent.get( 'id', None )
1100 intentStates.append( ( intentId, state ) )
1101 except ( ValueError, TypeError ):
1102 main.log.exception( "Error parsing intents." )
1103 intentStates.sort()
1104 for i, s in intentStates:
1105 count += 1
1106 main.log.info( "%-6s%-15s%-15s" %
1107 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001108 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001109 try:
1110 missing = False
1111 if leaders:
1112 parsedLeaders = json.loads( leaders )
1113 main.log.warn( json.dumps( parsedLeaders,
1114 sort_keys=True,
1115 indent=4,
1116 separators=( ',', ': ' ) ) )
1117 # check for all intent partitions
1118 # check for election
1119 topics = []
1120 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001121 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001122 # FIXME: this should only be after we start the app
1123 topics.append( "org.onosproject.election" )
1124 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -07001125 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -07001126 for topic in topics:
1127 if topic not in ONOStopics:
1128 main.log.error( "Error: " + topic +
1129 " not in leaders" )
1130 missing = True
1131 else:
1132 main.log.error( "leaders() returned None" )
1133 except ( ValueError, TypeError ):
1134 main.log.exception( "Error parsing leaders" )
1135 main.log.error( repr( leaders ) )
1136 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001137 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001138 node = main.CLIs[ i ]
1139 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -07001140 main.log.warn( str( node.name ) + " leaders output: \n" +
1141 str( response ) )
1142
Jon Halla440e872016-03-31 15:15:50 -07001143 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001144 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001145 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -07001146 parsedPartitions = json.loads( partitions )
1147 main.log.warn( json.dumps( parsedPartitions,
1148 sort_keys=True,
1149 indent=4,
1150 separators=( ',', ': ' ) ) )
1151 # TODO check for a leader in all paritions
1152 # TODO check for consistency among nodes
1153 else:
1154 main.log.error( "partitions() returned None" )
1155 except ( ValueError, TypeError ):
1156 main.log.exception( "Error parsing partitions" )
1157 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001158 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001159 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001160 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -07001161 parsedPending = json.loads( pendingMap )
1162 main.log.warn( json.dumps( parsedPending,
1163 sort_keys=True,
1164 indent=4,
1165 separators=( ',', ': ' ) ) )
1166 # TODO check something here?
1167 else:
1168 main.log.error( "pendingMap() returned None" )
1169 except ( ValueError, TypeError ):
1170 main.log.exception( "Error parsing pending map" )
1171 main.log.error( repr( pendingMap ) )
1172 # Print flowrules
Jon Hallf37d44d2017-05-24 10:37:30 -07001173 node = main.activeNodes[ 0 ]
1174 main.log.debug( main.CLIs[ node ].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001175 main.step( "Wait a minute then ping again" )
1176 # the wait is above
1177 PingResult = main.TRUE
1178 for i in range( 8, 18 ):
1179 ping = main.Mininet1.pingHost( src="h" + str( i ),
1180 target="h" + str( i + 10 ) )
1181 PingResult = PingResult and ping
1182 if ping == main.FALSE:
1183 main.log.warn( "Ping failed between h" + str( i ) +
1184 " and h" + str( i + 10 ) )
1185 elif ping == main.TRUE:
1186 main.log.info( "Ping test passed!" )
1187 # Don't set PingResult or you'd override failures
1188 if PingResult == main.FALSE:
1189 main.log.error(
1190 "Intents have not been installed correctly, pings failed." )
1191 # TODO: pretty print
1192 main.log.warn( "ONOS1 intents: " )
1193 try:
Jon Halla440e872016-03-31 15:15:50 -07001194 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001195 main.log.warn( json.dumps( json.loads( tmpIntents ),
1196 sort_keys=True,
1197 indent=4,
1198 separators=( ',', ': ' ) ) )
1199 except ( ValueError, TypeError ):
1200 main.log.warn( repr( tmpIntents ) )
1201 utilities.assert_equals(
1202 expect=main.TRUE,
1203 actual=PingResult,
1204 onpass="Intents have been installed correctly and pings work",
1205 onfail="Intents have not been installed correctly, pings failed." )
1206
1207 def CASE5( self, main ):
1208 """
1209 Reading state of ONOS
1210 """
1211 import json
1212 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001213 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001214 assert main, "main not defined"
1215 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001216 assert main.CLIs, "main.CLIs not defined"
1217 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001218
1219 main.case( "Setting up and gathering data for current state" )
1220 # The general idea for this test case is to pull the state of
1221 # ( intents,flows, topology,... ) from each ONOS node
1222 # We can then compare them with each other and also with past states
1223
1224 main.step( "Check that each switch has a master" )
1225 global mastershipState
1226 mastershipState = '[]'
1227
1228 # Assert that each device has a master
1229 rolesNotNull = main.TRUE
1230 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001231 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001232 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001233 name="rolesNotNull-" + str( i ),
1234 args=[] )
1235 threads.append( t )
1236 t.start()
1237
1238 for t in threads:
1239 t.join()
1240 rolesNotNull = rolesNotNull and t.result
1241 utilities.assert_equals(
1242 expect=main.TRUE,
1243 actual=rolesNotNull,
1244 onpass="Each device has a master",
1245 onfail="Some devices don't have a master assigned" )
1246
1247 main.step( "Get the Mastership of each switch from each controller" )
1248 ONOSMastership = []
1249 mastershipCheck = main.FALSE
1250 consistentMastership = True
1251 rolesResults = True
1252 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001253 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001254 t = main.Thread( target=main.CLIs[ i ].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001255 name="roles-" + str( i ),
1256 args=[] )
1257 threads.append( t )
1258 t.start()
1259
1260 for t in threads:
1261 t.join()
1262 ONOSMastership.append( t.result )
1263
Jon Halla440e872016-03-31 15:15:50 -07001264 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001265 node = str( main.activeNodes[ i ] + 1 )
1266 if not ONOSMastership[ i ] or "Error" in ONOSMastership[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07001267 main.log.error( "Error in getting ONOS" + node + " roles" )
1268 main.log.warn( "ONOS" + node + " mastership response: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001269 repr( ONOSMastership[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001270 rolesResults = False
1271 utilities.assert_equals(
1272 expect=True,
1273 actual=rolesResults,
1274 onpass="No error in reading roles output",
1275 onfail="Error in reading roles from ONOS" )
1276
1277 main.step( "Check for consistency in roles from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001278 if all( [ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001279 main.log.info(
1280 "Switch roles are consistent across all ONOS nodes" )
1281 else:
1282 consistentMastership = False
1283 utilities.assert_equals(
1284 expect=True,
1285 actual=consistentMastership,
1286 onpass="Switch roles are consistent across all ONOS nodes",
1287 onfail="ONOS nodes have different views of switch roles" )
1288
1289 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001290 for i in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001291 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001292 try:
1293 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001294 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001295 json.dumps(
1296 json.loads( ONOSMastership[ i ] ),
1297 sort_keys=True,
1298 indent=4,
1299 separators=( ',', ': ' ) ) )
1300 except ( ValueError, TypeError ):
1301 main.log.warn( repr( ONOSMastership[ i ] ) )
1302 elif rolesResults and consistentMastership:
1303 mastershipCheck = main.TRUE
1304 mastershipState = ONOSMastership[ 0 ]
1305
1306 main.step( "Get the intents from each controller" )
1307 global intentState
1308 intentState = []
1309 ONOSIntents = []
1310 intentCheck = main.FALSE
1311 consistentIntents = True
1312 intentsResults = True
1313 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001314 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001315 t = main.Thread( target=main.CLIs[ i ].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001316 name="intents-" + str( i ),
1317 args=[],
1318 kwargs={ 'jsonFormat': True } )
1319 threads.append( t )
1320 t.start()
1321
1322 for t in threads:
1323 t.join()
1324 ONOSIntents.append( t.result )
1325
Jon Halla440e872016-03-31 15:15:50 -07001326 for i in range( len( ONOSIntents ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001327 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001328 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Hall6e709752016-02-01 13:38:46 -08001329 main.log.error( "Error in getting ONOS" + node + " intents" )
1330 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001331 repr( ONOSIntents[ i ] ) )
1332 intentsResults = False
1333 utilities.assert_equals(
1334 expect=True,
1335 actual=intentsResults,
1336 onpass="No error in reading intents output",
1337 onfail="Error in reading intents from ONOS" )
1338
1339 main.step( "Check for consistency in Intents from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001340 if all( [ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001341 main.log.info( "Intents are consistent across all ONOS " +
1342 "nodes" )
1343 else:
1344 consistentIntents = False
1345 main.log.error( "Intents not consistent" )
1346 utilities.assert_equals(
1347 expect=True,
1348 actual=consistentIntents,
1349 onpass="Intents are consistent across all ONOS nodes",
1350 onfail="ONOS nodes have different views of intents" )
1351
1352 if intentsResults:
1353 # Try to make it easy to figure out what is happening
1354 #
1355 # Intent ONOS1 ONOS2 ...
1356 # 0x01 INSTALLED INSTALLING
1357 # ... ... ...
1358 # ... ... ...
1359 title = " Id"
Jon Halla440e872016-03-31 15:15:50 -07001360 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001361 title += " " * 10 + "ONOS" + str( n + 1 )
1362 main.log.warn( title )
1363 # get all intent keys in the cluster
1364 keys = []
Jon Halla440e872016-03-31 15:15:50 -07001365 try:
1366 # Get the set of all intent keys
Jon Hall5cf14d52015-07-16 12:15:19 -07001367 for nodeStr in ONOSIntents:
1368 node = json.loads( nodeStr )
1369 for intent in node:
Jon Halla440e872016-03-31 15:15:50 -07001370 keys.append( intent.get( 'id' ) )
1371 keys = set( keys )
1372 # For each intent key, print the state on each node
1373 for key in keys:
1374 row = "%-13s" % key
1375 for nodeStr in ONOSIntents:
1376 node = json.loads( nodeStr )
1377 for intent in node:
1378 if intent.get( 'id', "Error" ) == key:
1379 row += "%-15s" % intent.get( 'state' )
1380 main.log.warn( row )
1381 # End of intent state table
1382 except ValueError as e:
1383 main.log.exception( e )
1384 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001385
1386 if intentsResults and not consistentIntents:
1387 # print the json objects
Jon Hallf37d44d2017-05-24 10:37:30 -07001388 n = str( main.activeNodes[ -1 ] + 1 )
Jon Halla440e872016-03-31 15:15:50 -07001389 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001390 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1391 sort_keys=True,
1392 indent=4,
1393 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001394 for i in range( len( ONOSIntents ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001395 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001396 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07001397 main.log.debug( "ONOS" + node + " intents: " )
Jon Hallf37d44d2017-05-24 10:37:30 -07001398 main.log.debug( json.dumps( json.loads( ONOSIntents[ i ] ),
Jon Hall5cf14d52015-07-16 12:15:19 -07001399 sort_keys=True,
1400 indent=4,
1401 separators=( ',', ': ' ) ) )
1402 else:
Jon Halla440e872016-03-31 15:15:50 -07001403 main.log.debug( "ONOS" + node + " intents match ONOS" +
1404 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001405 elif intentsResults and consistentIntents:
1406 intentCheck = main.TRUE
1407 intentState = ONOSIntents[ 0 ]
1408
1409 main.step( "Get the flows from each controller" )
1410 global flowState
1411 flowState = []
1412 ONOSFlows = []
1413 ONOSFlowsJson = []
1414 flowCheck = main.FALSE
1415 consistentFlows = True
1416 flowsResults = True
1417 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001418 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001419 t = main.Thread( target=main.CLIs[ i ].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001420 name="flows-" + str( i ),
1421 args=[],
1422 kwargs={ 'jsonFormat': True } )
1423 threads.append( t )
1424 t.start()
1425
1426 # NOTE: Flows command can take some time to run
Jon Hallf37d44d2017-05-24 10:37:30 -07001427 time.sleep( 30 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001428 for t in threads:
1429 t.join()
1430 result = t.result
1431 ONOSFlows.append( result )
1432
Jon Halla440e872016-03-31 15:15:50 -07001433 for i in range( len( ONOSFlows ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001434 num = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001435 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1436 main.log.error( "Error in getting ONOS" + num + " flows" )
1437 main.log.warn( "ONOS" + num + " flows response: " +
1438 repr( ONOSFlows[ i ] ) )
1439 flowsResults = False
1440 ONOSFlowsJson.append( None )
1441 else:
1442 try:
1443 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1444 except ( ValueError, TypeError ):
1445 # FIXME: change this to log.error?
1446 main.log.exception( "Error in parsing ONOS" + num +
1447 " response as json." )
1448 main.log.error( repr( ONOSFlows[ i ] ) )
1449 ONOSFlowsJson.append( None )
1450 flowsResults = False
1451 utilities.assert_equals(
1452 expect=True,
1453 actual=flowsResults,
1454 onpass="No error in reading flows output",
1455 onfail="Error in reading flows from ONOS" )
1456
1457 main.step( "Check for consistency in Flows from each controller" )
1458 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1459 if all( tmp ):
1460 main.log.info( "Flow count is consistent across all ONOS nodes" )
1461 else:
1462 consistentFlows = False
1463 utilities.assert_equals(
1464 expect=True,
1465 actual=consistentFlows,
1466 onpass="The flow count is consistent across all ONOS nodes",
1467 onfail="ONOS nodes have different flow counts" )
1468
1469 if flowsResults and not consistentFlows:
Jon Halla440e872016-03-31 15:15:50 -07001470 for i in range( len( ONOSFlows ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001471 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001472 try:
1473 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001474 "ONOS" + node + " flows: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001475 json.dumps( json.loads( ONOSFlows[ i ] ), sort_keys=True,
Jon Hall5cf14d52015-07-16 12:15:19 -07001476 indent=4, separators=( ',', ': ' ) ) )
1477 except ( ValueError, TypeError ):
Jon Halla440e872016-03-31 15:15:50 -07001478 main.log.warn( "ONOS" + node + " flows: " +
1479 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001480 elif flowsResults and consistentFlows:
1481 flowCheck = main.TRUE
1482 flowState = ONOSFlows[ 0 ]
1483
1484 main.step( "Get the OF Table entries" )
1485 global flows
1486 flows = []
1487 for i in range( 1, 29 ):
Jon Halla440e872016-03-31 15:15:50 -07001488 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001489 if flowCheck == main.FALSE:
1490 for table in flows:
1491 main.log.warn( table )
1492 # TODO: Compare switch flow tables with ONOS flow tables
1493
1494 main.step( "Start continuous pings" )
1495 main.Mininet2.pingLong(
1496 src=main.params[ 'PING' ][ 'source1' ],
1497 target=main.params[ 'PING' ][ 'target1' ],
1498 pingTime=500 )
1499 main.Mininet2.pingLong(
1500 src=main.params[ 'PING' ][ 'source2' ],
1501 target=main.params[ 'PING' ][ 'target2' ],
1502 pingTime=500 )
1503 main.Mininet2.pingLong(
1504 src=main.params[ 'PING' ][ 'source3' ],
1505 target=main.params[ 'PING' ][ 'target3' ],
1506 pingTime=500 )
1507 main.Mininet2.pingLong(
1508 src=main.params[ 'PING' ][ 'source4' ],
1509 target=main.params[ 'PING' ][ 'target4' ],
1510 pingTime=500 )
1511 main.Mininet2.pingLong(
1512 src=main.params[ 'PING' ][ 'source5' ],
1513 target=main.params[ 'PING' ][ 'target5' ],
1514 pingTime=500 )
1515 main.Mininet2.pingLong(
1516 src=main.params[ 'PING' ][ 'source6' ],
1517 target=main.params[ 'PING' ][ 'target6' ],
1518 pingTime=500 )
1519 main.Mininet2.pingLong(
1520 src=main.params[ 'PING' ][ 'source7' ],
1521 target=main.params[ 'PING' ][ 'target7' ],
1522 pingTime=500 )
1523 main.Mininet2.pingLong(
1524 src=main.params[ 'PING' ][ 'source8' ],
1525 target=main.params[ 'PING' ][ 'target8' ],
1526 pingTime=500 )
1527 main.Mininet2.pingLong(
1528 src=main.params[ 'PING' ][ 'source9' ],
1529 target=main.params[ 'PING' ][ 'target9' ],
1530 pingTime=500 )
1531 main.Mininet2.pingLong(
1532 src=main.params[ 'PING' ][ 'source10' ],
1533 target=main.params[ 'PING' ][ 'target10' ],
1534 pingTime=500 )
1535
1536 main.step( "Collecting topology information from ONOS" )
1537 devices = []
1538 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001539 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001540 t = main.Thread( target=main.CLIs[ i ].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001541 name="devices-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001542 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001543 threads.append( t )
1544 t.start()
1545
1546 for t in threads:
1547 t.join()
1548 devices.append( t.result )
1549 hosts = []
1550 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001551 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001552 t = main.Thread( target=main.CLIs[ i ].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001553 name="hosts-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001554 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001555 threads.append( t )
1556 t.start()
1557
1558 for t in threads:
1559 t.join()
1560 try:
1561 hosts.append( json.loads( t.result ) )
1562 except ( ValueError, TypeError ):
1563 # FIXME: better handling of this, print which node
1564 # Maybe use thread name?
1565 main.log.exception( "Error parsing json output of hosts" )
Jon Hall3afe4c92015-12-14 19:30:38 -08001566 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001567 hosts.append( None )
1568
1569 ports = []
1570 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001571 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001572 t = main.Thread( target=main.CLIs[ i ].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001573 name="ports-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001574 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001575 threads.append( t )
1576 t.start()
1577
1578 for t in threads:
1579 t.join()
1580 ports.append( t.result )
1581 links = []
1582 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001583 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001584 t = main.Thread( target=main.CLIs[ i ].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001585 name="links-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001586 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001587 threads.append( t )
1588 t.start()
1589
1590 for t in threads:
1591 t.join()
1592 links.append( t.result )
1593 clusters = []
1594 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001595 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001596 t = main.Thread( target=main.CLIs[ i ].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001597 name="clusters-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001598 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001599 threads.append( t )
1600 t.start()
1601
1602 for t in threads:
1603 t.join()
1604 clusters.append( t.result )
1605 # Compare json objects for hosts and dataplane clusters
1606
1607 # hosts
1608 main.step( "Host view is consistent across ONOS nodes" )
1609 consistentHostsResult = main.TRUE
1610 for controller in range( len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001611 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001612 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001613 if hosts[ controller ] == hosts[ 0 ]:
1614 continue
1615 else: # hosts not consistent
1616 main.log.error( "hosts from ONOS" +
1617 controllerStr +
1618 " is inconsistent with ONOS1" )
1619 main.log.warn( repr( hosts[ controller ] ) )
1620 consistentHostsResult = main.FALSE
1621
1622 else:
1623 main.log.error( "Error in getting ONOS hosts from ONOS" +
1624 controllerStr )
1625 consistentHostsResult = main.FALSE
1626 main.log.warn( "ONOS" + controllerStr +
1627 " hosts response: " +
1628 repr( hosts[ controller ] ) )
1629 utilities.assert_equals(
1630 expect=main.TRUE,
1631 actual=consistentHostsResult,
1632 onpass="Hosts view is consistent across all ONOS nodes",
1633 onfail="ONOS nodes have different views of hosts" )
1634
1635 main.step( "Each host has an IP address" )
1636 ipResult = main.TRUE
1637 for controller in range( 0, len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001638 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall3afe4c92015-12-14 19:30:38 -08001639 if hosts[ controller ]:
1640 for host in hosts[ controller ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07001641 if not host.get( 'ipAddresses', [] ):
Jon Hallf3d16e72015-12-16 17:45:08 -08001642 main.log.error( "Error with host ips on controller" +
Jon Hall3afe4c92015-12-14 19:30:38 -08001643 controllerStr + ": " + str( host ) )
1644 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001645 utilities.assert_equals(
1646 expect=main.TRUE,
1647 actual=ipResult,
1648 onpass="The ips of the hosts aren't empty",
1649 onfail="The ip of at least one host is missing" )
1650
1651 # Strongly connected clusters of devices
1652 main.step( "Cluster view is consistent across ONOS nodes" )
1653 consistentClustersResult = main.TRUE
1654 for controller in range( len( clusters ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001655 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001656 if "Error" not in clusters[ controller ]:
1657 if clusters[ controller ] == clusters[ 0 ]:
1658 continue
1659 else: # clusters not consistent
1660 main.log.error( "clusters from ONOS" + controllerStr +
1661 " is inconsistent with ONOS1" )
1662 consistentClustersResult = main.FALSE
1663
1664 else:
1665 main.log.error( "Error in getting dataplane clusters " +
1666 "from ONOS" + controllerStr )
1667 consistentClustersResult = main.FALSE
1668 main.log.warn( "ONOS" + controllerStr +
1669 " clusters response: " +
1670 repr( clusters[ controller ] ) )
1671 utilities.assert_equals(
1672 expect=main.TRUE,
1673 actual=consistentClustersResult,
1674 onpass="Clusters view is consistent across all ONOS nodes",
1675 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001676 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001677 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001678
Jon Hall5cf14d52015-07-16 12:15:19 -07001679 # there should always only be one cluster
1680 main.step( "Cluster view correct across ONOS nodes" )
1681 try:
1682 numClusters = len( json.loads( clusters[ 0 ] ) )
1683 except ( ValueError, TypeError ):
1684 main.log.exception( "Error parsing clusters[0]: " +
1685 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001686 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001687 clusterResults = main.FALSE
1688 if numClusters == 1:
1689 clusterResults = main.TRUE
1690 utilities.assert_equals(
1691 expect=1,
1692 actual=numClusters,
1693 onpass="ONOS shows 1 SCC",
1694 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1695
1696 main.step( "Comparing ONOS topology to MN" )
1697 devicesResults = main.TRUE
1698 linksResults = main.TRUE
1699 hostsResults = main.TRUE
1700 mnSwitches = main.Mininet1.getSwitches()
1701 mnLinks = main.Mininet1.getLinks()
1702 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001703 for controller in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001704 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001705 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001706 "Error" not in devices[ controller ] and\
1707 "Error" not in ports[ controller ]:
1708 currentDevicesResult = main.Mininet1.compareSwitches(
1709 mnSwitches,
1710 json.loads( devices[ controller ] ),
1711 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001712 else:
1713 currentDevicesResult = main.FALSE
1714 utilities.assert_equals( expect=main.TRUE,
1715 actual=currentDevicesResult,
1716 onpass="ONOS" + controllerStr +
1717 " Switches view is correct",
1718 onfail="ONOS" + controllerStr +
1719 " Switches view is incorrect" )
1720 if links[ controller ] and "Error" not in links[ controller ]:
1721 currentLinksResult = main.Mininet1.compareLinks(
1722 mnSwitches, mnLinks,
1723 json.loads( links[ controller ] ) )
1724 else:
1725 currentLinksResult = main.FALSE
1726 utilities.assert_equals( expect=main.TRUE,
1727 actual=currentLinksResult,
1728 onpass="ONOS" + controllerStr +
1729 " links view is correct",
1730 onfail="ONOS" + controllerStr +
1731 " links view is incorrect" )
1732
Jon Hall657cdf62015-12-17 14:40:51 -08001733 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001734 currentHostsResult = main.Mininet1.compareHosts(
1735 mnHosts,
1736 hosts[ controller ] )
1737 else:
1738 currentHostsResult = main.FALSE
1739 utilities.assert_equals( expect=main.TRUE,
1740 actual=currentHostsResult,
1741 onpass="ONOS" + controllerStr +
1742 " hosts exist in Mininet",
1743 onfail="ONOS" + controllerStr +
1744 " hosts don't match Mininet" )
1745
1746 devicesResults = devicesResults and currentDevicesResult
1747 linksResults = linksResults and currentLinksResult
1748 hostsResults = hostsResults and currentHostsResult
1749
1750 main.step( "Device information is correct" )
1751 utilities.assert_equals(
1752 expect=main.TRUE,
1753 actual=devicesResults,
1754 onpass="Device information is correct",
1755 onfail="Device information is incorrect" )
1756
1757 main.step( "Links are correct" )
1758 utilities.assert_equals(
1759 expect=main.TRUE,
1760 actual=linksResults,
1761 onpass="Link are correct",
1762 onfail="Links are incorrect" )
1763
1764 main.step( "Hosts are correct" )
1765 utilities.assert_equals(
1766 expect=main.TRUE,
1767 actual=hostsResults,
1768 onpass="Hosts are correct",
1769 onfail="Hosts are incorrect" )
1770
1771 def CASE6( self, main ):
1772 """
1773 The Failure case.
1774 """
1775 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001776 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001777 assert main, "main not defined"
1778 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001779 assert main.CLIs, "main.CLIs not defined"
1780 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001781 try:
1782 labels
1783 except NameError:
1784 main.log.error( "labels not defined, setting to []" )
1785 global labels
1786 labels = []
1787 try:
1788 data
1789 except NameError:
1790 main.log.error( "data not defined, setting to []" )
1791 global data
1792 data = []
1793 # Reset non-persistent variables
1794 try:
1795 iCounterValue = 0
1796 except NameError:
1797 main.log.error( "iCounterValue not defined, setting to 0" )
1798 iCounterValue = 0
1799
1800 main.case( "Restart entire ONOS cluster" )
1801
Jon Hall5ec6b1b2015-09-17 18:20:14 -07001802 main.step( "Checking ONOS Logs for errors" )
1803 for node in main.nodes:
1804 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1805 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1806
Jon Hall5cf14d52015-07-16 12:15:19 -07001807 main.step( "Killing ONOS nodes" )
1808 killResults = main.TRUE
1809 killTime = time.time()
Jon Halle1a3b752015-07-22 13:02:46 -07001810 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001811 killed = main.ONOSbench.onosKill( node.ip_address )
1812 killResults = killResults and killed
1813 utilities.assert_equals( expect=main.TRUE, actual=killResults,
1814 onpass="ONOS nodes killed",
1815 onfail="ONOS kill unsuccessful" )
1816
1817 main.step( "Checking if ONOS is up yet" )
1818 for i in range( 2 ):
1819 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001820 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001821 started = main.ONOSbench.isup( node.ip_address )
1822 if not started:
1823 main.log.error( node.name + " didn't start!" )
1824 onosIsupResult = onosIsupResult and started
1825 if onosIsupResult == main.TRUE:
1826 break
1827 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
1828 onpass="ONOS restarted",
1829 onfail="ONOS restart NOT successful" )
1830
Jon Hall6509dbf2016-06-21 17:01:17 -07001831 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001832 cliResults = main.TRUE
1833 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -07001834 for i in range( main.numCtrls ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001835 t = main.Thread( target=main.CLIs[ i ].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -07001836 name="startOnosCli-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001837 args=[ main.nodes[ i ].ip_address ] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001838 threads.append( t )
1839 t.start()
1840
1841 for t in threads:
1842 t.join()
1843 cliResults = cliResults and t.result
1844 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1845 onpass="ONOS cli started",
1846 onfail="ONOS clis did not restart" )
1847
Jon Hall6e709752016-02-01 13:38:46 -08001848 for i in range( 10 ):
1849 ready = True
Jon Hall7ac7bc32016-05-05 10:57:02 -07001850 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001851 cli = main.CLIs[ i ]
Jon Hall6e709752016-02-01 13:38:46 -08001852 output = cli.summary()
1853 if not output:
1854 ready = False
Jon Halld2871c22016-07-26 11:01:14 -07001855 if ready:
1856 break
Jon Hall6e709752016-02-01 13:38:46 -08001857 time.sleep( 30 )
1858 utilities.assert_equals( expect=True, actual=ready,
1859 onpass="ONOS summary command succeded",
1860 onfail="ONOS summary command failed" )
1861 if not ready:
1862 main.cleanup()
1863 main.exit()
1864
Jon Hall5cf14d52015-07-16 12:15:19 -07001865 # Grab the time of restart so we chan check how long the gossip
1866 # protocol has had time to work
1867 main.restartTime = time.time() - killTime
1868 main.log.debug( "Restart time: " + str( main.restartTime ) )
1869 labels.append( "Restart" )
1870 data.append( str( main.restartTime ) )
1871
Jon Hall5cf14d52015-07-16 12:15:19 -07001872 # Rerun for election on restarted nodes
1873 runResults = main.TRUE
Jon Hall7ac7bc32016-05-05 10:57:02 -07001874 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001875 cli = main.CLIs[ i ]
Jon Halla440e872016-03-31 15:15:50 -07001876 run = cli.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07001877 if run != main.TRUE:
1878 main.log.error( "Error running for election on " + cli.name )
1879 runResults = runResults and run
1880 utilities.assert_equals( expect=main.TRUE, actual=runResults,
1881 onpass="Reran for election",
1882 onfail="Failed to rerun for election" )
1883
1884 # TODO: Make this configurable
1885 time.sleep( 60 )
Jon Hallf37d44d2017-05-24 10:37:30 -07001886 node = main.activeNodes[ 0 ]
1887 main.log.debug( main.CLIs[ node ].nodes( jsonFormat=False ) )
1888 main.log.debug( main.CLIs[ node ].leaders( jsonFormat=False ) )
1889 main.log.debug( main.CLIs[ node ].partitions( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001890
1891 def CASE7( self, main ):
1892 """
1893 Check state after ONOS failure
1894 """
1895 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001896 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001897 assert main, "main not defined"
1898 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001899 assert main.CLIs, "main.CLIs not defined"
1900 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001901 main.case( "Running ONOS Constant State Tests" )
1902
1903 main.step( "Check that each switch has a master" )
1904 # Assert that each device has a master
1905 rolesNotNull = main.TRUE
1906 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001907 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001908 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001909 name="rolesNotNull-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001910 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001911 threads.append( t )
1912 t.start()
1913
1914 for t in threads:
1915 t.join()
1916 rolesNotNull = rolesNotNull and t.result
1917 utilities.assert_equals(
1918 expect=main.TRUE,
1919 actual=rolesNotNull,
1920 onpass="Each device has a master",
1921 onfail="Some devices don't have a master assigned" )
1922
1923 main.step( "Read device roles from ONOS" )
1924 ONOSMastership = []
1925 mastershipCheck = main.FALSE
1926 consistentMastership = True
1927 rolesResults = True
1928 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001929 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001930 t = main.Thread( target=main.CLIs[ i ].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001931 name="roles-" + str( i ),
1932 args=[] )
1933 threads.append( t )
1934 t.start()
1935
1936 for t in threads:
1937 t.join()
1938 ONOSMastership.append( t.result )
1939
Jon Halla440e872016-03-31 15:15:50 -07001940 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001941 node = str( main.activeNodes[ i ] + 1 )
1942 if not ONOSMastership[ i ] or "Error" in ONOSMastership[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07001943 main.log.error( "Error in getting ONOS" + node + " roles" )
1944 main.log.warn( "ONOS" + node + " mastership response: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001945 repr( ONOSMastership[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001946 rolesResults = False
1947 utilities.assert_equals(
1948 expect=True,
1949 actual=rolesResults,
1950 onpass="No error in reading roles output",
1951 onfail="Error in reading roles from ONOS" )
1952
1953 main.step( "Check for consistency in roles from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001954 if all( [ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001955 main.log.info(
1956 "Switch roles are consistent across all ONOS nodes" )
1957 else:
1958 consistentMastership = False
1959 utilities.assert_equals(
1960 expect=True,
1961 actual=consistentMastership,
1962 onpass="Switch roles are consistent across all ONOS nodes",
1963 onfail="ONOS nodes have different views of switch roles" )
1964
1965 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001966 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001967 node = str( main.activeNodes[ i ] + 1 )
Jon Halla440e872016-03-31 15:15:50 -07001968 main.log.warn( "ONOS" + node + " roles: ",
Jon Hall6e709752016-02-01 13:38:46 -08001969 json.dumps( json.loads( ONOSMastership[ i ] ),
1970 sort_keys=True,
1971 indent=4,
1972 separators=( ',', ': ' ) ) )
1973 elif rolesResults and consistentMastership:
Jon Hall5cf14d52015-07-16 12:15:19 -07001974 mastershipCheck = main.TRUE
1975
Jon Hall5cf14d52015-07-16 12:15:19 -07001976 # NOTE: we expect mastership to change on controller failure
1977
1978 main.step( "Get the intents and compare across all nodes" )
1979 ONOSIntents = []
1980 intentCheck = main.FALSE
1981 consistentIntents = True
1982 intentsResults = True
1983 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001984 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001985 t = main.Thread( target=main.CLIs[ i ].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001986 name="intents-" + str( i ),
1987 args=[],
1988 kwargs={ 'jsonFormat': True } )
1989 threads.append( t )
1990 t.start()
1991
1992 for t in threads:
1993 t.join()
1994 ONOSIntents.append( t.result )
1995
Jon Hallf37d44d2017-05-24 10:37:30 -07001996 for i in range( len( ONOSIntents ) ):
1997 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001998 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07001999 main.log.error( "Error in getting ONOS" + node + " intents" )
2000 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07002001 repr( ONOSIntents[ i ] ) )
2002 intentsResults = False
2003 utilities.assert_equals(
2004 expect=True,
2005 actual=intentsResults,
2006 onpass="No error in reading intents output",
2007 onfail="Error in reading intents from ONOS" )
2008
2009 main.step( "Check for consistency in Intents from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002010 if all( [ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07002011 main.log.info( "Intents are consistent across all ONOS " +
2012 "nodes" )
2013 else:
2014 consistentIntents = False
2015
2016 # Try to make it easy to figure out what is happening
2017 #
2018 # Intent ONOS1 ONOS2 ...
2019 # 0x01 INSTALLED INSTALLING
2020 # ... ... ...
2021 # ... ... ...
2022 title = " ID"
Jon Halla440e872016-03-31 15:15:50 -07002023 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07002024 title += " " * 10 + "ONOS" + str( n + 1 )
2025 main.log.warn( title )
2026 # get all intent keys in the cluster
2027 keys = []
2028 for nodeStr in ONOSIntents:
2029 node = json.loads( nodeStr )
2030 for intent in node:
2031 keys.append( intent.get( 'id' ) )
2032 keys = set( keys )
2033 for key in keys:
2034 row = "%-13s" % key
2035 for nodeStr in ONOSIntents:
2036 node = json.loads( nodeStr )
2037 for intent in node:
2038 if intent.get( 'id' ) == key:
2039 row += "%-15s" % intent.get( 'state' )
2040 main.log.warn( row )
2041 # End table view
2042
2043 utilities.assert_equals(
2044 expect=True,
2045 actual=consistentIntents,
2046 onpass="Intents are consistent across all ONOS nodes",
2047 onfail="ONOS nodes have different views of intents" )
2048 intentStates = []
2049 for node in ONOSIntents: # Iter through ONOS nodes
2050 nodeStates = []
2051 # Iter through intents of a node
2052 try:
2053 for intent in json.loads( node ):
2054 nodeStates.append( intent[ 'state' ] )
2055 except ( ValueError, TypeError ):
2056 main.log.exception( "Error in parsing intents" )
2057 main.log.error( repr( node ) )
2058 intentStates.append( nodeStates )
Jon Hallf37d44d2017-05-24 10:37:30 -07002059 out = [ ( i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002060 main.log.info( dict( out ) )
2061
2062 if intentsResults and not consistentIntents:
Jon Halla440e872016-03-31 15:15:50 -07002063 for i in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002064 node = str( main.activeNodes[ i ] + 1 )
Jon Halla440e872016-03-31 15:15:50 -07002065 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07002066 main.log.warn( json.dumps(
2067 json.loads( ONOSIntents[ i ] ),
2068 sort_keys=True,
2069 indent=4,
2070 separators=( ',', ': ' ) ) )
2071 elif intentsResults and consistentIntents:
2072 intentCheck = main.TRUE
2073
2074 # NOTE: Store has no durability, so intents are lost across system
2075 # restarts
2076 """
2077 main.step( "Compare current intents with intents before the failure" )
2078 # NOTE: this requires case 5 to pass for intentState to be set.
2079 # maybe we should stop the test if that fails?
2080 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002081 try:
2082 intentState
2083 except NameError:
2084 main.log.warn( "No previous intent state was saved" )
2085 else:
2086 if intentState and intentState == ONOSIntents[ 0 ]:
2087 sameIntents = main.TRUE
2088 main.log.info( "Intents are consistent with before failure" )
2089 # TODO: possibly the states have changed? we may need to figure out
2090 # what the acceptable states are
2091 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2092 sameIntents = main.TRUE
2093 try:
2094 before = json.loads( intentState )
2095 after = json.loads( ONOSIntents[ 0 ] )
2096 for intent in before:
2097 if intent not in after:
2098 sameIntents = main.FALSE
2099 main.log.debug( "Intent is not currently in ONOS " +
2100 "(at least in the same form):" )
2101 main.log.debug( json.dumps( intent ) )
2102 except ( ValueError, TypeError ):
2103 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002104 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002105 main.log.debug( repr( intentState ) )
2106 if sameIntents == main.FALSE:
2107 try:
2108 main.log.debug( "ONOS intents before: " )
2109 main.log.debug( json.dumps( json.loads( intentState ),
2110 sort_keys=True, indent=4,
2111 separators=( ',', ': ' ) ) )
2112 main.log.debug( "Current ONOS intents: " )
2113 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2114 sort_keys=True, indent=4,
2115 separators=( ',', ': ' ) ) )
2116 except ( ValueError, TypeError ):
2117 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002118 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002119 main.log.debug( repr( intentState ) )
2120 utilities.assert_equals(
2121 expect=main.TRUE,
2122 actual=sameIntents,
2123 onpass="Intents are consistent with before failure",
2124 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002125 intentCheck = intentCheck and sameIntents
2126 """
2127 main.step( "Get the OF Table entries and compare to before " +
2128 "component failure" )
2129 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002130 for i in range( 28 ):
2131 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002132 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hallf37d44d2017-05-24 10:37:30 -07002133 curSwitch = main.Mininet1.flowTableComp( flows[ i ], tmpFlows )
Jon Hall41d39f12016-04-11 22:54:35 -07002134 FlowTables = FlowTables and curSwitch
2135 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002136 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002137 utilities.assert_equals(
2138 expect=main.TRUE,
2139 actual=FlowTables,
2140 onpass="No changes were found in the flow tables",
2141 onfail="Changes were found in the flow tables" )
2142
2143 main.Mininet2.pingLongKill()
Jon Hallf37d44d2017-05-24 10:37:30 -07002144 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002145 # main.step( "Check the continuous pings to ensure that no packets " +
2146 # "were dropped during component failure" )
2147 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2148 main.params[ 'TESTONIP' ] )
2149 LossInPings = main.FALSE
2150 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2151 for i in range( 8, 18 ):
2152 main.log.info(
2153 "Checking for a loss in pings along flow from s" +
2154 str( i ) )
2155 LossInPings = main.Mininet2.checkForLoss(
2156 "/tmp/ping.h" +
2157 str( i ) ) or LossInPings
2158 if LossInPings == main.TRUE:
2159 main.log.info( "Loss in ping detected" )
2160 elif LossInPings == main.ERROR:
2161 main.log.info( "There are multiple mininet process running" )
2162 elif LossInPings == main.FALSE:
2163 main.log.info( "No Loss in the pings" )
2164 main.log.info( "No loss of dataplane connectivity" )
2165 # utilities.assert_equals(
2166 # expect=main.FALSE,
2167 # actual=LossInPings,
2168 # onpass="No Loss of connectivity",
2169 # onfail="Loss of dataplane connectivity detected" )
2170
2171 # NOTE: Since intents are not persisted with IntnentStore,
2172 # we expect loss in dataplane connectivity
2173 LossInPings = main.FALSE
Jon Hallf37d44d2017-05-24 10:37:30 -07002174 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002175 main.step( "Leadership Election is still functional" )
2176 # Test of LeadershipElection
2177 leaderList = []
2178 leaderResult = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002179
2180 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002181 cli = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002182 leaderN = cli.electionTestLeader()
2183 leaderList.append( leaderN )
2184 if leaderN == main.FALSE:
2185 # error in response
2186 main.log.error( "Something is wrong with " +
2187 "electionTestLeader function, check the" +
2188 " error logs" )
2189 leaderResult = main.FALSE
2190 elif leaderN is None:
2191 main.log.error( cli.name +
2192 " shows no leader for the election-app." )
2193 leaderResult = main.FALSE
2194 if len( set( leaderList ) ) != 1:
2195 leaderResult = main.FALSE
2196 main.log.error(
2197 "Inconsistent view of leader for the election test app" )
2198 # TODO: print the list
2199 utilities.assert_equals(
2200 expect=main.TRUE,
2201 actual=leaderResult,
2202 onpass="Leadership election passed",
2203 onfail="Something went wrong with Leadership election" )
2204
2205 def CASE8( self, main ):
2206 """
2207 Compare topo
2208 """
2209 import json
2210 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002211 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002212 assert main, "main not defined"
2213 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002214 assert main.CLIs, "main.CLIs not defined"
2215 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002216
2217 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002218 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002219 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002220 topoResult = main.FALSE
Jon Hall6e709752016-02-01 13:38:46 -08002221 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002222 elapsed = 0
2223 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002224 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002225 startTime = time.time()
2226 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002227 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hallba609822015-09-18 12:00:21 -07002228 devicesResults = main.TRUE
2229 linksResults = main.TRUE
2230 hostsResults = main.TRUE
2231 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002232 count += 1
2233 cliStart = time.time()
2234 devices = []
2235 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002236 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002237 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002238 name="devices-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002239 args=[ main.CLIs[ i ].devices, [ None ] ],
2240 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002241 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002242 threads.append( t )
2243 t.start()
2244
2245 for t in threads:
2246 t.join()
2247 devices.append( t.result )
2248 hosts = []
2249 ipResult = main.TRUE
2250 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002251 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002252 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002253 name="hosts-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002254 args=[ main.CLIs[ i ].hosts, [ None ] ],
2255 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Halld8f6de82015-12-17 17:04:34 -08002256 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002257 threads.append( t )
2258 t.start()
2259
2260 for t in threads:
2261 t.join()
2262 try:
2263 hosts.append( json.loads( t.result ) )
2264 except ( ValueError, TypeError ):
2265 main.log.exception( "Error parsing hosts results" )
2266 main.log.error( repr( t.result ) )
Jon Hall3afe4c92015-12-14 19:30:38 -08002267 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002268 for controller in range( 0, len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002269 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002270 if hosts[ controller ]:
2271 for host in hosts[ controller ]:
2272 if host is None or host.get( 'ipAddresses', [] ) == []:
2273 main.log.error(
2274 "Error with host ipAddresses on controller" +
2275 controllerStr + ": " + str( host ) )
2276 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002277 ports = []
2278 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002279 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002280 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002281 name="ports-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002282 args=[ main.CLIs[ i ].ports, [ None ] ],
2283 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002284 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002285 threads.append( t )
2286 t.start()
2287
2288 for t in threads:
2289 t.join()
2290 ports.append( t.result )
2291 links = []
2292 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002293 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002294 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002295 name="links-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002296 args=[ main.CLIs[ i ].links, [ None ] ],
2297 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002298 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002299 threads.append( t )
2300 t.start()
2301
2302 for t in threads:
2303 t.join()
2304 links.append( t.result )
2305 clusters = []
2306 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002307 for i in main.activeNodes:
Jon Hall6e709752016-02-01 13:38:46 -08002308 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002309 name="clusters-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002310 args=[ main.CLIs[ i ].clusters, [ None ] ],
2311 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Hall6e709752016-02-01 13:38:46 -08002312 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002313 threads.append( t )
2314 t.start()
2315
2316 for t in threads:
2317 t.join()
2318 clusters.append( t.result )
2319
2320 elapsed = time.time() - startTime
2321 cliTime = time.time() - cliStart
2322 print "Elapsed time: " + str( elapsed )
2323 print "CLI time: " + str( cliTime )
2324
Jon Hall6e709752016-02-01 13:38:46 -08002325 if all( e is None for e in devices ) and\
2326 all( e is None for e in hosts ) and\
2327 all( e is None for e in ports ) and\
2328 all( e is None for e in links ) and\
2329 all( e is None for e in clusters ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002330 topoFailMsg = "Could not get topology from ONOS"
2331 main.log.error( topoFailMsg )
2332 continue # Try again, No use trying to compare
Jon Hall6e709752016-02-01 13:38:46 -08002333
Jon Hall5cf14d52015-07-16 12:15:19 -07002334 mnSwitches = main.Mininet1.getSwitches()
2335 mnLinks = main.Mininet1.getLinks()
2336 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07002337 for controller in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002338 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002339 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07002340 "Error" not in devices[ controller ] and\
2341 "Error" not in ports[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002342
Jon Hallc6793552016-01-19 14:18:37 -08002343 try:
2344 currentDevicesResult = main.Mininet1.compareSwitches(
2345 mnSwitches,
2346 json.loads( devices[ controller ] ),
2347 json.loads( ports[ controller ] ) )
2348 except ( TypeError, ValueError ) as e:
2349 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2350 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002351 else:
2352 currentDevicesResult = main.FALSE
2353 utilities.assert_equals( expect=main.TRUE,
2354 actual=currentDevicesResult,
2355 onpass="ONOS" + controllerStr +
2356 " Switches view is correct",
2357 onfail="ONOS" + controllerStr +
2358 " Switches view is incorrect" )
2359
2360 if links[ controller ] and "Error" not in links[ controller ]:
2361 currentLinksResult = main.Mininet1.compareLinks(
2362 mnSwitches, mnLinks,
2363 json.loads( links[ controller ] ) )
2364 else:
2365 currentLinksResult = main.FALSE
2366 utilities.assert_equals( expect=main.TRUE,
2367 actual=currentLinksResult,
2368 onpass="ONOS" + controllerStr +
2369 " links view is correct",
2370 onfail="ONOS" + controllerStr +
2371 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002372 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002373 currentHostsResult = main.Mininet1.compareHosts(
2374 mnHosts,
2375 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002376 elif hosts[ controller ] == []:
2377 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002378 else:
2379 currentHostsResult = main.FALSE
2380 utilities.assert_equals( expect=main.TRUE,
2381 actual=currentHostsResult,
2382 onpass="ONOS" + controllerStr +
2383 " hosts exist in Mininet",
2384 onfail="ONOS" + controllerStr +
2385 " hosts don't match Mininet" )
2386 # CHECKING HOST ATTACHMENT POINTS
2387 hostAttachment = True
Jon Halla440e872016-03-31 15:15:50 -07002388 zeroHosts = False
Jon Hall5cf14d52015-07-16 12:15:19 -07002389 # FIXME: topo-HA/obelisk specific mappings:
2390 # key is mac and value is dpid
2391 mappings = {}
2392 for i in range( 1, 29 ): # hosts 1 through 28
2393 # set up correct variables:
Jon Hallf37d44d2017-05-24 10:37:30 -07002394 macId = "00:" * 5 + hex( i ).split( "0x" )[ 1 ].upper().zfill( 2 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002395 if i == 1:
Jon Hallf37d44d2017-05-24 10:37:30 -07002396 deviceId = "1000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002397 elif i == 2:
Jon Hallf37d44d2017-05-24 10:37:30 -07002398 deviceId = "2000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002399 elif i == 3:
Jon Hallf37d44d2017-05-24 10:37:30 -07002400 deviceId = "3000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002401 elif i == 4:
Jon Hallf37d44d2017-05-24 10:37:30 -07002402 deviceId = "3004".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002403 elif i == 5:
Jon Hallf37d44d2017-05-24 10:37:30 -07002404 deviceId = "5000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002405 elif i == 6:
Jon Hallf37d44d2017-05-24 10:37:30 -07002406 deviceId = "6000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002407 elif i == 7:
Jon Hallf37d44d2017-05-24 10:37:30 -07002408 deviceId = "6007".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002409 elif i >= 8 and i <= 17:
2410 dpid = '3' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07002411 deviceId = dpid.zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002412 elif i >= 18 and i <= 27:
2413 dpid = '6' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07002414 deviceId = dpid.zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002415 elif i == 28:
Jon Hallf37d44d2017-05-24 10:37:30 -07002416 deviceId = "2800".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002417 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002418 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002419 if hosts[ controller ] == []:
2420 main.log.warn( "There are no hosts discovered" )
Jon Halla440e872016-03-31 15:15:50 -07002421 zeroHosts = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002422 else:
2423 for host in hosts[ controller ]:
2424 mac = None
2425 location = None
2426 device = None
2427 port = None
2428 try:
2429 mac = host.get( 'mac' )
2430 assert mac, "mac field could not be found for this host object"
2431
Jeremy Ronquillo0e538bc2017-06-13 15:16:09 -07002432 location = host.get( 'locations' )[ 0 ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002433 assert location, "location field could not be found for this host object"
2434
2435 # Trim the protocol identifier off deviceId
Jon Hallf37d44d2017-05-24 10:37:30 -07002436 device = str( location.get( 'elementId' ) ).split( ':' )[ 1 ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002437 assert device, "elementId field could not be found for this host location object"
2438
2439 port = location.get( 'port' )
2440 assert port, "port field could not be found for this host location object"
2441
2442 # Now check if this matches where they should be
2443 if mac and device and port:
2444 if str( port ) != "1":
2445 main.log.error( "The attachment port is incorrect for " +
2446 "host " + str( mac ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07002447 ". Expected: 1 Actual: " + str( port ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002448 hostAttachment = False
2449 if device != mappings[ str( mac ) ]:
2450 main.log.error( "The attachment device is incorrect for " +
2451 "host " + str( mac ) +
2452 ". Expected: " + mappings[ str( mac ) ] +
2453 " Actual: " + device )
2454 hostAttachment = False
2455 else:
2456 hostAttachment = False
2457 except AssertionError:
2458 main.log.exception( "Json object not as expected" )
2459 main.log.error( repr( host ) )
2460 hostAttachment = False
2461 else:
2462 main.log.error( "No hosts json output or \"Error\"" +
2463 " in output. hosts = " +
2464 repr( hosts[ controller ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002465 if zeroHosts is False:
Jon Hall5cf14d52015-07-16 12:15:19 -07002466 # TODO: Find a way to know if there should be hosts in a
2467 # given point of the test
2468 hostAttachment = True
2469
2470 # END CHECKING HOST ATTACHMENT POINTS
2471 devicesResults = devicesResults and currentDevicesResult
2472 linksResults = linksResults and currentLinksResult
2473 hostsResults = hostsResults and currentHostsResult
2474 hostAttachmentResults = hostAttachmentResults and\
2475 hostAttachment
2476 topoResult = ( devicesResults and linksResults
2477 and hostsResults and ipResult and
2478 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002479 utilities.assert_equals( expect=True,
2480 actual=topoResult,
2481 onpass="ONOS topology matches Mininet",
Jon Hall6e709752016-02-01 13:38:46 -08002482 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002483 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002484
2485 # Compare json objects for hosts and dataplane clusters
2486
2487 # hosts
2488 main.step( "Hosts view is consistent across all ONOS nodes" )
2489 consistentHostsResult = main.TRUE
2490 for controller in range( len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002491 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002492 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002493 if hosts[ controller ] == hosts[ 0 ]:
2494 continue
2495 else: # hosts not consistent
2496 main.log.error( "hosts from ONOS" + controllerStr +
2497 " is inconsistent with ONOS1" )
2498 main.log.warn( repr( hosts[ controller ] ) )
2499 consistentHostsResult = main.FALSE
2500
2501 else:
2502 main.log.error( "Error in getting ONOS hosts from ONOS" +
2503 controllerStr )
2504 consistentHostsResult = main.FALSE
2505 main.log.warn( "ONOS" + controllerStr +
2506 " hosts response: " +
2507 repr( hosts[ controller ] ) )
2508 utilities.assert_equals(
2509 expect=main.TRUE,
2510 actual=consistentHostsResult,
2511 onpass="Hosts view is consistent across all ONOS nodes",
2512 onfail="ONOS nodes have different views of hosts" )
2513
2514 main.step( "Hosts information is correct" )
2515 hostsResults = hostsResults and ipResult
2516 utilities.assert_equals(
2517 expect=main.TRUE,
2518 actual=hostsResults,
2519 onpass="Host information is correct",
2520 onfail="Host information is incorrect" )
2521
2522 main.step( "Host attachment points to the network" )
2523 utilities.assert_equals(
2524 expect=True,
2525 actual=hostAttachmentResults,
2526 onpass="Hosts are correctly attached to the network",
2527 onfail="ONOS did not correctly attach hosts to the network" )
2528
2529 # Strongly connected clusters of devices
2530 main.step( "Clusters view is consistent across all ONOS nodes" )
2531 consistentClustersResult = main.TRUE
2532 for controller in range( len( clusters ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002533 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002534 if "Error" not in clusters[ controller ]:
2535 if clusters[ controller ] == clusters[ 0 ]:
2536 continue
2537 else: # clusters not consistent
2538 main.log.error( "clusters from ONOS" +
2539 controllerStr +
2540 " is inconsistent with ONOS1" )
2541 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002542 else:
2543 main.log.error( "Error in getting dataplane clusters " +
2544 "from ONOS" + controllerStr )
2545 consistentClustersResult = main.FALSE
2546 main.log.warn( "ONOS" + controllerStr +
2547 " clusters response: " +
2548 repr( clusters[ controller ] ) )
2549 utilities.assert_equals(
2550 expect=main.TRUE,
2551 actual=consistentClustersResult,
2552 onpass="Clusters view is consistent across all ONOS nodes",
2553 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002554 if not consistentClustersResult:
2555 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002556
2557 main.step( "There is only one SCC" )
2558 # there should always only be one cluster
2559 try:
2560 numClusters = len( json.loads( clusters[ 0 ] ) )
2561 except ( ValueError, TypeError ):
2562 main.log.exception( "Error parsing clusters[0]: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07002563 repr( clusters[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002564 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002565 clusterResults = main.FALSE
2566 if numClusters == 1:
2567 clusterResults = main.TRUE
2568 utilities.assert_equals(
2569 expect=1,
2570 actual=numClusters,
2571 onpass="ONOS shows 1 SCC",
2572 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2573
2574 topoResult = ( devicesResults and linksResults
2575 and hostsResults and consistentHostsResult
2576 and consistentClustersResult and clusterResults
2577 and ipResult and hostAttachmentResults )
2578
2579 topoResult = topoResult and int( count <= 2 )
2580 note = "note it takes about " + str( int( cliTime ) ) + \
2581 " seconds for the test to make all the cli calls to fetch " +\
2582 "the topology from each ONOS instance"
2583 main.log.info(
2584 "Very crass estimate for topology discovery/convergence( " +
2585 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2586 str( count ) + " tries" )
2587
2588 main.step( "Device information is correct" )
2589 utilities.assert_equals(
2590 expect=main.TRUE,
2591 actual=devicesResults,
2592 onpass="Device information is correct",
2593 onfail="Device information is incorrect" )
2594
2595 main.step( "Links are correct" )
2596 utilities.assert_equals(
2597 expect=main.TRUE,
2598 actual=linksResults,
2599 onpass="Link are correct",
2600 onfail="Links are incorrect" )
2601
Jon Halla440e872016-03-31 15:15:50 -07002602 main.step( "Hosts are correct" )
2603 utilities.assert_equals(
2604 expect=main.TRUE,
2605 actual=hostsResults,
2606 onpass="Hosts are correct",
2607 onfail="Hosts are incorrect" )
2608
Jon Hall5cf14d52015-07-16 12:15:19 -07002609 # FIXME: move this to an ONOS state case
2610 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002611 nodeResults = utilities.retry( main.HA.nodesCheck,
2612 False,
Jon Hallf37d44d2017-05-24 10:37:30 -07002613 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -07002614 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002615
Jon Hall41d39f12016-04-11 22:54:35 -07002616 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002617 onpass="Nodes check successful",
2618 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002619 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002620 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002621 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hallf37d44d2017-05-24 10:37:30 -07002622 main.CLIs[ i ].name,
2623 main.CLIs[ i ].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002624
Jon Halld2871c22016-07-26 11:01:14 -07002625 if not topoResult:
2626 main.cleanup()
2627 main.exit()
2628
Jon Hall5cf14d52015-07-16 12:15:19 -07002629 def CASE9( self, main ):
2630 """
2631 Link s3-s28 down
2632 """
2633 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002634 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002635 assert main, "main not defined"
2636 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002637 assert main.CLIs, "main.CLIs not defined"
2638 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002639 # NOTE: You should probably run a topology check after this
2640
2641 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2642
2643 description = "Turn off a link to ensure that Link Discovery " +\
2644 "is working properly"
2645 main.case( description )
2646
2647 main.step( "Kill Link between s3 and s28" )
2648 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2649 main.log.info( "Waiting " + str( linkSleep ) +
2650 " seconds for link down to be discovered" )
2651 time.sleep( linkSleep )
2652 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2653 onpass="Link down successful",
2654 onfail="Failed to bring link down" )
2655 # TODO do some sort of check here
2656
2657 def CASE10( self, main ):
2658 """
2659 Link s3-s28 up
2660 """
2661 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002662 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002663 assert main, "main not defined"
2664 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002665 assert main.CLIs, "main.CLIs not defined"
2666 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002667 # NOTE: You should probably run a topology check after this
2668
2669 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2670
2671 description = "Restore a link to ensure that Link Discovery is " + \
2672 "working properly"
2673 main.case( description )
2674
2675 main.step( "Bring link between s3 and s28 back up" )
2676 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2677 main.log.info( "Waiting " + str( linkSleep ) +
2678 " seconds for link up to be discovered" )
2679 time.sleep( linkSleep )
2680 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2681 onpass="Link up successful",
2682 onfail="Failed to bring link up" )
2683 # TODO do some sort of check here
2684
2685 def CASE11( self, main ):
2686 """
2687 Switch Down
2688 """
2689 # NOTE: You should probably run a topology check after this
2690 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002691 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002692 assert main, "main not defined"
2693 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002694 assert main.CLIs, "main.CLIs not defined"
2695 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002696
2697 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2698
2699 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallf37d44d2017-05-24 10:37:30 -07002700 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002701 main.case( description )
2702 switch = main.params[ 'kill' ][ 'switch' ]
2703 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2704
2705 # TODO: Make this switch parameterizable
2706 main.step( "Kill " + switch )
2707 main.log.info( "Deleting " + switch )
2708 main.Mininet1.delSwitch( switch )
2709 main.log.info( "Waiting " + str( switchSleep ) +
2710 " seconds for switch down to be discovered" )
2711 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002712 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002713 # Peek at the deleted switch
2714 main.log.warn( str( device ) )
2715 result = main.FALSE
2716 if device and device[ 'available' ] is False:
2717 result = main.TRUE
2718 utilities.assert_equals( expect=main.TRUE, actual=result,
2719 onpass="Kill switch successful",
2720 onfail="Failed to kill switch?" )
2721
2722 def CASE12( self, main ):
2723 """
2724 Switch Up
2725 """
2726 # NOTE: You should probably run a topology check after this
2727 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002728 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002729 assert main, "main not defined"
2730 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002731 assert main.CLIs, "main.CLIs not defined"
2732 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002733 assert ONOS1Port, "ONOS1Port not defined"
2734 assert ONOS2Port, "ONOS2Port not defined"
2735 assert ONOS3Port, "ONOS3Port not defined"
2736 assert ONOS4Port, "ONOS4Port not defined"
2737 assert ONOS5Port, "ONOS5Port not defined"
2738 assert ONOS6Port, "ONOS6Port not defined"
2739 assert ONOS7Port, "ONOS7Port not defined"
2740
2741 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2742 switch = main.params[ 'kill' ][ 'switch' ]
2743 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2744 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallf37d44d2017-05-24 10:37:30 -07002745 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002746 description = "Adding a switch to ensure it is discovered correctly"
2747 main.case( description )
2748
2749 main.step( "Add back " + switch )
2750 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2751 for peer in links:
2752 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07002753 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002754 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2755 main.log.info( "Waiting " + str( switchSleep ) +
2756 " seconds for switch up to be discovered" )
2757 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002758 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002759 # Peek at the deleted switch
2760 main.log.warn( str( device ) )
2761 result = main.FALSE
2762 if device and device[ 'available' ]:
2763 result = main.TRUE
2764 utilities.assert_equals( expect=main.TRUE, actual=result,
2765 onpass="add switch successful",
2766 onfail="Failed to add switch?" )
2767
2768 def CASE13( self, main ):
2769 """
2770 Clean up
2771 """
2772 import os
2773 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002774 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002775 assert main, "main not defined"
2776 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002777 assert main.CLIs, "main.CLIs not defined"
2778 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002779
2780 # printing colors to terminal
2781 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2782 'blue': '\033[94m', 'green': '\033[92m',
2783 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2784 main.case( "Test Cleanup" )
2785 main.step( "Killing tcpdumps" )
2786 main.Mininet2.stopTcpdump()
2787
2788 testname = main.TEST
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002789 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002790 main.step( "Copying MN pcap and ONOS log files to test station" )
2791 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2792 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002793 # NOTE: MN Pcap file is being saved to logdir.
2794 # We scp this file as MN and TestON aren't necessarily the same vm
2795
2796 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002797 # TODO: Load these from params
2798 # NOTE: must end in /
2799 logFolder = "/opt/onos/log/"
2800 logFiles = [ "karaf.log", "karaf.log.1" ]
2801 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002802 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002803 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002804 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002805 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2806 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002807 # std*.log's
2808 # NOTE: must end in /
2809 logFolder = "/opt/onos/var/"
2810 logFiles = [ "stderr.log", "stdout.log" ]
2811 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002812 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002813 for node in main.nodes:
Jon Hall6e709752016-02-01 13:38:46 -08002814 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002815 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2816 logFolder + f, dstName )
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002817 else:
2818 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002819
2820 main.step( "Stopping Mininet" )
2821 mnResult = main.Mininet1.stopNet()
2822 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2823 onpass="Mininet stopped",
2824 onfail="MN cleanup NOT successful" )
2825
2826 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002827 for node in main.nodes:
Jon Hall5ec6b1b2015-09-17 18:20:14 -07002828 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2829 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002830
2831 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07002832 timerLog = open( main.logdir + "/Timers.csv", 'w' )
Jon Hall5cf14d52015-07-16 12:15:19 -07002833 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) )
2834 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) )
2835 timerLog.close()
Jon Hallf37d44d2017-05-24 10:37:30 -07002836 except NameError as e:
2837 main.log.exception( e )
Jon Hall5cf14d52015-07-16 12:15:19 -07002838
2839 def CASE14( self, main ):
2840 """
2841 start election app on all onos nodes
2842 """
Jon Halle1a3b752015-07-22 13:02:46 -07002843 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002844 assert main, "main not defined"
2845 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002846 assert main.CLIs, "main.CLIs not defined"
2847 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002848
Jon Hallf37d44d2017-05-24 10:37:30 -07002849 main.case( "Start Leadership Election app" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002850 main.step( "Install leadership election app" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002851 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Halla440e872016-03-31 15:15:50 -07002852 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002853 utilities.assert_equals(
2854 expect=main.TRUE,
2855 actual=appResult,
2856 onpass="Election app installed",
2857 onfail="Something went wrong with installing Leadership election" )
2858
2859 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002860 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002861 main.CLIs[ i ].electionTestRun()
2862 time.sleep( 5 )
2863 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall25463a82016-04-13 14:03:52 -07002864 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002865 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002866 expect=True,
2867 actual=sameResult,
2868 onpass="All nodes see the same leaderboards",
2869 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002870
Jon Hall25463a82016-04-13 14:03:52 -07002871 if sameResult:
2872 leader = leaders[ 0 ][ 0 ]
Jon Hallf37d44d2017-05-24 10:37:30 -07002873 if main.nodes[ main.activeNodes[ 0 ] ].ip_address in leader:
Jon Hall25463a82016-04-13 14:03:52 -07002874 correctLeader = True
2875 else:
2876 correctLeader = False
2877 main.step( "First node was elected leader" )
2878 utilities.assert_equals(
2879 expect=True,
2880 actual=correctLeader,
2881 onpass="Correct leader was elected",
2882 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002883
2884 def CASE15( self, main ):
2885 """
2886 Check that Leadership Election is still functional
acsmars9475b1c2015-08-28 18:02:08 -07002887 15.1 Run election on each node
2888 15.2 Check that each node has the same leaders and candidates
2889 15.3 Find current leader and withdraw
2890 15.4 Check that a new node was elected leader
2891 15.5 Check that that new leader was the candidate of old leader
2892 15.6 Run for election on old leader
2893 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2894 15.8 Make sure that the old leader was added to the candidate list
2895
2896 old and new variable prefixes refer to data from before vs after
2897 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002898 """
2899 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002900 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002901 assert main, "main not defined"
2902 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002903 assert main.CLIs, "main.CLIs not defined"
2904 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002905
Jon Hall5cf14d52015-07-16 12:15:19 -07002906 description = "Check that Leadership Election is still functional"
2907 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002908 # NOTE: Need to re-run after restarts since being a canidate is not persistant
acsmars9475b1c2015-08-28 18:02:08 -07002909
Jon Halla440e872016-03-31 15:15:50 -07002910 oldLeaders = [] # list of lists of each nodes' candidates before
2911 newLeaders = [] # list of lists of each nodes' candidates after
acsmars9475b1c2015-08-28 18:02:08 -07002912 oldLeader = '' # the old leader from oldLeaders, None if not same
2913 newLeader = '' # the new leaders fron newLoeaders, None if not same
2914 oldLeaderCLI = None # the CLI of the old leader used for re-electing
acsmars71adceb2015-08-31 15:09:26 -07002915 expectNoLeader = False # True when there is only one leader
2916 if main.numCtrls == 1:
2917 expectNoLeader = True
acsmars9475b1c2015-08-28 18:02:08 -07002918
Jon Hall5cf14d52015-07-16 12:15:19 -07002919 main.step( "Run for election on each node" )
acsmars9475b1c2015-08-28 18:02:08 -07002920 electionResult = main.TRUE
2921
Jon Halla440e872016-03-31 15:15:50 -07002922 for i in main.activeNodes: # run test election on each node
Jon Hallf37d44d2017-05-24 10:37:30 -07002923 if main.CLIs[ i ].electionTestRun() == main.FALSE:
acsmars9475b1c2015-08-28 18:02:08 -07002924 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002925 utilities.assert_equals(
2926 expect=main.TRUE,
acsmars9475b1c2015-08-28 18:02:08 -07002927 actual=electionResult,
2928 onpass="All nodes successfully ran for leadership",
2929 onfail="At least one node failed to run for leadership" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002930
acsmars3a72bde2015-09-02 14:16:22 -07002931 if electionResult == main.FALSE:
2932 main.log.error(
2933 "Skipping Test Case because Election Test App isn't loaded" )
2934 main.skipCase()
2935
acsmars9475b1c2015-08-28 18:02:08 -07002936 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002937 failMessage = "Nodes have different leaderboards"
Jon Hallf37d44d2017-05-24 10:37:30 -07002938 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002939 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002940 if sameResult:
2941 oldLeader = oldLeaders[ 0 ][ 0 ]
2942 main.log.warn( oldLeader )
acsmars9475b1c2015-08-28 18:02:08 -07002943 else:
Jon Halla440e872016-03-31 15:15:50 -07002944 oldLeader = None
Jon Hall5cf14d52015-07-16 12:15:19 -07002945 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002946 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002947 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002948 onpass="Leaderboards are consistent for the election topic",
acsmars9475b1c2015-08-28 18:02:08 -07002949 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002950
2951 main.step( "Find current leader and withdraw" )
acsmars9475b1c2015-08-28 18:02:08 -07002952 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002953 # do some sanity checking on leader before using it
acsmars9475b1c2015-08-28 18:02:08 -07002954 if oldLeader is None:
2955 main.log.error( "Leadership isn't consistent." )
2956 withdrawResult = main.FALSE
2957 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002958 for i in main.activeNodes:
acsmars9475b1c2015-08-28 18:02:08 -07002959 if oldLeader == main.nodes[ i ].ip_address:
2960 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002961 break
2962 else: # FOR/ELSE statement
2963 main.log.error( "Leader election, could not find current leader" )
2964 if oldLeader:
acsmars9475b1c2015-08-28 18:02:08 -07002965 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002966 utilities.assert_equals(
2967 expect=main.TRUE,
2968 actual=withdrawResult,
2969 onpass="Node was withdrawn from election",
2970 onfail="Node was not withdrawn from election" )
2971
acsmars9475b1c2015-08-28 18:02:08 -07002972 main.step( "Check that a new node was elected leader" )
acsmars9475b1c2015-08-28 18:02:08 -07002973 failMessage = "Nodes have different leaders"
acsmars9475b1c2015-08-28 18:02:08 -07002974 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002975 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002976 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002977 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002978 if newLeaders[ 0 ][ 0 ] == 'none':
2979 main.log.error( "No leader was elected on at least 1 node" )
2980 if not expectNoLeader:
2981 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002982 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002983
acsmars9475b1c2015-08-28 18:02:08 -07002984 # Check that the new leader is not the older leader, which was withdrawn
2985 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002986 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08002987 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07002988 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002989 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002990 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07002991 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002992 onpass="Leadership election passed",
2993 onfail="Something went wrong with Leadership election" )
2994
Jon Halla440e872016-03-31 15:15:50 -07002995 main.step( "Check that that new leader was the candidate of old leader" )
Jon Hall6e709752016-02-01 13:38:46 -08002996 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars9475b1c2015-08-28 18:02:08 -07002997 correctCandidateResult = main.TRUE
acsmars71adceb2015-08-31 15:09:26 -07002998 if expectNoLeader:
2999 if newLeader == 'none':
3000 main.log.info( "No leader expected. None found. Pass" )
3001 correctCandidateResult = main.TRUE
3002 else:
3003 main.log.info( "Expected no leader, got: " + str( newLeader ) )
3004 correctCandidateResult = main.FALSE
Jon Hallf37d44d2017-05-24 10:37:30 -07003005 elif len( oldLeaders[ 0 ] ) >= 3:
Jon Halla440e872016-03-31 15:15:50 -07003006 if newLeader == oldLeaders[ 0 ][ 2 ]:
3007 # correct leader was elected
3008 correctCandidateResult = main.TRUE
3009 else:
3010 correctCandidateResult = main.FALSE
3011 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
3012 newLeader, oldLeaders[ 0 ][ 2 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08003013 else:
3014 main.log.warn( "Could not determine who should be the correct leader" )
Jon Halla440e872016-03-31 15:15:50 -07003015 main.log.debug( oldLeaders[ 0 ] )
Jon Hall6e709752016-02-01 13:38:46 -08003016 correctCandidateResult = main.FALSE
acsmars9475b1c2015-08-28 18:02:08 -07003017 utilities.assert_equals(
3018 expect=main.TRUE,
3019 actual=correctCandidateResult,
3020 onpass="Correct Candidate Elected",
3021 onfail="Incorrect Candidate Elected" )
3022
Jon Hall5cf14d52015-07-16 12:15:19 -07003023 main.step( "Run for election on old leader( just so everyone " +
3024 "is in the hat )" )
acsmars9475b1c2015-08-28 18:02:08 -07003025 if oldLeaderCLI is not None:
3026 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07003027 else:
acsmars9475b1c2015-08-28 18:02:08 -07003028 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003029 runResult = main.FALSE
3030 utilities.assert_equals(
3031 expect=main.TRUE,
3032 actual=runResult,
3033 onpass="App re-ran for election",
3034 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07003035
acsmars9475b1c2015-08-28 18:02:08 -07003036 main.step(
3037 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07003038 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07003039 # Get new leaders and candidates
3040 reRunLeaders = []
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003041 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07003042 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07003043
acsmars9475b1c2015-08-28 18:02:08 -07003044 # Check that the re-elected node is last on the candidate List
Jon Hallf37d44d2017-05-24 10:37:30 -07003045 if not reRunLeaders[ 0 ]:
Jon Hall3a7843a2016-04-12 03:01:09 -07003046 positionResult = main.FALSE
3047 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07003048 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader ),
Jon Halla440e872016-03-31 15:15:50 -07003049 str( reRunLeaders[ 0 ] ) ) )
acsmars9475b1c2015-08-28 18:02:08 -07003050 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07003051 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07003052 expect=True,
acsmars9475b1c2015-08-28 18:02:08 -07003053 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07003054 onpass="Old leader successfully re-ran for election",
3055 onfail="Something went wrong with Leadership election after " +
3056 "the old leader re-ran for election" )
3057
3058 def CASE16( self, main ):
3059 """
3060 Install Distributed Primitives app
3061 """
3062 import time
Jon Halle1a3b752015-07-22 13:02:46 -07003063 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003064 assert main, "main not defined"
3065 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07003066 assert main.CLIs, "main.CLIs not defined"
3067 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07003068
3069 # Variables for the distributed primitives tests
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003070 main.pCounterName = "TestON-Partitions"
3071 main.pCounterValue = 0
Jon Hallf37d44d2017-05-24 10:37:30 -07003072 main.onosSet = set( [] )
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003073 main.onosSetName = "TestON-set"
Jon Hall5cf14d52015-07-16 12:15:19 -07003074
3075 description = "Install Primitives app"
3076 main.case( description )
3077 main.step( "Install Primitives app" )
3078 appName = "org.onosproject.distributedprimitives"
Jon Hallf37d44d2017-05-24 10:37:30 -07003079 node = main.activeNodes[ 0 ]
3080 appResults = main.CLIs[ node ].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003081 utilities.assert_equals( expect=main.TRUE,
3082 actual=appResults,
3083 onpass="Primitives app activated",
3084 onfail="Primitives app not activated" )
3085 time.sleep( 5 ) # To allow all nodes to activate
3086
3087 def CASE17( self, main ):
3088 """
3089 Check for basic functionality with distributed primitives
3090 """
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003091 main.HA.CASE17( main )