blob: 52f0b49830094a5b578bf767d1a3d3c2b400f340 [file] [log] [blame]
Jon Hall5cf14d52015-07-16 12:15:19 -07001"""
2Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign devices to controllers
9CASE21: Assign mastership to controllers
10CASE3: Assign intents
11CASE4: Ping across added host intents
12CASE5: Reading state of ONOS
13CASE6: The Failure case. Since this is the Sanity test, we do nothing.
14CASE7: Check state after control plane failure
15CASE8: Compare topo
16CASE9: Link s3-s28 down
17CASE10: Link s3-s28 up
18CASE11: Switch down
19CASE12: Switch up
20CASE13: Clean up
21CASE14: start election app on all onos nodes
22CASE15: Check that Leadership Election is still functional
23CASE16: Install Distributed Primitives app
24CASE17: Check for basic functionality with distributed primitives
25"""
Jon Hall5cf14d52015-07-16 12:15:19 -070026class HAsanity:
27
28 def __init__( self ):
29 self.default = ''
30
31 def CASE1( self, main ):
32 """
33 CASE1 is to compile ONOS and push it to the test machines
34
35 Startup sequence:
36 cell <name>
37 onos-verify-cell
38 NOTE: temporary - onos-remove-raft-logs
39 onos-uninstall
40 start mininet
41 git pull
42 mvn clean install
43 onos-package
44 onos-install -f
45 onos-wait-for-start
46 start cli sessions
47 start tcpdump
48 """
Jon Halle1a3b752015-07-22 13:02:46 -070049 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080050 import time
Jon Halla440e872016-03-31 15:15:50 -070051 import json
Jon Hall5cf14d52015-07-16 12:15:19 -070052 main.log.info( "ONOS HA Sanity test - initialization" )
53 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070054 main.caseExplanation = "Setup the test environment including " +\
Jon Hall5cf14d52015-07-16 12:15:19 -070055 "installing ONOS, starting Mininet and ONOS" +\
56 "cli sessions."
Jon Hall5cf14d52015-07-16 12:15:19 -070057
58 # load some variables from the params file
59 PULLCODE = False
60 if main.params[ 'Git' ] == 'True':
61 PULLCODE = True
62 gitBranch = main.params[ 'branch' ]
63 cellName = main.params[ 'ENV' ][ 'cellName' ]
64
Jon Halle1a3b752015-07-22 13:02:46 -070065 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070066 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070067 if main.ONOSbench.maxNodes < main.numCtrls:
68 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -070069 # TODO: refactor how to get onos port, maybe put into component tag?
Jon Halle1a3b752015-07-22 13:02:46 -070070 # set global variables
Jon Hall5cf14d52015-07-16 12:15:19 -070071 global ONOS1Port
72 global ONOS2Port
73 global ONOS3Port
74 global ONOS4Port
75 global ONOS5Port
76 global ONOS6Port
77 global ONOS7Port
Jon Halla440e872016-03-31 15:15:50 -070078 # These are for csv plotting in jenkins
79 global labels
80 global data
81 labels = []
82 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -070083
84 # FIXME: just get controller port from params?
85 # TODO: do we really need all these?
86 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
87 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
88 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
89 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
90 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
91 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
92 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
93
Jon Halle1a3b752015-07-22 13:02:46 -070094 try:
Jon Hall53c5e662016-04-13 16:06:56 -070095 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070096 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070097 except Exception as e:
98 main.log.exception( e )
99 main.cleanup()
100 main.exit()
101
102 main.CLIs = []
103 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700104 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700105 for i in range( 1, main.numCtrls + 1 ):
106 try:
107 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
108 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
109 ipList.append( main.nodes[ -1 ].ip_address )
110 except AttributeError:
111 break
Jon Hall5cf14d52015-07-16 12:15:19 -0700112
113 main.step( "Create cell file" )
114 cellAppString = main.params[ 'ENV' ][ 'appString' ]
115 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
116 main.Mininet1.ip_address,
Devin Lim461f0872017-06-05 16:49:33 -0700117 cellAppString, ipList, main.ONOScli1.user_name )
Jon Hall5cf14d52015-07-16 12:15:19 -0700118 main.step( "Applying cell variable to environment" )
119 cellResult = main.ONOSbench.setCell( cellName )
120 verifyResult = main.ONOSbench.verifyCell()
121
122 # FIXME:this is short term fix
123 main.log.info( "Removing raft logs" )
124 main.ONOSbench.onosRemoveRaftLogs()
125
126 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700127 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700128 main.ONOSbench.onosUninstall( node.ip_address )
129
130 # Make sure ONOS is DEAD
131 main.log.info( "Killing any ONOS processes" )
132 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700133 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700134 killed = main.ONOSbench.onosKill( node.ip_address )
135 killResults = killResults and killed
136
Jon Hall5cf14d52015-07-16 12:15:19 -0700137 gitPullResult = main.TRUE
138
139 main.step( "Starting Mininet" )
140 # scp topo file to mininet
141 # TODO: move to params?
142 topoName = "obelisk.py"
143 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700144 main.ONOSbench.scp( main.Mininet1,
145 filePath + topoName,
146 main.Mininet1.home,
147 direction="to" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700148 mnResult = main.Mininet1.startNet()
Jon Hall5cf14d52015-07-16 12:15:19 -0700149 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
150 onpass="Mininet Started",
151 onfail="Error starting Mininet" )
152
153 main.step( "Git checkout and pull " + gitBranch )
154 if PULLCODE:
155 main.ONOSbench.gitCheckout( gitBranch )
156 gitPullResult = main.ONOSbench.gitPull()
157 # values of 1 or 3 are good
158 utilities.assert_lesser( expect=0, actual=gitPullResult,
159 onpass="Git pull successful",
160 onfail="Git pull failed" )
161 main.ONOSbench.getVersion( report=True )
162
Jon Hall5cf14d52015-07-16 12:15:19 -0700163 # GRAPHS
164 # NOTE: important params here:
165 # job = name of Jenkins job
166 # Plot Name = Plot-HA, only can be used if multiple plots
167 # index = The number of the graph under plot name
168 job = "HAsanity"
169 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700170 index = "2"
Jon Hall5cf14d52015-07-16 12:15:19 -0700171 graphs = '<ac:structured-macro ac:name="html">\n'
172 graphs += '<ac:plain-text-body><![CDATA[\n'
173 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800174 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700175 '&width=500&height=300"' +\
176 'noborder="0" width="500" height="300" scrolling="yes" ' +\
177 'seamless="seamless"></iframe>\n'
178 graphs += ']]></ac:plain-text-body>\n'
179 graphs += '</ac:structured-macro>\n'
Jon Hallf37d44d2017-05-24 10:37:30 -0700180 main.log.wiki( graphs )
Jon Hall5cf14d52015-07-16 12:15:19 -0700181
182 main.step( "Creating ONOS package" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700183 packageResult = main.ONOSbench.buckBuild()
Jon Hall5cf14d52015-07-16 12:15:19 -0700184 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
185 onpass="ONOS package successful",
186 onfail="ONOS package failed" )
187
188 main.step( "Installing ONOS package" )
189 onosInstallResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700190 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700191 tmpResult = main.ONOSbench.onosInstall( options="-f",
192 node=node.ip_address )
193 onosInstallResult = onosInstallResult and tmpResult
194 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
195 onpass="ONOS install successful",
196 onfail="ONOS install failed" )
197
You Wangf5de25b2017-01-06 15:13:01 -0800198 main.step( "Set up ONOS secure SSH" )
199 secureSshResult = main.TRUE
200 for node in main.nodes:
201 secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=node.ip_address )
202 utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
203 onpass="Test step PASS",
204 onfail="Test step FAIL" )
205
Jon Hall5cf14d52015-07-16 12:15:19 -0700206 main.step( "Checking if ONOS is up yet" )
207 for i in range( 2 ):
208 onosIsupResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700209 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700210 started = main.ONOSbench.isup( node.ip_address )
211 if not started:
Jon Hallc6793552016-01-19 14:18:37 -0800212 main.log.error( node.name + " hasn't started" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700213 onosIsupResult = onosIsupResult and started
214 if onosIsupResult == main.TRUE:
215 break
216 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
217 onpass="ONOS startup successful",
218 onfail="ONOS startup failed" )
219
Jon Hall6509dbf2016-06-21 17:01:17 -0700220 main.step( "Starting ONOS CLI sessions" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700221 cliResults = main.TRUE
222 threads = []
Jon Halle1a3b752015-07-22 13:02:46 -0700223 for i in range( main.numCtrls ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700224 t = main.Thread( target=main.CLIs[ i ].startOnosCli,
Jon Hall5cf14d52015-07-16 12:15:19 -0700225 name="startOnosCli-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -0700226 args=[ main.nodes[ i ].ip_address ] )
Jon Hall5cf14d52015-07-16 12:15:19 -0700227 threads.append( t )
228 t.start()
229
230 for t in threads:
231 t.join()
232 cliResults = cliResults and t.result
233 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
234 onpass="ONOS cli startup successful",
235 onfail="ONOS cli startup failed" )
236
Jon Halla440e872016-03-31 15:15:50 -0700237 # Create a list of active nodes for use when some nodes are stopped
238 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
239
Jon Hall5cf14d52015-07-16 12:15:19 -0700240 if main.params[ 'tcpdump' ].lower() == "true":
241 main.step( "Start Packet Capture MN" )
242 main.Mininet2.startTcpdump(
243 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
244 + "-MN.pcap",
245 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
246 port=main.params[ 'MNtcpdump' ][ 'port' ] )
247
Jon Halla440e872016-03-31 15:15:50 -0700248 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700249 nodeResults = utilities.retry( main.HA.nodesCheck,
250 False,
Jon Hallf37d44d2017-05-24 10:37:30 -0700251 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -0700252 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700253
Jon Hall41d39f12016-04-11 22:54:35 -0700254 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700255 onpass="Nodes check successful",
256 onfail="Nodes check NOT successful" )
257
258 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700259 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700260 cli = main.CLIs[ i ]
Jon Halla440e872016-03-31 15:15:50 -0700261 main.log.debug( "{} components not ACTIVE: \n{}".format(
262 cli.name,
263 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700264 main.log.error( "Failed to start ONOS, stopping test" )
265 main.cleanup()
266 main.exit()
267
Jon Hall172b7ba2016-04-07 18:12:20 -0700268 main.step( "Activate apps defined in the params file" )
269 # get data from the params
270 apps = main.params.get( 'apps' )
271 if apps:
Jon Hallf37d44d2017-05-24 10:37:30 -0700272 apps = apps.split( ',' )
Jon Hall172b7ba2016-04-07 18:12:20 -0700273 main.log.warn( apps )
274 activateResult = True
275 for app in apps:
276 main.CLIs[ 0 ].app( app, "Activate" )
277 # TODO: check this worked
278 time.sleep( 10 ) # wait for apps to activate
279 for app in apps:
280 state = main.CLIs[ 0 ].appStatus( app )
281 if state == "ACTIVE":
Jon Hall937bc812017-01-31 16:44:10 -0800282 activateResult = activateResult and True
Jon Hall172b7ba2016-04-07 18:12:20 -0700283 else:
284 main.log.error( "{} is in {} state".format( app, state ) )
Jon Hall937bc812017-01-31 16:44:10 -0800285 activateResult = False
Jon Hall172b7ba2016-04-07 18:12:20 -0700286 utilities.assert_equals( expect=True,
287 actual=activateResult,
288 onpass="Successfully activated apps",
289 onfail="Failed to activate apps" )
290 else:
291 main.log.warn( "No apps were specified to be loaded after startup" )
292
293 main.step( "Set ONOS configurations" )
294 config = main.params.get( 'ONOS_Configuration' )
295 if config:
296 main.log.debug( config )
297 checkResult = main.TRUE
298 for component in config:
Jon Hallf37d44d2017-05-24 10:37:30 -0700299 for setting in config[ component ]:
300 value = config[ component ][ setting ]
Jon Hall172b7ba2016-04-07 18:12:20 -0700301 check = main.CLIs[ 0 ].setCfg( component, setting, value )
302 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
303 checkResult = check and checkResult
304 utilities.assert_equals( expect=main.TRUE,
305 actual=checkResult,
306 onpass="Successfully set config",
307 onfail="Failed to set config" )
308 else:
309 main.log.warn( "No configurations were specified to be changed after startup" )
310
Jon Hall9d2dcad2016-04-08 10:15:20 -0700311 main.step( "App Ids check" )
312 appCheck = main.TRUE
313 threads = []
314 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700315 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall9d2dcad2016-04-08 10:15:20 -0700316 name="appToIDCheck-" + str( i ),
317 args=[] )
318 threads.append( t )
319 t.start()
320
321 for t in threads:
322 t.join()
323 appCheck = appCheck and t.result
324 if appCheck != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700325 node = main.activeNodes[ 0 ]
326 main.log.warn( main.CLIs[ node ].apps() )
327 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700328 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
329 onpass="App Ids seem to be correct",
330 onfail="Something is wrong with app Ids" )
331
Jon Hall5cf14d52015-07-16 12:15:19 -0700332 def CASE2( self, main ):
333 """
334 Assign devices to controllers
335 """
336 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700337 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700338 assert main, "main not defined"
339 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700340 assert main.CLIs, "main.CLIs not defined"
341 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700342 assert ONOS1Port, "ONOS1Port not defined"
343 assert ONOS2Port, "ONOS2Port not defined"
344 assert ONOS3Port, "ONOS3Port not defined"
345 assert ONOS4Port, "ONOS4Port not defined"
346 assert ONOS5Port, "ONOS5Port not defined"
347 assert ONOS6Port, "ONOS6Port not defined"
348 assert ONOS7Port, "ONOS7Port not defined"
349
350 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700351 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700352 "and check that an ONOS node becomes the " +\
353 "master of the device."
354 main.step( "Assign switches to controllers" )
355
356 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700357 for i in range( main.numCtrls ):
358 ipList.append( main.nodes[ i ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700359 swList = []
360 for i in range( 1, 29 ):
361 swList.append( "s" + str( i ) )
362 main.Mininet1.assignSwController( sw=swList, ip=ipList )
363
364 mastershipCheck = main.TRUE
365 for i in range( 1, 29 ):
366 response = main.Mininet1.getSwController( "s" + str( i ) )
367 try:
368 main.log.info( str( response ) )
369 except Exception:
370 main.log.info( repr( response ) )
Jon Halle1a3b752015-07-22 13:02:46 -0700371 for node in main.nodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700372 if re.search( "tcp:" + node.ip_address, response ):
373 mastershipCheck = mastershipCheck and main.TRUE
374 else:
375 main.log.error( "Error, node " + node.ip_address + " is " +
376 "not in the list of controllers s" +
377 str( i ) + " is connecting to." )
378 mastershipCheck = main.FALSE
379 utilities.assert_equals(
380 expect=main.TRUE,
381 actual=mastershipCheck,
382 onpass="Switch mastership assigned correctly",
383 onfail="Switches not assigned correctly to controllers" )
384
385 def CASE21( self, main ):
386 """
387 Assign mastership to controllers
388 """
Jon Hall5cf14d52015-07-16 12:15:19 -0700389 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700390 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700391 assert main, "main not defined"
392 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700393 assert main.CLIs, "main.CLIs not defined"
394 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700395 assert ONOS1Port, "ONOS1Port not defined"
396 assert ONOS2Port, "ONOS2Port not defined"
397 assert ONOS3Port, "ONOS3Port not defined"
398 assert ONOS4Port, "ONOS4Port not defined"
399 assert ONOS5Port, "ONOS5Port not defined"
400 assert ONOS6Port, "ONOS6Port not defined"
401 assert ONOS7Port, "ONOS7Port not defined"
402
403 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700404 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700405 "device. Then manually assign" +\
406 " mastership to specific ONOS nodes using" +\
407 " 'device-role'"
408 main.step( "Assign mastership of switches to specific controllers" )
409 # Manually assign mastership to the controller we want
410 roleCall = main.TRUE
411
Jon Hallf37d44d2017-05-24 10:37:30 -0700412 ipList = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700413 deviceList = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700414 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700415 try:
416 # Assign mastership to specific controllers. This assignment was
417 # determined for a 7 node cluser, but will work with any sized
418 # cluster
419 for i in range( 1, 29 ): # switches 1 through 28
420 # set up correct variables:
421 if i == 1:
422 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700423 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700424 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700425 elif i == 2:
Jon Halle1a3b752015-07-22 13:02:46 -0700426 c = 1 % main.numCtrls
427 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700428 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700429 elif i == 3:
Jon Halle1a3b752015-07-22 13:02:46 -0700430 c = 1 % main.numCtrls
431 ip = main.nodes[ c ].ip_address # ONOS2
Jon Halla440e872016-03-31 15:15:50 -0700432 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700433 elif i == 4:
Jon Halle1a3b752015-07-22 13:02:46 -0700434 c = 3 % main.numCtrls
435 ip = main.nodes[ c ].ip_address # ONOS4
Jon Halla440e872016-03-31 15:15:50 -0700436 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700437 elif i == 5:
Jon Halle1a3b752015-07-22 13:02:46 -0700438 c = 2 % main.numCtrls
439 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700440 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700441 elif i == 6:
Jon Halle1a3b752015-07-22 13:02:46 -0700442 c = 2 % main.numCtrls
443 ip = main.nodes[ c ].ip_address # ONOS3
Jon Halla440e872016-03-31 15:15:50 -0700444 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700445 elif i == 7:
Jon Halle1a3b752015-07-22 13:02:46 -0700446 c = 5 % main.numCtrls
447 ip = main.nodes[ c ].ip_address # ONOS6
Jon Halla440e872016-03-31 15:15:50 -0700448 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700449 elif i >= 8 and i <= 17:
Jon Halle1a3b752015-07-22 13:02:46 -0700450 c = 4 % main.numCtrls
451 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall5cf14d52015-07-16 12:15:19 -0700452 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700453 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700454 elif i >= 18 and i <= 27:
Jon Halle1a3b752015-07-22 13:02:46 -0700455 c = 6 % main.numCtrls
456 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall5cf14d52015-07-16 12:15:19 -0700457 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700458 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700459 elif i == 28:
460 c = 0
Jon Halle1a3b752015-07-22 13:02:46 -0700461 ip = main.nodes[ c ].ip_address # ONOS1
Jon Halla440e872016-03-31 15:15:50 -0700462 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700463 else:
464 main.log.error( "You didn't write an else statement for " +
465 "switch s" + str( i ) )
466 roleCall = main.FALSE
467 # Assign switch
468 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
469 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700470 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
Jon Hall5cf14d52015-07-16 12:15:19 -0700471 ipList.append( ip )
472 deviceList.append( deviceId )
473 except ( AttributeError, AssertionError ):
474 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700475 main.log.info( onosCli.devices() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700476 utilities.assert_equals(
477 expect=main.TRUE,
478 actual=roleCall,
479 onpass="Re-assigned switch mastership to designated controller",
480 onfail="Something wrong with deviceRole calls" )
481
482 main.step( "Check mastership was correctly assigned" )
483 roleCheck = main.TRUE
484 # NOTE: This is due to the fact that device mastership change is not
485 # atomic and is actually a multi step process
486 time.sleep( 5 )
487 for i in range( len( ipList ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -0700488 ip = ipList[ i ]
489 deviceId = deviceList[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700490 # Check assignment
Jon Halla440e872016-03-31 15:15:50 -0700491 master = onosCli.getRole( deviceId ).get( 'master' )
Jon Hall5cf14d52015-07-16 12:15:19 -0700492 if ip in master:
493 roleCheck = roleCheck and main.TRUE
494 else:
495 roleCheck = roleCheck and main.FALSE
496 main.log.error( "Error, controller " + ip + " is not" +
497 " master " + "of device " +
498 str( deviceId ) + ". Master is " +
499 repr( master ) + "." )
500 utilities.assert_equals(
501 expect=main.TRUE,
502 actual=roleCheck,
503 onpass="Switches were successfully reassigned to designated " +
504 "controller",
505 onfail="Switches were not successfully reassigned" )
506
507 def CASE3( self, main ):
508 """
509 Assign intents
510 """
511 import time
512 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700513 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700514 assert main, "main not defined"
515 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700516 assert main.CLIs, "main.CLIs not defined"
517 assert main.nodes, "main.nodes not defined"
Jon Halla440e872016-03-31 15:15:50 -0700518 try:
519 labels
520 except NameError:
521 main.log.error( "labels not defined, setting to []" )
522 labels = []
523 try:
524 data
525 except NameError:
526 main.log.error( "data not defined, setting to []" )
527 data = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700528 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700529 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700530 "assign predetermined host-to-host intents." +\
531 " After installation, check that the intent" +\
532 " is distributed to all nodes and the state" +\
533 " is INSTALLED"
534
535 # install onos-app-fwd
536 main.step( "Install reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700537 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Halla440e872016-03-31 15:15:50 -0700538 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700539 utilities.assert_equals( expect=main.TRUE, actual=installResults,
540 onpass="Install fwd successful",
541 onfail="Install fwd failed" )
542
543 main.step( "Check app ids" )
544 appCheck = main.TRUE
545 threads = []
Jon Halla440e872016-03-31 15:15:50 -0700546 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700547 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700548 name="appToIDCheck-" + str( i ),
549 args=[] )
550 threads.append( t )
551 t.start()
552
553 for t in threads:
554 t.join()
555 appCheck = appCheck and t.result
556 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700557 main.log.warn( onosCli.apps() )
558 main.log.warn( onosCli.appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700559 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
560 onpass="App Ids seem to be correct",
561 onfail="Something is wrong with app Ids" )
562
563 main.step( "Discovering Hosts( Via pingall for now )" )
564 # FIXME: Once we have a host discovery mechanism, use that instead
565 # REACTIVE FWD test
566 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700567 passMsg = "Reactive Pingall test passed"
568 time1 = time.time()
569 pingResult = main.Mininet1.pingall()
570 time2 = time.time()
571 if not pingResult:
Jon Hallf37d44d2017-05-24 10:37:30 -0700572 main.log.warn( "First pingall failed. Trying again..." )
Jon Hall5cf14d52015-07-16 12:15:19 -0700573 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700574 passMsg += " on the second try"
575 utilities.assert_equals(
576 expect=main.TRUE,
577 actual=pingResult,
Jon Hallf37d44d2017-05-24 10:37:30 -0700578 onpass=passMsg,
Jon Hall96091e62015-09-21 17:34:17 -0700579 onfail="Reactive Pingall failed, " +
580 "one or more ping pairs failed" )
581 main.log.info( "Time for pingall: %2f seconds" %
582 ( time2 - time1 ) )
Jon Halld2871c22016-07-26 11:01:14 -0700583 if not pingResult:
584 main.cleanup()
585 main.exit()
Jon Hall5cf14d52015-07-16 12:15:19 -0700586 # timeout for fwd flows
587 time.sleep( 11 )
588 # uninstall onos-app-fwd
589 main.step( "Uninstall reactive forwarding app" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700590 node = main.activeNodes[ 0 ]
591 uninstallResult = main.CLIs[ node ].deactivateApp( "org.onosproject.fwd" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700592 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
593 onpass="Uninstall fwd successful",
594 onfail="Uninstall fwd failed" )
Jon Hall5cf14d52015-07-16 12:15:19 -0700595
596 main.step( "Check app ids" )
597 threads = []
598 appCheck2 = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700599 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700600 t = main.Thread( target=main.CLIs[ i ].appToIDCheck,
Jon Hall5cf14d52015-07-16 12:15:19 -0700601 name="appToIDCheck-" + str( i ),
602 args=[] )
603 threads.append( t )
604 t.start()
605
606 for t in threads:
607 t.join()
608 appCheck2 = appCheck2 and t.result
609 if appCheck2 != main.TRUE:
Jon Hallf37d44d2017-05-24 10:37:30 -0700610 node = main.activeNodes[ 0 ]
611 main.log.warn( main.CLIs[ node ].apps() )
612 main.log.warn( main.CLIs[ node ].appIDs() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700613 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
614 onpass="App Ids seem to be correct",
615 onfail="Something is wrong with app Ids" )
616
617 main.step( "Add host intents via cli" )
618 intentIds = []
Jon Hall6e709752016-02-01 13:38:46 -0800619 # TODO: move the host numbers to params
620 # Maybe look at all the paths we ping?
Jon Hall5cf14d52015-07-16 12:15:19 -0700621 intentAddResult = True
622 hostResult = main.TRUE
623 for i in range( 8, 18 ):
624 main.log.info( "Adding host intent between h" + str( i ) +
625 " and h" + str( i + 10 ) )
626 host1 = "00:00:00:00:00:" + \
627 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
628 host2 = "00:00:00:00:00:" + \
629 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
630 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700631 host1Dict = onosCli.getHost( host1 )
632 host2Dict = onosCli.getHost( host2 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700633 host1Id = None
634 host2Id = None
635 if host1Dict and host2Dict:
636 host1Id = host1Dict.get( 'id', None )
637 host2Id = host2Dict.get( 'id', None )
638 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700639 nodeNum = ( i % len( main.activeNodes ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700640 node = main.activeNodes[ nodeNum ]
641 tmpId = main.CLIs[ node ].addHostIntent( host1Id, host2Id )
Jon Hall5cf14d52015-07-16 12:15:19 -0700642 if tmpId:
643 main.log.info( "Added intent with id: " + tmpId )
644 intentIds.append( tmpId )
645 else:
646 main.log.error( "addHostIntent returned: " +
647 repr( tmpId ) )
648 else:
649 main.log.error( "Error, getHost() failed for h" + str( i ) +
650 " and/or h" + str( i + 10 ) )
Jon Hallf37d44d2017-05-24 10:37:30 -0700651 node = main.activeNodes[ 0 ]
652 hosts = main.CLIs[ node ].hosts()
Jon Hall5cf14d52015-07-16 12:15:19 -0700653 main.log.warn( "Hosts output: " )
654 try:
655 main.log.warn( json.dumps( json.loads( hosts ),
656 sort_keys=True,
657 indent=4,
658 separators=( ',', ': ' ) ) )
659 except ( ValueError, TypeError ):
660 main.log.warn( repr( hosts ) )
661 hostResult = main.FALSE
662 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
663 onpass="Found a host id for each host",
664 onfail="Error looking up host ids" )
665
666 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700667 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700668 main.log.info( "Submitted intents: " + str( intentIds ) )
669 main.log.info( "Intents in ONOS: " + str( onosIds ) )
670 for intent in intentIds:
671 if intent in onosIds:
672 pass # intent submitted is in onos
673 else:
674 intentAddResult = False
675 if intentAddResult:
676 intentStop = time.time()
677 else:
678 intentStop = None
679 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700680 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700681 intentStates = []
682 installedCheck = True
683 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
684 count = 0
685 try:
686 for intent in json.loads( intents ):
687 state = intent.get( 'state', None )
688 if "INSTALLED" not in state:
689 installedCheck = False
690 intentId = intent.get( 'id', None )
691 intentStates.append( ( intentId, state ) )
692 except ( ValueError, TypeError ):
693 main.log.exception( "Error parsing intents" )
694 # add submitted intents not in the store
695 tmplist = [ i for i, s in intentStates ]
696 missingIntents = False
697 for i in intentIds:
698 if i not in tmplist:
699 intentStates.append( ( i, " - " ) )
700 missingIntents = True
701 intentStates.sort()
702 for i, s in intentStates:
703 count += 1
704 main.log.info( "%-6s%-15s%-15s" %
705 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700706 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700707 try:
708 missing = False
709 if leaders:
710 parsedLeaders = json.loads( leaders )
711 main.log.warn( json.dumps( parsedLeaders,
712 sort_keys=True,
713 indent=4,
714 separators=( ',', ': ' ) ) )
715 # check for all intent partitions
716 topics = []
717 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700718 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700719 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700720 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700721 for topic in topics:
722 if topic not in ONOStopics:
723 main.log.error( "Error: " + topic +
724 " not in leaders" )
725 missing = True
726 else:
727 main.log.error( "leaders() returned None" )
728 except ( ValueError, TypeError ):
729 main.log.exception( "Error parsing leaders" )
730 main.log.error( repr( leaders ) )
731 # Check all nodes
732 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700733 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700734 response = main.CLIs[ i ].leaders( jsonFormat=False )
735 main.log.warn( str( main.CLIs[ i ].name ) + " leaders output: \n" +
Jon Hall5cf14d52015-07-16 12:15:19 -0700736 str( response ) )
737
Jon Halla440e872016-03-31 15:15:50 -0700738 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700739 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700740 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -0700741 parsedPartitions = json.loads( partitions )
742 main.log.warn( json.dumps( parsedPartitions,
743 sort_keys=True,
744 indent=4,
745 separators=( ',', ': ' ) ) )
746 # TODO check for a leader in all paritions
747 # TODO check for consistency among nodes
748 else:
749 main.log.error( "partitions() returned None" )
750 except ( ValueError, TypeError ):
751 main.log.exception( "Error parsing partitions" )
752 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700753 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700754 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700755 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -0700756 parsedPending = json.loads( pendingMap )
757 main.log.warn( json.dumps( parsedPending,
758 sort_keys=True,
759 indent=4,
760 separators=( ',', ': ' ) ) )
761 # TODO check something here?
762 else:
763 main.log.error( "pendingMap() returned None" )
764 except ( ValueError, TypeError ):
765 main.log.exception( "Error parsing pending map" )
766 main.log.error( repr( pendingMap ) )
767
768 intentAddResult = bool( intentAddResult and not missingIntents and
769 installedCheck )
770 if not intentAddResult:
771 main.log.error( "Error in pushing host intents to ONOS" )
772
773 main.step( "Intent Anti-Entropy dispersion" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700774 for j in range( 100 ):
Jon Hall5cf14d52015-07-16 12:15:19 -0700775 correct = True
776 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700777 for i in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -0700778 onosIds = []
Jon Hallf37d44d2017-05-24 10:37:30 -0700779 ids = main.CLIs[ i ].getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700780 onosIds.append( ids )
Jon Hallf37d44d2017-05-24 10:37:30 -0700781 main.log.debug( "Intents in " + main.CLIs[ i ].name + ": " +
Jon Hall5cf14d52015-07-16 12:15:19 -0700782 str( sorted( onosIds ) ) )
783 if sorted( ids ) != sorted( intentIds ):
784 main.log.warn( "Set of intent IDs doesn't match" )
785 correct = False
786 break
787 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700788 intents = json.loads( main.CLIs[ i ].intents() )
Jon Hall5cf14d52015-07-16 12:15:19 -0700789 for intent in intents:
790 if intent[ 'state' ] != "INSTALLED":
791 main.log.warn( "Intent " + intent[ 'id' ] +
792 " is " + intent[ 'state' ] )
793 correct = False
794 break
795 if correct:
796 break
797 else:
Jon Hallf37d44d2017-05-24 10:37:30 -0700798 time.sleep( 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -0700799 if not intentStop:
800 intentStop = time.time()
801 global gossipTime
802 gossipTime = intentStop - intentStart
803 main.log.info( "It took about " + str( gossipTime ) +
804 " seconds for all intents to appear in each node" )
Jon Hallf37d44d2017-05-24 10:37:30 -0700805 gossipPeriod = int( main.params[ 'timers' ][ 'gossip' ] )
Jon Halla440e872016-03-31 15:15:50 -0700806 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall5cf14d52015-07-16 12:15:19 -0700807 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700808 expect=maxGossipTime, actual=gossipTime,
Jon Hall5cf14d52015-07-16 12:15:19 -0700809 onpass="ECM anti-entropy for intents worked within " +
810 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700811 onfail="Intent ECM anti-entropy took too long. " +
812 "Expected time:{}, Actual time:{}".format( maxGossipTime,
813 gossipTime ) )
814 if gossipTime <= maxGossipTime:
Jon Hall5cf14d52015-07-16 12:15:19 -0700815 intentAddResult = True
816
817 if not intentAddResult or "key" in pendingMap:
818 import time
819 installedCheck = True
820 main.log.info( "Sleeping 60 seconds to see if intents are found" )
821 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700822 onosIds = onosCli.getAllIntentsId()
Jon Hall5cf14d52015-07-16 12:15:19 -0700823 main.log.info( "Submitted intents: " + str( intentIds ) )
824 main.log.info( "Intents in ONOS: " + str( onosIds ) )
825 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700826 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700827 intentStates = []
828 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
829 count = 0
830 try:
831 for intent in json.loads( intents ):
832 # Iter through intents of a node
833 state = intent.get( 'state', None )
834 if "INSTALLED" not in state:
835 installedCheck = False
836 intentId = intent.get( 'id', None )
837 intentStates.append( ( intentId, state ) )
838 except ( ValueError, TypeError ):
839 main.log.exception( "Error parsing intents" )
840 # add submitted intents not in the store
841 tmplist = [ i for i, s in intentStates ]
842 for i in intentIds:
843 if i not in tmplist:
844 intentStates.append( ( i, " - " ) )
845 intentStates.sort()
846 for i, s in intentStates:
847 count += 1
848 main.log.info( "%-6s%-15s%-15s" %
849 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700850 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -0700851 try:
852 missing = False
853 if leaders:
854 parsedLeaders = json.loads( leaders )
855 main.log.warn( json.dumps( parsedLeaders,
856 sort_keys=True,
857 indent=4,
858 separators=( ',', ': ' ) ) )
859 # check for all intent partitions
860 # check for election
861 topics = []
862 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700863 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -0700864 # FIXME: this should only be after we start the app
865 topics.append( "org.onosproject.election" )
866 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -0700867 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700868 for topic in topics:
869 if topic not in ONOStopics:
870 main.log.error( "Error: " + topic +
871 " not in leaders" )
872 missing = True
873 else:
874 main.log.error( "leaders() returned None" )
875 except ( ValueError, TypeError ):
876 main.log.exception( "Error parsing leaders" )
877 main.log.error( repr( leaders ) )
878 # Check all nodes
879 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700880 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -0700881 node = main.CLIs[ i ]
882 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -0700883 main.log.warn( str( node.name ) + " leaders output: \n" +
884 str( response ) )
885
Jon Halla440e872016-03-31 15:15:50 -0700886 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -0700887 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700888 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -0700889 parsedPartitions = json.loads( partitions )
890 main.log.warn( json.dumps( parsedPartitions,
891 sort_keys=True,
892 indent=4,
893 separators=( ',', ': ' ) ) )
894 # TODO check for a leader in all paritions
895 # TODO check for consistency among nodes
896 else:
897 main.log.error( "partitions() returned None" )
898 except ( ValueError, TypeError ):
899 main.log.exception( "Error parsing partitions" )
900 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700901 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -0700902 try:
Jon Hallf37d44d2017-05-24 10:37:30 -0700903 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -0700904 parsedPending = json.loads( pendingMap )
905 main.log.warn( json.dumps( parsedPending,
906 sort_keys=True,
907 indent=4,
908 separators=( ',', ': ' ) ) )
909 # TODO check something here?
910 else:
911 main.log.error( "pendingMap() returned None" )
912 except ( ValueError, TypeError ):
913 main.log.exception( "Error parsing pending map" )
914 main.log.error( repr( pendingMap ) )
915
916 def CASE4( self, main ):
917 """
918 Ping across added host intents
919 """
920 import json
921 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700922 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -0700923 assert main, "main not defined"
924 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700925 assert main.CLIs, "main.CLIs not defined"
926 assert main.nodes, "main.nodes not defined"
Jon Halla440e872016-03-31 15:15:50 -0700927 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700928 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall5cf14d52015-07-16 12:15:19 -0700929 "functionality and check the state of " +\
930 "the intent"
Jon Hall5cf14d52015-07-16 12:15:19 -0700931
Jon Hallf37d44d2017-05-24 10:37:30 -0700932 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -0700933 main.step( "Check Intent state" )
934 installedCheck = False
935 loopCount = 0
936 while not installedCheck and loopCount < 40:
937 installedCheck = True
938 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700939 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -0700940 intentStates = []
941 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
942 count = 0
943 # Iter through intents of a node
944 try:
945 for intent in json.loads( intents ):
946 state = intent.get( 'state', None )
947 if "INSTALLED" not in state:
948 installedCheck = False
949 intentId = intent.get( 'id', None )
950 intentStates.append( ( intentId, state ) )
951 except ( ValueError, TypeError ):
952 main.log.exception( "Error parsing intents." )
953 # Print states
954 intentStates.sort()
955 for i, s in intentStates:
956 count += 1
957 main.log.info( "%-6s%-15s%-15s" %
958 ( str( count ), str( i ), str( s ) ) )
959 if not installedCheck:
960 time.sleep( 1 )
961 loopCount += 1
962 utilities.assert_equals( expect=True, actual=installedCheck,
963 onpass="Intents are all INSTALLED",
964 onfail="Intents are not all in " +
965 "INSTALLED state" )
966
Jon Hall9d2dcad2016-04-08 10:15:20 -0700967 main.step( "Ping across added host intents" )
Jon Hall9d2dcad2016-04-08 10:15:20 -0700968 PingResult = main.TRUE
969 for i in range( 8, 18 ):
970 ping = main.Mininet1.pingHost( src="h" + str( i ),
971 target="h" + str( i + 10 ) )
972 PingResult = PingResult and ping
973 if ping == main.FALSE:
974 main.log.warn( "Ping failed between h" + str( i ) +
975 " and h" + str( i + 10 ) )
976 elif ping == main.TRUE:
977 main.log.info( "Ping test passed!" )
978 # Don't set PingResult or you'd override failures
979 if PingResult == main.FALSE:
980 main.log.error(
981 "Intents have not been installed correctly, pings failed." )
982 # TODO: pretty print
983 main.log.warn( "ONOS1 intents: " )
984 try:
985 tmpIntents = onosCli.intents()
986 main.log.warn( json.dumps( json.loads( tmpIntents ),
987 sort_keys=True,
988 indent=4,
989 separators=( ',', ': ' ) ) )
990 except ( ValueError, TypeError ):
991 main.log.warn( repr( tmpIntents ) )
992 utilities.assert_equals(
993 expect=main.TRUE,
994 actual=PingResult,
995 onpass="Intents have been installed correctly and pings work",
996 onfail="Intents have not been installed correctly, pings failed." )
997
Jon Hall5cf14d52015-07-16 12:15:19 -0700998 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -0700999 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001000 topicCheck = main.TRUE
1001 try:
1002 if leaders:
1003 parsedLeaders = json.loads( leaders )
1004 main.log.warn( json.dumps( parsedLeaders,
1005 sort_keys=True,
1006 indent=4,
1007 separators=( ',', ': ' ) ) )
1008 # check for all intent partitions
1009 # check for election
1010 # TODO: Look at Devices as topics now that it uses this system
1011 topics = []
1012 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001013 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001014 # FIXME: this should only be after we start the app
1015 # FIXME: topics.append( "org.onosproject.election" )
1016 # Print leaders output
1017 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -07001018 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -07001019 for topic in topics:
1020 if topic not in ONOStopics:
1021 main.log.error( "Error: " + topic +
1022 " not in leaders" )
1023 topicCheck = main.FALSE
1024 else:
1025 main.log.error( "leaders() returned None" )
1026 topicCheck = main.FALSE
1027 except ( ValueError, TypeError ):
1028 topicCheck = main.FALSE
1029 main.log.exception( "Error parsing leaders" )
1030 main.log.error( repr( leaders ) )
1031 # TODO: Check for a leader of these topics
1032 # Check all nodes
1033 if topicCheck:
Jon Halla440e872016-03-31 15:15:50 -07001034 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001035 node = main.CLIs[ i ]
1036 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -07001037 main.log.warn( str( node.name ) + " leaders output: \n" +
1038 str( response ) )
1039
1040 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
1041 onpass="intent Partitions is in leaders",
1042 onfail="Some topics were lost " )
1043 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -07001044 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001045 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001046 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -07001047 parsedPartitions = json.loads( partitions )
1048 main.log.warn( json.dumps( parsedPartitions,
1049 sort_keys=True,
1050 indent=4,
1051 separators=( ',', ': ' ) ) )
1052 # TODO check for a leader in all paritions
1053 # TODO check for consistency among nodes
1054 else:
1055 main.log.error( "partitions() returned None" )
1056 except ( ValueError, TypeError ):
1057 main.log.exception( "Error parsing partitions" )
1058 main.log.error( repr( partitions ) )
1059 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001060 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001061 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001062 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -07001063 parsedPending = json.loads( pendingMap )
1064 main.log.warn( json.dumps( parsedPending,
1065 sort_keys=True,
1066 indent=4,
1067 separators=( ',', ': ' ) ) )
1068 # TODO check something here?
1069 else:
1070 main.log.error( "pendingMap() returned None" )
1071 except ( ValueError, TypeError ):
1072 main.log.exception( "Error parsing pending map" )
1073 main.log.error( repr( pendingMap ) )
1074
1075 if not installedCheck:
1076 main.log.info( "Waiting 60 seconds to see if the state of " +
1077 "intents change" )
1078 time.sleep( 60 )
1079 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001080 intents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001081 intentStates = []
1082 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1083 count = 0
1084 # Iter through intents of a node
1085 try:
1086 for intent in json.loads( intents ):
1087 state = intent.get( 'state', None )
1088 if "INSTALLED" not in state:
1089 installedCheck = False
1090 intentId = intent.get( 'id', None )
1091 intentStates.append( ( intentId, state ) )
1092 except ( ValueError, TypeError ):
1093 main.log.exception( "Error parsing intents." )
1094 intentStates.sort()
1095 for i, s in intentStates:
1096 count += 1
1097 main.log.info( "%-6s%-15s%-15s" %
1098 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001099 leaders = onosCli.leaders()
Jon Hall5cf14d52015-07-16 12:15:19 -07001100 try:
1101 missing = False
1102 if leaders:
1103 parsedLeaders = json.loads( leaders )
1104 main.log.warn( json.dumps( parsedLeaders,
1105 sort_keys=True,
1106 indent=4,
1107 separators=( ',', ': ' ) ) )
1108 # check for all intent partitions
1109 # check for election
1110 topics = []
1111 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -07001112 topics.append( "work-partition-" + str( i ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001113 # FIXME: this should only be after we start the app
1114 topics.append( "org.onosproject.election" )
1115 main.log.debug( topics )
Jon Hallf37d44d2017-05-24 10:37:30 -07001116 ONOStopics = [ j[ 'topic' ] for j in parsedLeaders ]
Jon Hall5cf14d52015-07-16 12:15:19 -07001117 for topic in topics:
1118 if topic not in ONOStopics:
1119 main.log.error( "Error: " + topic +
1120 " not in leaders" )
1121 missing = True
1122 else:
1123 main.log.error( "leaders() returned None" )
1124 except ( ValueError, TypeError ):
1125 main.log.exception( "Error parsing leaders" )
1126 main.log.error( repr( leaders ) )
1127 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001128 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001129 node = main.CLIs[ i ]
1130 response = node.leaders( jsonFormat=False )
Jon Hall5cf14d52015-07-16 12:15:19 -07001131 main.log.warn( str( node.name ) + " leaders output: \n" +
1132 str( response ) )
1133
Jon Halla440e872016-03-31 15:15:50 -07001134 partitions = onosCli.partitions()
Jon Hall5cf14d52015-07-16 12:15:19 -07001135 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001136 if partitions:
Jon Hall5cf14d52015-07-16 12:15:19 -07001137 parsedPartitions = json.loads( partitions )
1138 main.log.warn( json.dumps( parsedPartitions,
1139 sort_keys=True,
1140 indent=4,
1141 separators=( ',', ': ' ) ) )
1142 # TODO check for a leader in all paritions
1143 # TODO check for consistency among nodes
1144 else:
1145 main.log.error( "partitions() returned None" )
1146 except ( ValueError, TypeError ):
1147 main.log.exception( "Error parsing partitions" )
1148 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001149 pendingMap = onosCli.pendingMap()
Jon Hall5cf14d52015-07-16 12:15:19 -07001150 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001151 if pendingMap:
Jon Hall5cf14d52015-07-16 12:15:19 -07001152 parsedPending = json.loads( pendingMap )
1153 main.log.warn( json.dumps( parsedPending,
1154 sort_keys=True,
1155 indent=4,
1156 separators=( ',', ': ' ) ) )
1157 # TODO check something here?
1158 else:
1159 main.log.error( "pendingMap() returned None" )
1160 except ( ValueError, TypeError ):
1161 main.log.exception( "Error parsing pending map" )
1162 main.log.error( repr( pendingMap ) )
1163 # Print flowrules
Jon Hallf37d44d2017-05-24 10:37:30 -07001164 node = main.activeNodes[ 0 ]
1165 main.log.debug( main.CLIs[ node ].flows( jsonFormat=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001166 main.step( "Wait a minute then ping again" )
1167 # the wait is above
1168 PingResult = main.TRUE
1169 for i in range( 8, 18 ):
1170 ping = main.Mininet1.pingHost( src="h" + str( i ),
1171 target="h" + str( i + 10 ) )
1172 PingResult = PingResult and ping
1173 if ping == main.FALSE:
1174 main.log.warn( "Ping failed between h" + str( i ) +
1175 " and h" + str( i + 10 ) )
1176 elif ping == main.TRUE:
1177 main.log.info( "Ping test passed!" )
1178 # Don't set PingResult or you'd override failures
1179 if PingResult == main.FALSE:
1180 main.log.error(
1181 "Intents have not been installed correctly, pings failed." )
1182 # TODO: pretty print
1183 main.log.warn( "ONOS1 intents: " )
1184 try:
Jon Halla440e872016-03-31 15:15:50 -07001185 tmpIntents = onosCli.intents()
Jon Hall5cf14d52015-07-16 12:15:19 -07001186 main.log.warn( json.dumps( json.loads( tmpIntents ),
1187 sort_keys=True,
1188 indent=4,
1189 separators=( ',', ': ' ) ) )
1190 except ( ValueError, TypeError ):
1191 main.log.warn( repr( tmpIntents ) )
1192 utilities.assert_equals(
1193 expect=main.TRUE,
1194 actual=PingResult,
1195 onpass="Intents have been installed correctly and pings work",
1196 onfail="Intents have not been installed correctly, pings failed." )
1197
1198 def CASE5( self, main ):
1199 """
1200 Reading state of ONOS
1201 """
1202 import json
1203 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001204 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001205 assert main, "main not defined"
1206 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001207 assert main.CLIs, "main.CLIs not defined"
1208 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001209
1210 main.case( "Setting up and gathering data for current state" )
1211 # The general idea for this test case is to pull the state of
1212 # ( intents,flows, topology,... ) from each ONOS node
1213 # We can then compare them with each other and also with past states
1214
1215 main.step( "Check that each switch has a master" )
1216 global mastershipState
1217 mastershipState = '[]'
1218
1219 # Assert that each device has a master
1220 rolesNotNull = main.TRUE
1221 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001222 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001223 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001224 name="rolesNotNull-" + str( i ),
1225 args=[] )
1226 threads.append( t )
1227 t.start()
1228
1229 for t in threads:
1230 t.join()
1231 rolesNotNull = rolesNotNull and t.result
1232 utilities.assert_equals(
1233 expect=main.TRUE,
1234 actual=rolesNotNull,
1235 onpass="Each device has a master",
1236 onfail="Some devices don't have a master assigned" )
1237
1238 main.step( "Get the Mastership of each switch from each controller" )
1239 ONOSMastership = []
1240 mastershipCheck = main.FALSE
1241 consistentMastership = True
1242 rolesResults = True
1243 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001244 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001245 t = main.Thread( target=main.CLIs[ i ].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001246 name="roles-" + str( i ),
1247 args=[] )
1248 threads.append( t )
1249 t.start()
1250
1251 for t in threads:
1252 t.join()
1253 ONOSMastership.append( t.result )
1254
Jon Halla440e872016-03-31 15:15:50 -07001255 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001256 node = str( main.activeNodes[ i ] + 1 )
1257 if not ONOSMastership[ i ] or "Error" in ONOSMastership[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07001258 main.log.error( "Error in getting ONOS" + node + " roles" )
1259 main.log.warn( "ONOS" + node + " mastership response: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001260 repr( ONOSMastership[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001261 rolesResults = False
1262 utilities.assert_equals(
1263 expect=True,
1264 actual=rolesResults,
1265 onpass="No error in reading roles output",
1266 onfail="Error in reading roles from ONOS" )
1267
1268 main.step( "Check for consistency in roles from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001269 if all( [ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001270 main.log.info(
1271 "Switch roles are consistent across all ONOS nodes" )
1272 else:
1273 consistentMastership = False
1274 utilities.assert_equals(
1275 expect=True,
1276 actual=consistentMastership,
1277 onpass="Switch roles are consistent across all ONOS nodes",
1278 onfail="ONOS nodes have different views of switch roles" )
1279
1280 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001281 for i in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001282 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001283 try:
1284 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001285 "ONOS" + node + " roles: ",
Jon Hall5cf14d52015-07-16 12:15:19 -07001286 json.dumps(
1287 json.loads( ONOSMastership[ i ] ),
1288 sort_keys=True,
1289 indent=4,
1290 separators=( ',', ': ' ) ) )
1291 except ( ValueError, TypeError ):
1292 main.log.warn( repr( ONOSMastership[ i ] ) )
1293 elif rolesResults and consistentMastership:
1294 mastershipCheck = main.TRUE
1295 mastershipState = ONOSMastership[ 0 ]
1296
1297 main.step( "Get the intents from each controller" )
1298 global intentState
1299 intentState = []
1300 ONOSIntents = []
1301 intentCheck = main.FALSE
1302 consistentIntents = True
1303 intentsResults = True
1304 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001305 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001306 t = main.Thread( target=main.CLIs[ i ].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001307 name="intents-" + str( i ),
1308 args=[],
1309 kwargs={ 'jsonFormat': True } )
1310 threads.append( t )
1311 t.start()
1312
1313 for t in threads:
1314 t.join()
1315 ONOSIntents.append( t.result )
1316
Jon Halla440e872016-03-31 15:15:50 -07001317 for i in range( len( ONOSIntents ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001318 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001319 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07001320 main.log.error( "Error in getting ONOS" + node + " intents" )
1321 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001322 repr( ONOSIntents[ i ] ) )
1323 intentsResults = False
1324 utilities.assert_equals(
1325 expect=True,
1326 actual=intentsResults,
1327 onpass="No error in reading intents output",
1328 onfail="Error in reading intents from ONOS" )
1329
1330 main.step( "Check for consistency in Intents from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001331 if all( [ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001332 main.log.info( "Intents are consistent across all ONOS " +
1333 "nodes" )
1334 else:
1335 consistentIntents = False
1336 main.log.error( "Intents not consistent" )
1337 utilities.assert_equals(
1338 expect=True,
1339 actual=consistentIntents,
1340 onpass="Intents are consistent across all ONOS nodes",
1341 onfail="ONOS nodes have different views of intents" )
1342
1343 if intentsResults:
1344 # Try to make it easy to figure out what is happening
1345 #
1346 # Intent ONOS1 ONOS2 ...
1347 # 0x01 INSTALLED INSTALLING
1348 # ... ... ...
1349 # ... ... ...
1350 title = " Id"
Jon Halla440e872016-03-31 15:15:50 -07001351 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001352 title += " " * 10 + "ONOS" + str( n + 1 )
1353 main.log.warn( title )
Jon Halle1a3b752015-07-22 13:02:46 -07001354 # get all intent keys in the cluster
Jon Hall5cf14d52015-07-16 12:15:19 -07001355 keys = []
1356 try:
1357 # Get the set of all intent keys
1358 for nodeStr in ONOSIntents:
1359 node = json.loads( nodeStr )
1360 for intent in node:
1361 keys.append( intent.get( 'id' ) )
1362 keys = set( keys )
1363 # For each intent key, print the state on each node
1364 for key in keys:
1365 row = "%-13s" % key
1366 for nodeStr in ONOSIntents:
1367 node = json.loads( nodeStr )
1368 for intent in node:
1369 if intent.get( 'id', "Error" ) == key:
1370 row += "%-15s" % intent.get( 'state' )
1371 main.log.warn( row )
1372 # End of intent state table
1373 except ValueError as e:
1374 main.log.exception( e )
1375 main.log.debug( "nodeStr was: " + repr( nodeStr ) )
1376
1377 if intentsResults and not consistentIntents:
1378 # print the json objects
Jon Hallf37d44d2017-05-24 10:37:30 -07001379 n = str( main.activeNodes[ -1 ] + 1 )
Jon Halla440e872016-03-31 15:15:50 -07001380 main.log.debug( "ONOS" + n + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001381 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1382 sort_keys=True,
1383 indent=4,
1384 separators=( ',', ': ' ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001385 for i in range( len( ONOSIntents ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001386 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001387 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07001388 main.log.debug( "ONOS" + node + " intents: " )
Jon Hallf37d44d2017-05-24 10:37:30 -07001389 main.log.debug( json.dumps( json.loads( ONOSIntents[ i ] ),
Jon Hall5cf14d52015-07-16 12:15:19 -07001390 sort_keys=True,
1391 indent=4,
1392 separators=( ',', ': ' ) ) )
1393 else:
Jon Halla440e872016-03-31 15:15:50 -07001394 main.log.debug( "ONOS" + node + " intents match ONOS" +
1395 n + " intents" )
Jon Hall5cf14d52015-07-16 12:15:19 -07001396 elif intentsResults and consistentIntents:
1397 intentCheck = main.TRUE
1398 intentState = ONOSIntents[ 0 ]
1399
1400 main.step( "Get the flows from each controller" )
1401 global flowState
1402 flowState = []
1403 ONOSFlows = []
1404 ONOSFlowsJson = []
1405 flowCheck = main.FALSE
1406 consistentFlows = True
1407 flowsResults = True
1408 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001409 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001410 t = main.Thread( target=main.CLIs[ i ].flows,
Jon Hall5cf14d52015-07-16 12:15:19 -07001411 name="flows-" + str( i ),
1412 args=[],
1413 kwargs={ 'jsonFormat': True } )
1414 threads.append( t )
1415 t.start()
1416
1417 # NOTE: Flows command can take some time to run
Jon Hallf37d44d2017-05-24 10:37:30 -07001418 time.sleep( 30 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001419 for t in threads:
1420 t.join()
1421 result = t.result
1422 ONOSFlows.append( result )
1423
Jon Halla440e872016-03-31 15:15:50 -07001424 for i in range( len( ONOSFlows ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001425 num = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001426 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1427 main.log.error( "Error in getting ONOS" + num + " flows" )
1428 main.log.warn( "ONOS" + num + " flows response: " +
1429 repr( ONOSFlows[ i ] ) )
1430 flowsResults = False
1431 ONOSFlowsJson.append( None )
1432 else:
1433 try:
1434 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1435 except ( ValueError, TypeError ):
1436 # FIXME: change this to log.error?
1437 main.log.exception( "Error in parsing ONOS" + num +
1438 " response as json." )
1439 main.log.error( repr( ONOSFlows[ i ] ) )
1440 ONOSFlowsJson.append( None )
1441 flowsResults = False
1442 utilities.assert_equals(
1443 expect=True,
1444 actual=flowsResults,
1445 onpass="No error in reading flows output",
1446 onfail="Error in reading flows from ONOS" )
1447
1448 main.step( "Check for consistency in Flows from each controller" )
1449 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1450 if all( tmp ):
1451 main.log.info( "Flow count is consistent across all ONOS nodes" )
1452 else:
1453 consistentFlows = False
1454 utilities.assert_equals(
1455 expect=True,
1456 actual=consistentFlows,
1457 onpass="The flow count is consistent across all ONOS nodes",
1458 onfail="ONOS nodes have different flow counts" )
1459
1460 if flowsResults and not consistentFlows:
Jon Halla440e872016-03-31 15:15:50 -07001461 for i in range( len( ONOSFlows ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001462 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001463 try:
1464 main.log.warn(
Jon Halla440e872016-03-31 15:15:50 -07001465 "ONOS" + node + " flows: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001466 json.dumps( json.loads( ONOSFlows[ i ] ), sort_keys=True,
Jon Hall5cf14d52015-07-16 12:15:19 -07001467 indent=4, separators=( ',', ': ' ) ) )
1468 except ( ValueError, TypeError ):
Jon Halla440e872016-03-31 15:15:50 -07001469 main.log.warn( "ONOS" + node + " flows: " +
1470 repr( ONOSFlows[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001471 elif flowsResults and consistentFlows:
1472 flowCheck = main.TRUE
1473 flowState = ONOSFlows[ 0 ]
1474
1475 main.step( "Get the OF Table entries" )
1476 global flows
1477 flows = []
1478 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001479 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001480 if flowCheck == main.FALSE:
1481 for table in flows:
1482 main.log.warn( table )
1483 # TODO: Compare switch flow tables with ONOS flow tables
1484
1485 main.step( "Start continuous pings" )
1486 main.Mininet2.pingLong(
1487 src=main.params[ 'PING' ][ 'source1' ],
1488 target=main.params[ 'PING' ][ 'target1' ],
1489 pingTime=500 )
1490 main.Mininet2.pingLong(
1491 src=main.params[ 'PING' ][ 'source2' ],
1492 target=main.params[ 'PING' ][ 'target2' ],
1493 pingTime=500 )
1494 main.Mininet2.pingLong(
1495 src=main.params[ 'PING' ][ 'source3' ],
1496 target=main.params[ 'PING' ][ 'target3' ],
1497 pingTime=500 )
1498 main.Mininet2.pingLong(
1499 src=main.params[ 'PING' ][ 'source4' ],
1500 target=main.params[ 'PING' ][ 'target4' ],
1501 pingTime=500 )
1502 main.Mininet2.pingLong(
1503 src=main.params[ 'PING' ][ 'source5' ],
1504 target=main.params[ 'PING' ][ 'target5' ],
1505 pingTime=500 )
1506 main.Mininet2.pingLong(
1507 src=main.params[ 'PING' ][ 'source6' ],
1508 target=main.params[ 'PING' ][ 'target6' ],
1509 pingTime=500 )
1510 main.Mininet2.pingLong(
1511 src=main.params[ 'PING' ][ 'source7' ],
1512 target=main.params[ 'PING' ][ 'target7' ],
1513 pingTime=500 )
1514 main.Mininet2.pingLong(
1515 src=main.params[ 'PING' ][ 'source8' ],
1516 target=main.params[ 'PING' ][ 'target8' ],
1517 pingTime=500 )
1518 main.Mininet2.pingLong(
1519 src=main.params[ 'PING' ][ 'source9' ],
1520 target=main.params[ 'PING' ][ 'target9' ],
1521 pingTime=500 )
1522 main.Mininet2.pingLong(
1523 src=main.params[ 'PING' ][ 'source10' ],
1524 target=main.params[ 'PING' ][ 'target10' ],
1525 pingTime=500 )
1526
1527 main.step( "Collecting topology information from ONOS" )
1528 devices = []
1529 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001530 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001531 t = main.Thread( target=main.CLIs[ i ].devices,
Jon Hall5cf14d52015-07-16 12:15:19 -07001532 name="devices-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001533 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001534 threads.append( t )
1535 t.start()
1536
1537 for t in threads:
1538 t.join()
1539 devices.append( t.result )
1540 hosts = []
1541 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001542 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001543 t = main.Thread( target=main.CLIs[ i ].hosts,
Jon Hall5cf14d52015-07-16 12:15:19 -07001544 name="hosts-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001545 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001546 threads.append( t )
1547 t.start()
1548
1549 for t in threads:
1550 t.join()
1551 try:
1552 hosts.append( json.loads( t.result ) )
1553 except ( ValueError, TypeError ):
1554 # FIXME: better handling of this, print which node
1555 # Maybe use thread name?
1556 main.log.exception( "Error parsing json output of hosts" )
Jon Hallf3d16e72015-12-16 17:45:08 -08001557 main.log.warn( repr( t.result ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001558 hosts.append( None )
1559
1560 ports = []
1561 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001562 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001563 t = main.Thread( target=main.CLIs[ i ].ports,
Jon Hall5cf14d52015-07-16 12:15:19 -07001564 name="ports-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001565 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001566 threads.append( t )
1567 t.start()
1568
1569 for t in threads:
1570 t.join()
1571 ports.append( t.result )
1572 links = []
1573 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001574 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001575 t = main.Thread( target=main.CLIs[ i ].links,
Jon Hall5cf14d52015-07-16 12:15:19 -07001576 name="links-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001577 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001578 threads.append( t )
1579 t.start()
1580
1581 for t in threads:
1582 t.join()
1583 links.append( t.result )
1584 clusters = []
1585 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001586 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001587 t = main.Thread( target=main.CLIs[ i ].clusters,
Jon Hall5cf14d52015-07-16 12:15:19 -07001588 name="clusters-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001589 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001590 threads.append( t )
1591 t.start()
1592
1593 for t in threads:
1594 t.join()
1595 clusters.append( t.result )
1596 # Compare json objects for hosts and dataplane clusters
1597
1598 # hosts
1599 main.step( "Host view is consistent across ONOS nodes" )
1600 consistentHostsResult = main.TRUE
1601 for controller in range( len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001602 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001603 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001604 if hosts[ controller ] == hosts[ 0 ]:
1605 continue
1606 else: # hosts not consistent
1607 main.log.error( "hosts from ONOS" +
1608 controllerStr +
1609 " is inconsistent with ONOS1" )
1610 main.log.warn( repr( hosts[ controller ] ) )
1611 consistentHostsResult = main.FALSE
1612
1613 else:
1614 main.log.error( "Error in getting ONOS hosts from ONOS" +
1615 controllerStr )
1616 consistentHostsResult = main.FALSE
1617 main.log.warn( "ONOS" + controllerStr +
1618 " hosts response: " +
1619 repr( hosts[ controller ] ) )
1620 utilities.assert_equals(
1621 expect=main.TRUE,
1622 actual=consistentHostsResult,
1623 onpass="Hosts view is consistent across all ONOS nodes",
1624 onfail="ONOS nodes have different views of hosts" )
1625
1626 main.step( "Each host has an IP address" )
1627 ipResult = main.TRUE
1628 for controller in range( 0, len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001629 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallf3d16e72015-12-16 17:45:08 -08001630 if hosts[ controller ]:
1631 for host in hosts[ controller ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07001632 if not host.get( 'ipAddresses', [] ):
Jon Hallf3d16e72015-12-16 17:45:08 -08001633 main.log.error( "Error with host ips on controller" +
1634 controllerStr + ": " + str( host ) )
1635 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07001636 utilities.assert_equals(
1637 expect=main.TRUE,
1638 actual=ipResult,
1639 onpass="The ips of the hosts aren't empty",
1640 onfail="The ip of at least one host is missing" )
1641
1642 # Strongly connected clusters of devices
1643 main.step( "Cluster view is consistent across ONOS nodes" )
1644 consistentClustersResult = main.TRUE
1645 for controller in range( len( clusters ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001646 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001647 if "Error" not in clusters[ controller ]:
1648 if clusters[ controller ] == clusters[ 0 ]:
1649 continue
1650 else: # clusters not consistent
1651 main.log.error( "clusters from ONOS" + controllerStr +
1652 " is inconsistent with ONOS1" )
1653 consistentClustersResult = main.FALSE
1654
1655 else:
1656 main.log.error( "Error in getting dataplane clusters " +
1657 "from ONOS" + controllerStr )
1658 consistentClustersResult = main.FALSE
1659 main.log.warn( "ONOS" + controllerStr +
1660 " clusters response: " +
1661 repr( clusters[ controller ] ) )
1662 utilities.assert_equals(
1663 expect=main.TRUE,
1664 actual=consistentClustersResult,
1665 onpass="Clusters view is consistent across all ONOS nodes",
1666 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07001667 if not consistentClustersResult:
Jon Hall172b7ba2016-04-07 18:12:20 -07001668 main.log.debug( clusters )
Jon Hall64948022016-05-12 13:38:50 -07001669
Jon Hall5cf14d52015-07-16 12:15:19 -07001670 # there should always only be one cluster
1671 main.step( "Cluster view correct across ONOS nodes" )
1672 try:
1673 numClusters = len( json.loads( clusters[ 0 ] ) )
1674 except ( ValueError, TypeError ):
1675 main.log.exception( "Error parsing clusters[0]: " +
1676 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001677 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07001678 clusterResults = main.FALSE
1679 if numClusters == 1:
1680 clusterResults = main.TRUE
1681 utilities.assert_equals(
1682 expect=1,
1683 actual=numClusters,
1684 onpass="ONOS shows 1 SCC",
1685 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1686
1687 main.step( "Comparing ONOS topology to MN" )
1688 devicesResults = main.TRUE
1689 linksResults = main.TRUE
1690 hostsResults = main.TRUE
1691 mnSwitches = main.Mininet1.getSwitches()
1692 mnLinks = main.Mininet1.getLinks()
1693 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001694 for controller in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001695 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001696 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07001697 "Error" not in devices[ controller ] and\
1698 "Error" not in ports[ controller ]:
1699 currentDevicesResult = main.Mininet1.compareSwitches(
1700 mnSwitches,
1701 json.loads( devices[ controller ] ),
1702 json.loads( ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001703 else:
1704 currentDevicesResult = main.FALSE
1705 utilities.assert_equals( expect=main.TRUE,
1706 actual=currentDevicesResult,
1707 onpass="ONOS" + controllerStr +
1708 " Switches view is correct",
1709 onfail="ONOS" + controllerStr +
1710 " Switches view is incorrect" )
1711 if links[ controller ] and "Error" not in links[ controller ]:
1712 currentLinksResult = main.Mininet1.compareLinks(
1713 mnSwitches, mnLinks,
1714 json.loads( links[ controller ] ) )
1715 else:
1716 currentLinksResult = main.FALSE
1717 utilities.assert_equals( expect=main.TRUE,
1718 actual=currentLinksResult,
1719 onpass="ONOS" + controllerStr +
1720 " links view is correct",
1721 onfail="ONOS" + controllerStr +
1722 " links view is incorrect" )
1723
Jon Hall657cdf62015-12-17 14:40:51 -08001724 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07001725 currentHostsResult = main.Mininet1.compareHosts(
1726 mnHosts,
1727 hosts[ controller ] )
1728 else:
1729 currentHostsResult = main.FALSE
1730 utilities.assert_equals( expect=main.TRUE,
1731 actual=currentHostsResult,
1732 onpass="ONOS" + controllerStr +
1733 " hosts exist in Mininet",
1734 onfail="ONOS" + controllerStr +
1735 " hosts don't match Mininet" )
1736
1737 devicesResults = devicesResults and currentDevicesResult
1738 linksResults = linksResults and currentLinksResult
1739 hostsResults = hostsResults and currentHostsResult
1740
1741 main.step( "Device information is correct" )
1742 utilities.assert_equals(
1743 expect=main.TRUE,
1744 actual=devicesResults,
1745 onpass="Device information is correct",
1746 onfail="Device information is incorrect" )
1747
1748 main.step( "Links are correct" )
1749 utilities.assert_equals(
1750 expect=main.TRUE,
1751 actual=linksResults,
1752 onpass="Link are correct",
1753 onfail="Links are incorrect" )
1754
1755 main.step( "Hosts are correct" )
1756 utilities.assert_equals(
1757 expect=main.TRUE,
1758 actual=hostsResults,
1759 onpass="Hosts are correct",
1760 onfail="Hosts are incorrect" )
1761
1762 def CASE6( self, main ):
1763 """
1764 The Failure case. Since this is the Sanity test, we do nothing.
1765 """
1766 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001767 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001768 assert main, "main not defined"
1769 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001770 assert main.CLIs, "main.CLIs not defined"
1771 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001772 main.case( "Wait 60 seconds instead of inducing a failure" )
1773 time.sleep( 60 )
1774 utilities.assert_equals(
1775 expect=main.TRUE,
1776 actual=main.TRUE,
1777 onpass="Sleeping 60 seconds",
1778 onfail="Something is terribly wrong with my math" )
1779
1780 def CASE7( self, main ):
1781 """
1782 Check state after ONOS failure
1783 """
1784 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001785 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001786 assert main, "main not defined"
1787 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07001788 assert main.CLIs, "main.CLIs not defined"
1789 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07001790 main.case( "Running ONOS Constant State Tests" )
1791
1792 main.step( "Check that each switch has a master" )
1793 # Assert that each device has a master
1794 rolesNotNull = main.TRUE
1795 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001796 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001797 t = main.Thread( target=main.CLIs[ i ].rolesNotNull,
Jon Hall5cf14d52015-07-16 12:15:19 -07001798 name="rolesNotNull-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07001799 args=[] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001800 threads.append( t )
1801 t.start()
1802
1803 for t in threads:
1804 t.join()
1805 rolesNotNull = rolesNotNull and t.result
1806 utilities.assert_equals(
1807 expect=main.TRUE,
1808 actual=rolesNotNull,
1809 onpass="Each device has a master",
1810 onfail="Some devices don't have a master assigned" )
1811
1812 main.step( "Read device roles from ONOS" )
1813 ONOSMastership = []
1814 mastershipCheck = main.FALSE
1815 consistentMastership = True
1816 rolesResults = True
1817 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001818 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001819 t = main.Thread( target=main.CLIs[ i ].roles,
Jon Hall5cf14d52015-07-16 12:15:19 -07001820 name="roles-" + str( i ),
1821 args=[] )
1822 threads.append( t )
1823 t.start()
1824
1825 for t in threads:
1826 t.join()
1827 ONOSMastership.append( t.result )
1828
Jon Halla440e872016-03-31 15:15:50 -07001829 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001830 node = str( main.activeNodes[ i ] + 1 )
1831 if not ONOSMastership[ i ] or "Error" in ONOSMastership[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07001832 main.log.error( "Error in getting ONOS" + node + " roles" )
1833 main.log.warn( "ONOS" + node + " mastership response: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07001834 repr( ONOSMastership[ i ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001835 rolesResults = False
1836 utilities.assert_equals(
1837 expect=True,
1838 actual=rolesResults,
1839 onpass="No error in reading roles output",
1840 onfail="Error in reading roles from ONOS" )
1841
1842 main.step( "Check for consistency in roles from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001843 if all( [ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001844 main.log.info(
1845 "Switch roles are consistent across all ONOS nodes" )
1846 else:
1847 consistentMastership = False
1848 utilities.assert_equals(
1849 expect=True,
1850 actual=consistentMastership,
1851 onpass="Switch roles are consistent across all ONOS nodes",
1852 onfail="ONOS nodes have different views of switch roles" )
1853
1854 if rolesResults and not consistentMastership:
Jon Halla440e872016-03-31 15:15:50 -07001855 for i in range( len( ONOSMastership ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001856 node = str( main.activeNodes[ i ] + 1 )
Jon Halla440e872016-03-31 15:15:50 -07001857 main.log.warn( "ONOS" + node + " roles: ",
1858 json.dumps( json.loads( ONOSMastership[ i ] ),
1859 sort_keys=True,
1860 indent=4,
1861 separators=( ',', ': ' ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001862
1863 description2 = "Compare switch roles from before failure"
1864 main.step( description2 )
1865 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07001866 currentJson = json.loads( ONOSMastership[ 0 ] )
Jon Hall5cf14d52015-07-16 12:15:19 -07001867 oldJson = json.loads( mastershipState )
1868 except ( ValueError, TypeError ):
1869 main.log.exception( "Something is wrong with parsing " +
1870 "ONOSMastership[0] or mastershipState" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001871 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[ 0 ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07001872 main.log.error( "mastershipState" + repr( mastershipState ) )
1873 main.cleanup()
1874 main.exit()
1875 mastershipCheck = main.TRUE
1876 for i in range( 1, 29 ):
1877 switchDPID = str(
1878 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1879 current = [ switch[ 'master' ] for switch in currentJson
1880 if switchDPID in switch[ 'id' ] ]
1881 old = [ switch[ 'master' ] for switch in oldJson
1882 if switchDPID in switch[ 'id' ] ]
1883 if current == old:
1884 mastershipCheck = mastershipCheck and main.TRUE
1885 else:
1886 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1887 mastershipCheck = main.FALSE
1888 utilities.assert_equals(
1889 expect=main.TRUE,
1890 actual=mastershipCheck,
1891 onpass="Mastership of Switches was not changed",
1892 onfail="Mastership of some switches changed" )
1893 mastershipCheck = mastershipCheck and consistentMastership
1894
1895 main.step( "Get the intents and compare across all nodes" )
1896 ONOSIntents = []
1897 intentCheck = main.FALSE
1898 consistentIntents = True
1899 intentsResults = True
1900 threads = []
Jon Halla440e872016-03-31 15:15:50 -07001901 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07001902 t = main.Thread( target=main.CLIs[ i ].intents,
Jon Hall5cf14d52015-07-16 12:15:19 -07001903 name="intents-" + str( i ),
1904 args=[],
1905 kwargs={ 'jsonFormat': True } )
1906 threads.append( t )
1907 t.start()
1908
1909 for t in threads:
1910 t.join()
1911 ONOSIntents.append( t.result )
1912
Jon Hallf37d44d2017-05-24 10:37:30 -07001913 for i in range( len( ONOSIntents ) ):
1914 node = str( main.activeNodes[ i ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07001915 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
Jon Halla440e872016-03-31 15:15:50 -07001916 main.log.error( "Error in getting ONOS" + node + " intents" )
1917 main.log.warn( "ONOS" + node + " intents response: " +
Jon Hall5cf14d52015-07-16 12:15:19 -07001918 repr( ONOSIntents[ i ] ) )
1919 intentsResults = False
1920 utilities.assert_equals(
1921 expect=True,
1922 actual=intentsResults,
1923 onpass="No error in reading intents output",
1924 onfail="Error in reading intents from ONOS" )
1925
1926 main.step( "Check for consistency in Intents from each controller" )
Jon Hallf37d44d2017-05-24 10:37:30 -07001927 if all( [ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall5cf14d52015-07-16 12:15:19 -07001928 main.log.info( "Intents are consistent across all ONOS " +
1929 "nodes" )
1930 else:
1931 consistentIntents = False
1932
1933 # Try to make it easy to figure out what is happening
1934 #
1935 # Intent ONOS1 ONOS2 ...
1936 # 0x01 INSTALLED INSTALLING
1937 # ... ... ...
1938 # ... ... ...
1939 title = " ID"
Jon Halla440e872016-03-31 15:15:50 -07001940 for n in main.activeNodes:
Jon Hall5cf14d52015-07-16 12:15:19 -07001941 title += " " * 10 + "ONOS" + str( n + 1 )
1942 main.log.warn( title )
1943 # get all intent keys in the cluster
1944 keys = []
1945 for nodeStr in ONOSIntents:
1946 node = json.loads( nodeStr )
1947 for intent in node:
1948 keys.append( intent.get( 'id' ) )
1949 keys = set( keys )
1950 for key in keys:
1951 row = "%-13s" % key
1952 for nodeStr in ONOSIntents:
1953 node = json.loads( nodeStr )
1954 for intent in node:
1955 if intent.get( 'id' ) == key:
1956 row += "%-15s" % intent.get( 'state' )
1957 main.log.warn( row )
1958 # End table view
1959
1960 utilities.assert_equals(
1961 expect=True,
1962 actual=consistentIntents,
1963 onpass="Intents are consistent across all ONOS nodes",
1964 onfail="ONOS nodes have different views of intents" )
1965 intentStates = []
1966 for node in ONOSIntents: # Iter through ONOS nodes
1967 nodeStates = []
1968 # Iter through intents of a node
1969 try:
1970 for intent in json.loads( node ):
1971 nodeStates.append( intent[ 'state' ] )
1972 except ( ValueError, TypeError ):
1973 main.log.exception( "Error in parsing intents" )
1974 main.log.error( repr( node ) )
1975 intentStates.append( nodeStates )
Jon Hallf37d44d2017-05-24 10:37:30 -07001976 out = [ ( i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
Jon Hall5cf14d52015-07-16 12:15:19 -07001977 main.log.info( dict( out ) )
1978
1979 if intentsResults and not consistentIntents:
Jon Halla440e872016-03-31 15:15:50 -07001980 for i in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07001981 node = str( main.activeNodes[ i ] + 1 )
Jon Halla440e872016-03-31 15:15:50 -07001982 main.log.warn( "ONOS" + node + " intents: " )
Jon Hall5cf14d52015-07-16 12:15:19 -07001983 main.log.warn( json.dumps(
1984 json.loads( ONOSIntents[ i ] ),
1985 sort_keys=True,
1986 indent=4,
1987 separators=( ',', ': ' ) ) )
1988 elif intentsResults and consistentIntents:
1989 intentCheck = main.TRUE
1990
1991 # NOTE: Store has no durability, so intents are lost across system
1992 # restarts
1993 main.step( "Compare current intents with intents before the failure" )
1994 # NOTE: this requires case 5 to pass for intentState to be set.
1995 # maybe we should stop the test if that fails?
1996 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07001997 try:
1998 intentState
1999 except NameError:
2000 main.log.warn( "No previous intent state was saved" )
2001 else:
2002 if intentState and intentState == ONOSIntents[ 0 ]:
2003 sameIntents = main.TRUE
2004 main.log.info( "Intents are consistent with before failure" )
2005 # TODO: possibly the states have changed? we may need to figure out
2006 # what the acceptable states are
2007 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
2008 sameIntents = main.TRUE
2009 try:
2010 before = json.loads( intentState )
2011 after = json.loads( ONOSIntents[ 0 ] )
2012 for intent in before:
2013 if intent not in after:
2014 sameIntents = main.FALSE
2015 main.log.debug( "Intent is not currently in ONOS " +
2016 "(at least in the same form):" )
2017 main.log.debug( json.dumps( intent ) )
2018 except ( ValueError, TypeError ):
2019 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002020 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002021 main.log.debug( repr( intentState ) )
2022 if sameIntents == main.FALSE:
2023 try:
2024 main.log.debug( "ONOS intents before: " )
2025 main.log.debug( json.dumps( json.loads( intentState ),
2026 sort_keys=True, indent=4,
2027 separators=( ',', ': ' ) ) )
2028 main.log.debug( "Current ONOS intents: " )
2029 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
2030 sort_keys=True, indent=4,
2031 separators=( ',', ': ' ) ) )
2032 except ( ValueError, TypeError ):
2033 main.log.exception( "Exception printing intents" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002034 main.log.debug( repr( ONOSIntents[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002035 main.log.debug( repr( intentState ) )
2036 utilities.assert_equals(
2037 expect=main.TRUE,
2038 actual=sameIntents,
2039 onpass="Intents are consistent with before failure",
2040 onfail="The Intents changed during failure" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002041 intentCheck = intentCheck and sameIntents
2042
2043 main.step( "Get the OF Table entries and compare to before " +
2044 "component failure" )
2045 FlowTables = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002046 for i in range( 28 ):
2047 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08002048 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hallf37d44d2017-05-24 10:37:30 -07002049 curSwitch = main.Mininet1.flowTableComp( flows[ i ], tmpFlows )
Jon Hall41d39f12016-04-11 22:54:35 -07002050 FlowTables = FlowTables and curSwitch
2051 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08002052 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002053 utilities.assert_equals(
2054 expect=main.TRUE,
2055 actual=FlowTables,
2056 onpass="No changes were found in the flow tables",
2057 onfail="Changes were found in the flow tables" )
2058
2059 main.Mininet2.pingLongKill()
Jon Hallf37d44d2017-05-24 10:37:30 -07002060 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002061 main.step( "Check the continuous pings to ensure that no packets " +
2062 "were dropped during component failure" )
2063 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
2064 main.params[ 'TESTONIP' ] )
2065 LossInPings = main.FALSE
2066 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
2067 for i in range( 8, 18 ):
2068 main.log.info(
2069 "Checking for a loss in pings along flow from s" +
2070 str( i ) )
2071 LossInPings = main.Mininet2.checkForLoss(
2072 "/tmp/ping.h" +
2073 str( i ) ) or LossInPings
2074 if LossInPings == main.TRUE:
2075 main.log.info( "Loss in ping detected" )
2076 elif LossInPings == main.ERROR:
2077 main.log.info( "There are multiple mininet process running" )
2078 elif LossInPings == main.FALSE:
2079 main.log.info( "No Loss in the pings" )
2080 main.log.info( "No loss of dataplane connectivity" )
2081 utilities.assert_equals(
2082 expect=main.FALSE,
2083 actual=LossInPings,
2084 onpass="No Loss of connectivity",
2085 onfail="Loss of dataplane connectivity detected" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002086 """
Jon Hall5cf14d52015-07-16 12:15:19 -07002087 main.step( "Leadership Election is still functional" )
2088 # Test of LeadershipElection
Jon Halla440e872016-03-31 15:15:50 -07002089 leaderList = []
2090
Jon Hall5cf14d52015-07-16 12:15:19 -07002091 # NOTE: this only works for the sanity test. In case of failures,
2092 # leader will likely change
Jon Halla440e872016-03-31 15:15:50 -07002093 leader = main.nodes[ main.activeNodes[ 0 ] ].ip_address
Jon Hall5cf14d52015-07-16 12:15:19 -07002094 leaderResult = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002095
2096 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002097 cli = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002098 leaderN = cli.electionTestLeader()
Jon Halla440e872016-03-31 15:15:50 -07002099 leaderList.append( leaderN )
Jon Hall5cf14d52015-07-16 12:15:19 -07002100 # verify leader is ONOS1
2101 if leaderN == leader:
2102 # all is well
2103 # NOTE: In failure scenario, this could be a new node, maybe
2104 # check != ONOS1
2105 pass
2106 elif leaderN == main.FALSE:
2107 # error in response
2108 main.log.error( "Something is wrong with " +
2109 "electionTestLeader function, check the" +
2110 " error logs" )
2111 leaderResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002112 elif leaderN is None:
2113 main.log.error( cli.name +
2114 " shows no leader for the election-app was" +
2115 " elected after the old one died" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002116 leaderResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002117 if len( set( leaderList ) ) != 1:
2118 leaderResult = main.FALSE
2119 main.log.error(
2120 "Inconsistent view of leader for the election test app" )
2121 # TODO: print the list
Jon Hall5cf14d52015-07-16 12:15:19 -07002122 utilities.assert_equals(
2123 expect=main.TRUE,
2124 actual=leaderResult,
2125 onpass="Leadership election passed",
2126 onfail="Something went wrong with Leadership election" )
2127
2128 def CASE8( self, main ):
2129 """
2130 Compare topo
2131 """
2132 import json
2133 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002134 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002135 assert main, "main not defined"
2136 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002137 assert main.CLIs, "main.CLIs not defined"
2138 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002139
2140 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07002141 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall5cf14d52015-07-16 12:15:19 -07002142 " and ONOS"
Jon Hall5cf14d52015-07-16 12:15:19 -07002143 topoResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002144 topoFailMsg = "ONOS topology don't match Mininet"
Jon Hall5cf14d52015-07-16 12:15:19 -07002145 elapsed = 0
2146 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08002147 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002148 startTime = time.time()
2149 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08002150 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07002151 devicesResults = main.TRUE
2152 linksResults = main.TRUE
2153 hostsResults = main.TRUE
2154 hostAttachmentResults = True
Jon Hall5cf14d52015-07-16 12:15:19 -07002155 count += 1
2156 cliStart = time.time()
2157 devices = []
2158 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002159 for i in main.activeNodes:
2160 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002161 name="devices-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002162 args=[ main.CLIs[ i ].devices, [ None ] ],
2163 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Halla440e872016-03-31 15:15:50 -07002164 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002165 threads.append( t )
2166 t.start()
2167
2168 for t in threads:
2169 t.join()
2170 devices.append( t.result )
2171 hosts = []
2172 ipResult = main.TRUE
2173 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002174 for i in main.activeNodes:
Jon Halld8f6de82015-12-17 17:04:34 -08002175 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002176 name="hosts-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002177 args=[ main.CLIs[ i ].hosts, [ None ] ],
2178 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Halld8f6de82015-12-17 17:04:34 -08002179 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002180 threads.append( t )
2181 t.start()
2182
2183 for t in threads:
2184 t.join()
2185 try:
2186 hosts.append( json.loads( t.result ) )
2187 except ( ValueError, TypeError ):
2188 main.log.exception( "Error parsing hosts results" )
2189 main.log.error( repr( t.result ) )
Jon Hallf3d16e72015-12-16 17:45:08 -08002190 hosts.append( None )
Jon Hall5cf14d52015-07-16 12:15:19 -07002191 for controller in range( 0, len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002192 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hallacd1b182015-12-17 11:43:20 -08002193 if hosts[ controller ]:
2194 for host in hosts[ controller ]:
2195 if host is None or host.get( 'ipAddresses', [] ) == []:
2196 main.log.error(
2197 "Error with host ipAddresses on controller" +
2198 controllerStr + ": " + str( host ) )
2199 ipResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002200 ports = []
2201 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002202 for i in main.activeNodes:
2203 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002204 name="ports-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002205 args=[ main.CLIs[ i ].ports, [ None ] ],
2206 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Halla440e872016-03-31 15:15:50 -07002207 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002208 threads.append( t )
2209 t.start()
2210
2211 for t in threads:
2212 t.join()
2213 ports.append( t.result )
2214 links = []
2215 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002216 for i in main.activeNodes:
2217 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002218 name="links-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002219 args=[ main.CLIs[ i ].links, [ None ] ],
2220 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Halla440e872016-03-31 15:15:50 -07002221 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002222 threads.append( t )
2223 t.start()
2224
2225 for t in threads:
2226 t.join()
2227 links.append( t.result )
2228 clusters = []
2229 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002230 for i in main.activeNodes:
2231 t = main.Thread( target=utilities.retry,
Jon Hall5cf14d52015-07-16 12:15:19 -07002232 name="clusters-" + str( i ),
Jon Hallf37d44d2017-05-24 10:37:30 -07002233 args=[ main.CLIs[ i ].clusters, [ None ] ],
2234 kwargs={ 'sleep': 5, 'attempts': 5,
Jon Halla440e872016-03-31 15:15:50 -07002235 'randomTime': True } )
Jon Hall5cf14d52015-07-16 12:15:19 -07002236 threads.append( t )
2237 t.start()
2238
2239 for t in threads:
2240 t.join()
2241 clusters.append( t.result )
2242
2243 elapsed = time.time() - startTime
2244 cliTime = time.time() - cliStart
2245 print "Elapsed time: " + str( elapsed )
2246 print "CLI time: " + str( cliTime )
2247
Jon Halla440e872016-03-31 15:15:50 -07002248 if all( e is None for e in devices ) and\
2249 all( e is None for e in hosts ) and\
2250 all( e is None for e in ports ) and\
2251 all( e is None for e in links ) and\
2252 all( e is None for e in clusters ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002253 topoFailMsg = "Could not get topology from ONOS"
2254 main.log.error( topoFailMsg )
2255 continue # Try again, No use trying to compare
Jon Halla440e872016-03-31 15:15:50 -07002256
Jon Hall5cf14d52015-07-16 12:15:19 -07002257 mnSwitches = main.Mininet1.getSwitches()
2258 mnLinks = main.Mininet1.getLinks()
2259 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07002260 for controller in range( len( main.activeNodes ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002261 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002262 if devices[ controller ] and ports[ controller ] and\
Jon Hallf37d44d2017-05-24 10:37:30 -07002263 "Error" not in devices[ controller ] and\
2264 "Error" not in ports[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002265
Jon Hallc6793552016-01-19 14:18:37 -08002266 try:
2267 currentDevicesResult = main.Mininet1.compareSwitches(
2268 mnSwitches,
2269 json.loads( devices[ controller ] ),
2270 json.loads( ports[ controller ] ) )
2271 except ( TypeError, ValueError ) as e:
2272 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
2273 devices[ controller ], ports[ controller ] ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002274 else:
2275 currentDevicesResult = main.FALSE
2276 utilities.assert_equals( expect=main.TRUE,
2277 actual=currentDevicesResult,
2278 onpass="ONOS" + controllerStr +
2279 " Switches view is correct",
2280 onfail="ONOS" + controllerStr +
2281 " Switches view is incorrect" )
2282
2283 if links[ controller ] and "Error" not in links[ controller ]:
2284 currentLinksResult = main.Mininet1.compareLinks(
2285 mnSwitches, mnLinks,
2286 json.loads( links[ controller ] ) )
2287 else:
2288 currentLinksResult = main.FALSE
2289 utilities.assert_equals( expect=main.TRUE,
2290 actual=currentLinksResult,
2291 onpass="ONOS" + controllerStr +
2292 " links view is correct",
2293 onfail="ONOS" + controllerStr +
2294 " links view is incorrect" )
Jon Hall657cdf62015-12-17 14:40:51 -08002295 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002296 currentHostsResult = main.Mininet1.compareHosts(
2297 mnHosts,
2298 hosts[ controller ] )
Jon Hall13b446e2016-01-05 12:17:01 -08002299 elif hosts[ controller ] == []:
2300 currentHostsResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002301 else:
2302 currentHostsResult = main.FALSE
2303 utilities.assert_equals( expect=main.TRUE,
2304 actual=currentHostsResult,
2305 onpass="ONOS" + controllerStr +
2306 " hosts exist in Mininet",
2307 onfail="ONOS" + controllerStr +
2308 " hosts don't match Mininet" )
2309 # CHECKING HOST ATTACHMENT POINTS
2310 hostAttachment = True
2311 zeroHosts = False
2312 # FIXME: topo-HA/obelisk specific mappings:
2313 # key is mac and value is dpid
2314 mappings = {}
2315 for i in range( 1, 29 ): # hosts 1 through 28
2316 # set up correct variables:
Jon Hallf37d44d2017-05-24 10:37:30 -07002317 macId = "00:" * 5 + hex( i ).split( "0x" )[ 1 ].upper().zfill( 2 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002318 if i == 1:
Jon Hallf37d44d2017-05-24 10:37:30 -07002319 deviceId = "1000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002320 elif i == 2:
Jon Hallf37d44d2017-05-24 10:37:30 -07002321 deviceId = "2000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002322 elif i == 3:
Jon Hallf37d44d2017-05-24 10:37:30 -07002323 deviceId = "3000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002324 elif i == 4:
Jon Hallf37d44d2017-05-24 10:37:30 -07002325 deviceId = "3004".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002326 elif i == 5:
Jon Hallf37d44d2017-05-24 10:37:30 -07002327 deviceId = "5000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002328 elif i == 6:
Jon Hallf37d44d2017-05-24 10:37:30 -07002329 deviceId = "6000".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002330 elif i == 7:
Jon Hallf37d44d2017-05-24 10:37:30 -07002331 deviceId = "6007".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002332 elif i >= 8 and i <= 17:
2333 dpid = '3' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07002334 deviceId = dpid.zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002335 elif i >= 18 and i <= 27:
2336 dpid = '6' + str( i ).zfill( 3 )
Jon Hallf37d44d2017-05-24 10:37:30 -07002337 deviceId = dpid.zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002338 elif i == 28:
Jon Hallf37d44d2017-05-24 10:37:30 -07002339 deviceId = "2800".zfill( 16 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002340 mappings[ macId ] = deviceId
Jon Halld8f6de82015-12-17 17:04:34 -08002341 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002342 if hosts[ controller ] == []:
2343 main.log.warn( "There are no hosts discovered" )
2344 zeroHosts = True
2345 else:
2346 for host in hosts[ controller ]:
2347 mac = None
2348 location = None
2349 device = None
2350 port = None
2351 try:
2352 mac = host.get( 'mac' )
2353 assert mac, "mac field could not be found for this host object"
2354
2355 location = host.get( 'location' )
2356 assert location, "location field could not be found for this host object"
2357
2358 # Trim the protocol identifier off deviceId
Jon Hallf37d44d2017-05-24 10:37:30 -07002359 device = str( location.get( 'elementId' ) ).split( ':' )[ 1 ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002360 assert device, "elementId field could not be found for this host location object"
2361
2362 port = location.get( 'port' )
2363 assert port, "port field could not be found for this host location object"
2364
2365 # Now check if this matches where they should be
2366 if mac and device and port:
2367 if str( port ) != "1":
2368 main.log.error( "The attachment port is incorrect for " +
2369 "host " + str( mac ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07002370 ". Expected: 1 Actual: " + str( port ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002371 hostAttachment = False
2372 if device != mappings[ str( mac ) ]:
2373 main.log.error( "The attachment device is incorrect for " +
2374 "host " + str( mac ) +
2375 ". Expected: " + mappings[ str( mac ) ] +
2376 " Actual: " + device )
2377 hostAttachment = False
2378 else:
2379 hostAttachment = False
2380 except AssertionError:
2381 main.log.exception( "Json object not as expected" )
2382 main.log.error( repr( host ) )
2383 hostAttachment = False
2384 else:
2385 main.log.error( "No hosts json output or \"Error\"" +
2386 " in output. hosts = " +
2387 repr( hosts[ controller ] ) )
2388 if zeroHosts is False:
2389 hostAttachment = True
2390
2391 # END CHECKING HOST ATTACHMENT POINTS
2392 devicesResults = devicesResults and currentDevicesResult
2393 linksResults = linksResults and currentLinksResult
2394 hostsResults = hostsResults and currentHostsResult
2395 hostAttachmentResults = hostAttachmentResults and\
2396 hostAttachment
2397 topoResult = ( devicesResults and linksResults
2398 and hostsResults and ipResult and
2399 hostAttachmentResults )
Jon Halle9b1fa32015-12-08 15:32:21 -08002400 utilities.assert_equals( expect=True,
2401 actual=topoResult,
2402 onpass="ONOS topology matches Mininet",
Jon Halla440e872016-03-31 15:15:50 -07002403 onfail=topoFailMsg )
Jon Halle9b1fa32015-12-08 15:32:21 -08002404 # End of While loop to pull ONOS state
Jon Hall5cf14d52015-07-16 12:15:19 -07002405
2406 # Compare json objects for hosts and dataplane clusters
2407
2408 # hosts
2409 main.step( "Hosts view is consistent across all ONOS nodes" )
2410 consistentHostsResult = main.TRUE
2411 for controller in range( len( hosts ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002412 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall13b446e2016-01-05 12:17:01 -08002413 if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
Jon Hall5cf14d52015-07-16 12:15:19 -07002414 if hosts[ controller ] == hosts[ 0 ]:
2415 continue
2416 else: # hosts not consistent
2417 main.log.error( "hosts from ONOS" + controllerStr +
2418 " is inconsistent with ONOS1" )
2419 main.log.warn( repr( hosts[ controller ] ) )
2420 consistentHostsResult = main.FALSE
2421
2422 else:
2423 main.log.error( "Error in getting ONOS hosts from ONOS" +
2424 controllerStr )
2425 consistentHostsResult = main.FALSE
2426 main.log.warn( "ONOS" + controllerStr +
2427 " hosts response: " +
2428 repr( hosts[ controller ] ) )
2429 utilities.assert_equals(
2430 expect=main.TRUE,
2431 actual=consistentHostsResult,
2432 onpass="Hosts view is consistent across all ONOS nodes",
2433 onfail="ONOS nodes have different views of hosts" )
2434
2435 main.step( "Hosts information is correct" )
2436 hostsResults = hostsResults and ipResult
2437 utilities.assert_equals(
2438 expect=main.TRUE,
2439 actual=hostsResults,
2440 onpass="Host information is correct",
2441 onfail="Host information is incorrect" )
2442
2443 main.step( "Host attachment points to the network" )
2444 utilities.assert_equals(
2445 expect=True,
2446 actual=hostAttachmentResults,
2447 onpass="Hosts are correctly attached to the network",
2448 onfail="ONOS did not correctly attach hosts to the network" )
2449
2450 # Strongly connected clusters of devices
2451 main.step( "Clusters view is consistent across all ONOS nodes" )
2452 consistentClustersResult = main.TRUE
2453 for controller in range( len( clusters ) ):
Jon Hallf37d44d2017-05-24 10:37:30 -07002454 controllerStr = str( main.activeNodes[ controller ] + 1 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002455 if "Error" not in clusters[ controller ]:
2456 if clusters[ controller ] == clusters[ 0 ]:
2457 continue
2458 else: # clusters not consistent
2459 main.log.error( "clusters from ONOS" +
2460 controllerStr +
2461 " is inconsistent with ONOS1" )
2462 consistentClustersResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002463 else:
2464 main.log.error( "Error in getting dataplane clusters " +
2465 "from ONOS" + controllerStr )
2466 consistentClustersResult = main.FALSE
2467 main.log.warn( "ONOS" + controllerStr +
2468 " clusters response: " +
2469 repr( clusters[ controller ] ) )
2470 utilities.assert_equals(
2471 expect=main.TRUE,
2472 actual=consistentClustersResult,
2473 onpass="Clusters view is consistent across all ONOS nodes",
2474 onfail="ONOS nodes have different views of clusters" )
Jon Hall64948022016-05-12 13:38:50 -07002475 if not consistentClustersResult:
2476 main.log.debug( clusters )
Jon Hall5cf14d52015-07-16 12:15:19 -07002477
2478 main.step( "There is only one SCC" )
2479 # there should always only be one cluster
2480 try:
2481 numClusters = len( json.loads( clusters[ 0 ] ) )
2482 except ( ValueError, TypeError ):
2483 main.log.exception( "Error parsing clusters[0]: " +
Jon Hallf37d44d2017-05-24 10:37:30 -07002484 repr( clusters[ 0 ] ) )
Jon Halla440e872016-03-31 15:15:50 -07002485 numClusters = "ERROR"
Jon Hall5cf14d52015-07-16 12:15:19 -07002486 clusterResults = main.FALSE
2487 if numClusters == 1:
2488 clusterResults = main.TRUE
2489 utilities.assert_equals(
2490 expect=1,
2491 actual=numClusters,
2492 onpass="ONOS shows 1 SCC",
2493 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
2494
2495 topoResult = ( devicesResults and linksResults
2496 and hostsResults and consistentHostsResult
2497 and consistentClustersResult and clusterResults
2498 and ipResult and hostAttachmentResults )
2499
2500 topoResult = topoResult and int( count <= 2 )
2501 note = "note it takes about " + str( int( cliTime ) ) + \
2502 " seconds for the test to make all the cli calls to fetch " +\
2503 "the topology from each ONOS instance"
2504 main.log.info(
2505 "Very crass estimate for topology discovery/convergence( " +
2506 str( note ) + " ): " + str( elapsed ) + " seconds, " +
2507 str( count ) + " tries" )
2508
2509 main.step( "Device information is correct" )
2510 utilities.assert_equals(
2511 expect=main.TRUE,
2512 actual=devicesResults,
2513 onpass="Device information is correct",
2514 onfail="Device information is incorrect" )
2515
2516 main.step( "Links are correct" )
2517 utilities.assert_equals(
2518 expect=main.TRUE,
2519 actual=linksResults,
2520 onpass="Link are correct",
2521 onfail="Links are incorrect" )
2522
2523 main.step( "Hosts are correct" )
2524 utilities.assert_equals(
2525 expect=main.TRUE,
2526 actual=hostsResults,
2527 onpass="Hosts are correct",
2528 onfail="Hosts are incorrect" )
2529
2530 # FIXME: move this to an ONOS state case
2531 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -07002532 nodeResults = utilities.retry( main.HA.nodesCheck,
2533 False,
Jon Hallf37d44d2017-05-24 10:37:30 -07002534 args=[ main.activeNodes ],
Jon Hall41d39f12016-04-11 22:54:35 -07002535 attempts=5 )
Jon Hall5cf14d52015-07-16 12:15:19 -07002536
Jon Hall41d39f12016-04-11 22:54:35 -07002537 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Hall5cf14d52015-07-16 12:15:19 -07002538 onpass="Nodes check successful",
2539 onfail="Nodes check NOT successful" )
Jon Halla440e872016-03-31 15:15:50 -07002540 if not nodeResults:
Jon Hall41d39f12016-04-11 22:54:35 -07002541 for i in main.activeNodes:
Jon Halla440e872016-03-31 15:15:50 -07002542 main.log.debug( "{} components not ACTIVE: \n{}".format(
Jon Hallf37d44d2017-05-24 10:37:30 -07002543 main.CLIs[ i ].name,
2544 main.CLIs[ i ].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002545
Jon Halld2871c22016-07-26 11:01:14 -07002546 if not topoResult:
2547 main.cleanup()
2548 main.exit()
2549
Jon Hall5cf14d52015-07-16 12:15:19 -07002550 def CASE9( self, main ):
2551 """
2552 Link s3-s28 down
2553 """
2554 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002555 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002556 assert main, "main not defined"
2557 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002558 assert main.CLIs, "main.CLIs not defined"
2559 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002560 # NOTE: You should probably run a topology check after this
2561
2562 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2563
2564 description = "Turn off a link to ensure that Link Discovery " +\
2565 "is working properly"
2566 main.case( description )
2567
2568 main.step( "Kill Link between s3 and s28" )
2569 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
2570 main.log.info( "Waiting " + str( linkSleep ) +
2571 " seconds for link down to be discovered" )
2572 time.sleep( linkSleep )
2573 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
2574 onpass="Link down successful",
2575 onfail="Failed to bring link down" )
2576 # TODO do some sort of check here
2577
2578 def CASE10( self, main ):
2579 """
2580 Link s3-s28 up
2581 """
2582 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002583 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002584 assert main, "main not defined"
2585 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002586 assert main.CLIs, "main.CLIs not defined"
2587 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002588 # NOTE: You should probably run a topology check after this
2589
2590 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
2591
2592 description = "Restore a link to ensure that Link Discovery is " + \
2593 "working properly"
2594 main.case( description )
2595
2596 main.step( "Bring link between s3 and s28 back up" )
2597 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
2598 main.log.info( "Waiting " + str( linkSleep ) +
2599 " seconds for link up to be discovered" )
2600 time.sleep( linkSleep )
2601 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
2602 onpass="Link up successful",
2603 onfail="Failed to bring link up" )
2604 # TODO do some sort of check here
2605
2606 def CASE11( self, main ):
2607 """
2608 Switch Down
2609 """
2610 # NOTE: You should probably run a topology check after this
2611 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002612 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002613 assert main, "main not defined"
2614 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002615 assert main.CLIs, "main.CLIs not defined"
2616 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002617
2618 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2619
2620 description = "Killing a switch to ensure it is discovered correctly"
Jon Hallf37d44d2017-05-24 10:37:30 -07002621 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002622 main.case( description )
2623 switch = main.params[ 'kill' ][ 'switch' ]
2624 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2625
2626 # TODO: Make this switch parameterizable
2627 main.step( "Kill " + switch )
2628 main.log.info( "Deleting " + switch )
2629 main.Mininet1.delSwitch( switch )
2630 main.log.info( "Waiting " + str( switchSleep ) +
2631 " seconds for switch down to be discovered" )
2632 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002633 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002634 # Peek at the deleted switch
2635 main.log.warn( str( device ) )
2636 result = main.FALSE
2637 if device and device[ 'available' ] is False:
2638 result = main.TRUE
2639 utilities.assert_equals( expect=main.TRUE, actual=result,
2640 onpass="Kill switch successful",
2641 onfail="Failed to kill switch?" )
2642
2643 def CASE12( self, main ):
2644 """
2645 Switch Up
2646 """
2647 # NOTE: You should probably run a topology check after this
2648 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002649 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002650 assert main, "main not defined"
2651 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002652 assert main.CLIs, "main.CLIs not defined"
2653 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002654 assert ONOS1Port, "ONOS1Port not defined"
2655 assert ONOS2Port, "ONOS2Port not defined"
2656 assert ONOS3Port, "ONOS3Port not defined"
2657 assert ONOS4Port, "ONOS4Port not defined"
2658 assert ONOS5Port, "ONOS5Port not defined"
2659 assert ONOS6Port, "ONOS6Port not defined"
2660 assert ONOS7Port, "ONOS7Port not defined"
2661
2662 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
2663 switch = main.params[ 'kill' ][ 'switch' ]
2664 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2665 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallf37d44d2017-05-24 10:37:30 -07002666 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002667 description = "Adding a switch to ensure it is discovered correctly"
2668 main.case( description )
2669
2670 main.step( "Add back " + switch )
2671 main.Mininet1.addSwitch( switch, dpid=switchDPID )
2672 for peer in links:
2673 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07002674 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002675 main.Mininet1.assignSwController( sw=switch, ip=ipList )
2676 main.log.info( "Waiting " + str( switchSleep ) +
2677 " seconds for switch up to be discovered" )
2678 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07002679 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall5cf14d52015-07-16 12:15:19 -07002680 # Peek at the deleted switch
2681 main.log.warn( str( device ) )
2682 result = main.FALSE
2683 if device and device[ 'available' ]:
2684 result = main.TRUE
2685 utilities.assert_equals( expect=main.TRUE, actual=result,
2686 onpass="add switch successful",
2687 onfail="Failed to add switch?" )
2688
2689 def CASE13( self, main ):
2690 """
2691 Clean up
2692 """
2693 import os
2694 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002695 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002696 assert main, "main not defined"
2697 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002698 assert main.CLIs, "main.CLIs not defined"
2699 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002700
2701 # printing colors to terminal
2702 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2703 'blue': '\033[94m', 'green': '\033[92m',
2704 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
2705 main.case( "Test Cleanup" )
2706 main.step( "Killing tcpdumps" )
2707 main.Mininet2.stopTcpdump()
2708
2709 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002710 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall5cf14d52015-07-16 12:15:19 -07002711 main.step( "Copying MN pcap and ONOS log files to test station" )
2712 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2713 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002714 # NOTE: MN Pcap file is being saved to logdir.
2715 # We scp this file as MN and TestON aren't necessarily the same vm
2716
2717 # FIXME: To be replaced with a Jenkin's post script
Jon Hall5cf14d52015-07-16 12:15:19 -07002718 # TODO: Load these from params
2719 # NOTE: must end in /
2720 logFolder = "/opt/onos/log/"
2721 logFiles = [ "karaf.log", "karaf.log.1" ]
2722 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002723 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002724 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002725 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002726 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2727 logFolder + f, dstName )
Jon Hall5cf14d52015-07-16 12:15:19 -07002728 # std*.log's
2729 # NOTE: must end in /
2730 logFolder = "/opt/onos/var/"
2731 logFiles = [ "stderr.log", "stdout.log" ]
2732 # NOTE: must end in /
Jon Hall5cf14d52015-07-16 12:15:19 -07002733 for f in logFiles:
Jon Halle1a3b752015-07-22 13:02:46 -07002734 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002735 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002736 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2737 logFolder + f, dstName )
2738 else:
2739 main.log.debug( "skipping saving log files" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002740
2741 main.step( "Stopping Mininet" )
2742 mnResult = main.Mininet1.stopNet()
2743 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2744 onpass="Mininet stopped",
2745 onfail="MN cleanup NOT successful" )
2746
2747 main.step( "Checking ONOS Logs for errors" )
Jon Halle1a3b752015-07-22 13:02:46 -07002748 for node in main.nodes:
Jon Hall96091e62015-09-21 17:34:17 -07002749 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2750 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall5cf14d52015-07-16 12:15:19 -07002751
2752 try:
Jon Hallf37d44d2017-05-24 10:37:30 -07002753 timerLog = open( main.logdir + "/Timers.csv", 'w' )
Jon Hall5cf14d52015-07-16 12:15:19 -07002754 # Overwrite with empty line and close
2755 labels = "Gossip Intents"
2756 data = str( gossipTime )
2757 timerLog.write( labels + "\n" + data )
2758 timerLog.close()
Jon Hallf37d44d2017-05-24 10:37:30 -07002759 except NameError as e:
2760 main.log.exception( e )
Jon Hall5cf14d52015-07-16 12:15:19 -07002761
2762 def CASE14( self, main ):
2763 """
2764 start election app on all onos nodes
2765 """
Jon Halle1a3b752015-07-22 13:02:46 -07002766 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002767 assert main, "main not defined"
2768 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002769 assert main.CLIs, "main.CLIs not defined"
2770 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002771
Jon Hallf37d44d2017-05-24 10:37:30 -07002772 main.case( "Start Leadership Election app" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002773 main.step( "Install leadership election app" )
Jon Hallf37d44d2017-05-24 10:37:30 -07002774 onosCli = main.CLIs[ main.activeNodes[ 0 ] ]
Jon Halla440e872016-03-31 15:15:50 -07002775 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002776 utilities.assert_equals(
2777 expect=main.TRUE,
2778 actual=appResult,
2779 onpass="Election app installed",
2780 onfail="Something went wrong with installing Leadership election" )
2781
2782 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002783 for i in main.activeNodes:
Jon Hallf37d44d2017-05-24 10:37:30 -07002784 main.CLIs[ i ].electionTestRun()
2785 time.sleep( 5 )
2786 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall25463a82016-04-13 14:03:52 -07002787 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall5cf14d52015-07-16 12:15:19 -07002788 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002789 expect=True,
2790 actual=sameResult,
2791 onpass="All nodes see the same leaderboards",
2792 onfail="Inconsistent leaderboards" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002793
Jon Hall25463a82016-04-13 14:03:52 -07002794 if sameResult:
2795 leader = leaders[ 0 ][ 0 ]
Jon Hallf37d44d2017-05-24 10:37:30 -07002796 if main.nodes[ main.activeNodes[ 0 ] ].ip_address in leader:
Jon Hall25463a82016-04-13 14:03:52 -07002797 correctLeader = True
2798 else:
2799 correctLeader = False
2800 main.step( "First node was elected leader" )
2801 utilities.assert_equals(
2802 expect=True,
2803 actual=correctLeader,
2804 onpass="Correct leader was elected",
2805 onfail="Incorrect leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002806
2807 def CASE15( self, main ):
2808 """
2809 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002810 15.1 Run election on each node
2811 15.2 Check that each node has the same leaders and candidates
2812 15.3 Find current leader and withdraw
2813 15.4 Check that a new node was elected leader
2814 15.5 Check that that new leader was the candidate of old leader
2815 15.6 Run for election on old leader
2816 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2817 15.8 Make sure that the old leader was added to the candidate list
2818
2819 old and new variable prefixes refer to data from before vs after
2820 withdrawl and later before withdrawl vs after re-election
Jon Hall5cf14d52015-07-16 12:15:19 -07002821 """
2822 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002823 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002824 assert main, "main not defined"
2825 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002826 assert main.CLIs, "main.CLIs not defined"
2827 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002828
Jon Halla440e872016-03-31 15:15:50 -07002829 description = "Check that Leadership Election is still functional"
Jon Hall5cf14d52015-07-16 12:15:19 -07002830 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002831 # NOTE: Need to re-run after restarts since being a canidate is not persistant
Jon Hall5cf14d52015-07-16 12:15:19 -07002832
Jon Halla440e872016-03-31 15:15:50 -07002833 oldLeaders = [] # list of lists of each nodes' candidates before
2834 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002835 oldLeader = '' # the old leader from oldLeaders, None if not same
2836 newLeader = '' # the new leaders fron newLoeaders, None if not same
2837 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2838 expectNoLeader = False # True when there is only one leader
2839 if main.numCtrls == 1:
2840 expectNoLeader = True
2841
2842 main.step( "Run for election on each node" )
2843 electionResult = main.TRUE
2844
Jon Halla440e872016-03-31 15:15:50 -07002845 for i in main.activeNodes: # run test election on each node
Jon Hallf37d44d2017-05-24 10:37:30 -07002846 if main.CLIs[ i ].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002847 electionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002848 utilities.assert_equals(
2849 expect=main.TRUE,
acsmars71adceb2015-08-31 15:09:26 -07002850 actual=electionResult,
2851 onpass="All nodes successfully ran for leadership",
2852 onfail="At least one node failed to run for leadership" )
2853
acsmars3a72bde2015-09-02 14:16:22 -07002854 if electionResult == main.FALSE:
2855 main.log.error(
Jon Halla440e872016-03-31 15:15:50 -07002856 "Skipping Test Case because Election Test App isn't loaded" )
acsmars3a72bde2015-09-02 14:16:22 -07002857 main.skipCase()
2858
acsmars71adceb2015-08-31 15:09:26 -07002859 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002860 failMessage = "Nodes have different leaderboards"
Jon Hallf37d44d2017-05-24 10:37:30 -07002861 activeCLIs = [ main.CLIs[ i ] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002862 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002863 if sameResult:
2864 oldLeader = oldLeaders[ 0 ][ 0 ]
2865 main.log.warn( oldLeader )
acsmars71adceb2015-08-31 15:09:26 -07002866 else:
Jon Halla440e872016-03-31 15:15:50 -07002867 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002868 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002869 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002870 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002871 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002872 onfail=failMessage )
Jon Hall5cf14d52015-07-16 12:15:19 -07002873
2874 main.step( "Find current leader and withdraw" )
acsmars71adceb2015-08-31 15:09:26 -07002875 withdrawResult = main.TRUE
Jon Hall5cf14d52015-07-16 12:15:19 -07002876 # do some sanity checking on leader before using it
acsmars71adceb2015-08-31 15:09:26 -07002877 if oldLeader is None:
2878 main.log.error( "Leadership isn't consistent." )
2879 withdrawResult = main.FALSE
2880 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002881 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002882 if oldLeader == main.nodes[ i ].ip_address:
2883 oldLeaderCLI = main.CLIs[ i ]
Jon Hall5cf14d52015-07-16 12:15:19 -07002884 break
2885 else: # FOR/ELSE statement
2886 main.log.error( "Leader election, could not find current leader" )
2887 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002888 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall5cf14d52015-07-16 12:15:19 -07002889 utilities.assert_equals(
2890 expect=main.TRUE,
2891 actual=withdrawResult,
2892 onpass="Node was withdrawn from election",
2893 onfail="Node was not withdrawn from election" )
2894
acsmars71adceb2015-08-31 15:09:26 -07002895 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002896 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07002897 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002898 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002899 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002900 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002901 if newLeaders[ 0 ][ 0 ] == 'none':
2902 main.log.error( "No leader was elected on at least 1 node" )
2903 if not expectNoLeader:
2904 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002905 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002906
2907 # Check that the new leader is not the older leader, which was withdrawn
2908 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002909 newLeaderResult = False
Jon Hall6e709752016-02-01 13:38:46 -08002910 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
Jon Hallf37d44d2017-05-24 10:37:30 -07002911 " as the current leader" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002912 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002913 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002914 actual=newLeaderResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002915 onpass="Leadership election passed",
2916 onfail="Something went wrong with Leadership election" )
2917
Jon Halla440e872016-03-31 15:15:50 -07002918 main.step( "Check that that new leader was the candidate of old leader" )
2919 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07002920 correctCandidateResult = main.TRUE
2921 if expectNoLeader:
2922 if newLeader == 'none':
2923 main.log.info( "No leader expected. None found. Pass" )
2924 correctCandidateResult = main.TRUE
2925 else:
2926 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2927 correctCandidateResult = main.FALSE
Jon Hallf37d44d2017-05-24 10:37:30 -07002928 elif len( oldLeaders[ 0 ] ) >= 3:
Jon Halla440e872016-03-31 15:15:50 -07002929 if newLeader == oldLeaders[ 0 ][ 2 ]:
2930 # correct leader was elected
2931 correctCandidateResult = main.TRUE
2932 else:
2933 correctCandidateResult = main.FALSE
2934 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
2935 newLeader, oldLeaders[ 0 ][ 2 ] ) )
2936 else:
2937 main.log.warn( "Could not determine who should be the correct leader" )
2938 main.log.debug( oldLeaders[ 0 ] )
acsmars71adceb2015-08-31 15:09:26 -07002939 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002940 utilities.assert_equals(
2941 expect=main.TRUE,
2942 actual=correctCandidateResult,
2943 onpass="Correct Candidate Elected",
2944 onfail="Incorrect Candidate Elected" )
2945
Jon Hall5cf14d52015-07-16 12:15:19 -07002946 main.step( "Run for election on old leader( just so everyone " +
2947 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002948 if oldLeaderCLI is not None:
2949 runResult = oldLeaderCLI.electionTestRun()
Jon Hall5cf14d52015-07-16 12:15:19 -07002950 else:
acsmars71adceb2015-08-31 15:09:26 -07002951 main.log.error( "No old leader to re-elect" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002952 runResult = main.FALSE
2953 utilities.assert_equals(
2954 expect=main.TRUE,
2955 actual=runResult,
2956 onpass="App re-ran for election",
2957 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07002958
acsmars71adceb2015-08-31 15:09:26 -07002959 main.step(
2960 "Check that oldLeader is a candidate, and leader if only 1 node" )
Jon Hall5cf14d52015-07-16 12:15:19 -07002961 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07002962 # Get new leaders and candidates
2963 reRunLeaders = []
Jon Hall7a6ebfd2017-03-13 10:58:58 -07002964 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07002965 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07002966
2967 # Check that the re-elected node is last on the candidate List
Jon Hallf37d44d2017-05-24 10:37:30 -07002968 if not reRunLeaders[ 0 ]:
Jon Hall3a7843a2016-04-12 03:01:09 -07002969 positionResult = main.FALSE
2970 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Hallf37d44d2017-05-24 10:37:30 -07002971 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader ),
Jon Halla440e872016-03-31 15:15:50 -07002972 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07002973 positionResult = main.FALSE
Jon Hall5cf14d52015-07-16 12:15:19 -07002974 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002975 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002976 actual=positionResult,
Jon Hall5cf14d52015-07-16 12:15:19 -07002977 onpass="Old leader successfully re-ran for election",
2978 onfail="Something went wrong with Leadership election after " +
2979 "the old leader re-ran for election" )
2980
2981 def CASE16( self, main ):
2982 """
2983 Install Distributed Primitives app
2984 """
2985 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002986 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002987 assert main, "main not defined"
2988 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002989 assert main.CLIs, "main.CLIs not defined"
2990 assert main.nodes, "main.nodes not defined"
Jon Hall5cf14d52015-07-16 12:15:19 -07002991
2992 # Variables for the distributed primitives tests
Jon Hall7a6ebfd2017-03-13 10:58:58 -07002993 main.pCounterName = "TestON-Partitions"
2994 main.pCounterValue = 0
Jon Hallf37d44d2017-05-24 10:37:30 -07002995 main.onosSet = set( [] )
Jon Hall7a6ebfd2017-03-13 10:58:58 -07002996 main.onosSetName = "TestON-set"
Jon Hall5cf14d52015-07-16 12:15:19 -07002997
2998 description = "Install Primitives app"
2999 main.case( description )
3000 main.step( "Install Primitives app" )
3001 appName = "org.onosproject.distributedprimitives"
Jon Hallf37d44d2017-05-24 10:37:30 -07003002 node = main.activeNodes[ 0 ]
3003 appResults = main.CLIs[ node ].activateApp( appName )
Jon Hall5cf14d52015-07-16 12:15:19 -07003004 utilities.assert_equals( expect=main.TRUE,
3005 actual=appResults,
3006 onpass="Primitives app activated",
3007 onfail="Primitives app not activated" )
3008 time.sleep( 5 ) # To allow all nodes to activate
3009
3010 def CASE17( self, main ):
3011 """
3012 Check for basic functionality with distributed primitives
3013 """
Jon Hall7a6ebfd2017-03-13 10:58:58 -07003014 main.HA.CASE17( main )