blob: 20f6cc8fceae0150f64f1844c90180f132b39ff3 [file] [log] [blame]
Jon Hall6aec96b2015-01-19 14:49:31 -08001"""
Jon Hallb1290e82014-11-18 16:17:48 -05002Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case. Since this is the Sanity test, we do nothing.
Jon Hall368769f2014-11-19 15:43:35 -080013CASE7: Check state after control plane failure
Jon Hallb1290e82014-11-18 16:17:48 -050014CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
Jon Hall669173b2014-12-17 11:36:30 -080020CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -080022"""
Jon Hall8f89dda2015-01-22 16:03:33 -080023
24
Jon Hallb1290e82014-11-18 16:17:48 -050025class HATestSanity:
26
Jon Hall6aec96b2015-01-19 14:49:31 -080027 def __init__( self ):
Jon Hallb1290e82014-11-18 16:17:48 -050028 self.default = ''
29
Jon Hall6aec96b2015-01-19 14:49:31 -080030 def CASE1( self, main ):
31 """
Jon Hallb1290e82014-11-18 16:17:48 -050032 CASE1 is to compile ONOS and push it to the test machines
33
34 Startup sequence:
Jon Hallb1290e82014-11-18 16:17:48 -050035 cell <name>
36 onos-verify-cell
37 NOTE: temporary - onos-remove-raft-logs
Jon Hall58c76b72015-02-23 11:09:24 -080038 onos-uninstall
39 start mininet
40 git pull
41 mvn clean install
42 onos-package
Jon Hallb1290e82014-11-18 16:17:48 -050043 onos-install -f
44 onos-wait-for-start
Jon Hall58c76b72015-02-23 11:09:24 -080045 start cli sessions
46 start tcpdump
Jon Hall6aec96b2015-01-19 14:49:31 -080047 """
48 main.log.report( "ONOS HA Sanity test - initialization" )
49 main.case( "Setting up test environment" )
50 # TODO: save all the timers and output them for plotting
Jon Hallb1290e82014-11-18 16:17:48 -050051
Jon Hall65844a32015-03-09 19:09:37 -070052 # load some variables from the params file
Jon Hall8f89dda2015-01-22 16:03:33 -080053 PULLCODE = False
Jon Hall6aec96b2015-01-19 14:49:31 -080054 if main.params[ 'Git' ] == 'True':
Jon Hall8f89dda2015-01-22 16:03:33 -080055 PULLCODE = True
Jon Hall529a37f2015-01-28 10:02:00 -080056 gitBranch = main.params[ 'branch' ]
Jon Hall8f89dda2015-01-22 16:03:33 -080057 cellName = main.params[ 'ENV' ][ 'cellName' ]
Jon Hall6aec96b2015-01-19 14:49:31 -080058
59 # set global variables
Jon Hall8f89dda2015-01-22 16:03:33 -080060 global ONOS1Port
Jon Hall8f89dda2015-01-22 16:03:33 -080061 global ONOS2Port
Jon Hall8f89dda2015-01-22 16:03:33 -080062 global ONOS3Port
Jon Hall8f89dda2015-01-22 16:03:33 -080063 global ONOS4Port
Jon Hall8f89dda2015-01-22 16:03:33 -080064 global ONOS5Port
Jon Hall8f89dda2015-01-22 16:03:33 -080065 global ONOS6Port
Jon Hall8f89dda2015-01-22 16:03:33 -080066 global ONOS7Port
67 global numControllers
Jon Hall8f89dda2015-01-22 16:03:33 -080068 numControllers = int( main.params[ 'num_controllers' ] )
Jon Hall5cfd23c2015-03-19 11:40:57 -070069
Jon Hall65844a32015-03-09 19:09:37 -070070 # FIXME: just get controller port from params?
71 # TODO: do we really need all these?
72 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
73 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
74 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
75 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
76 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
77 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
78 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
79
80 global CLIs
81 CLIs = []
82 global nodes
83 nodes = []
84 for i in range( 1, numControllers + 1 ):
85 CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
86 nodes.append( getattr( main, 'ONOS' + str( i ) ) )
Jon Hallb1290e82014-11-18 16:17:48 -050087
Jon Hall6aec96b2015-01-19 14:49:31 -080088 main.step( "Applying cell variable to environment" )
Jon Hall8f89dda2015-01-22 16:03:33 -080089 cellResult = main.ONOSbench.setCell( cellName )
90 verifyResult = main.ONOSbench.verifyCell()
Jon Hall368769f2014-11-19 15:43:35 -080091
Jon Hall6aec96b2015-01-19 14:49:31 -080092 # FIXME:this is short term fix
93 main.log.report( "Removing raft logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -080094 main.ONOSbench.onosRemoveRaftLogs()
Jon Hall65844a32015-03-09 19:09:37 -070095
Jon Hall6aec96b2015-01-19 14:49:31 -080096 main.log.report( "Uninstalling ONOS" )
Jon Hall65844a32015-03-09 19:09:37 -070097 for node in nodes:
98 main.ONOSbench.onosUninstall( node.ip_address )
Jon Hallb1290e82014-11-18 16:17:48 -050099
Jon Hall8f89dda2015-01-22 16:03:33 -0800100 cleanInstallResult = main.TRUE
101 gitPullResult = main.TRUE
Jon Hallb1290e82014-11-18 16:17:48 -0500102
Jon Hall97f31752015-02-04 12:01:04 -0800103 main.step( "Starting Mininet" )
104 main.Mininet1.startNet( )
105
Jon Hall6aec96b2015-01-19 14:49:31 -0800106 main.step( "Compiling the latest version of ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800107 if PULLCODE:
Jon Hall58c76b72015-02-23 11:09:24 -0800108 main.step( "Git checkout and pull " + gitBranch )
Jon Hall529a37f2015-01-28 10:02:00 -0800109 main.ONOSbench.gitCheckout( gitBranch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800110 gitPullResult = main.ONOSbench.gitPull()
Jon Hall5cfd23c2015-03-19 11:40:57 -0700111 if gitPullResult == main.ERROR:
112 main.log.error( "Error pulling git branch" )
Jon Hallb1290e82014-11-18 16:17:48 -0500113
Jon Hall6aec96b2015-01-19 14:49:31 -0800114 main.step( "Using mvn clean & install" )
Jon Hall529a37f2015-01-28 10:02:00 -0800115 cleanInstallResult = main.ONOSbench.cleanInstall()
116 else:
117 main.log.warn( "Did not pull new code so skipping mvn " +
118 "clean install" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800119 main.ONOSbench.getVersion( report=True )
Jon Hallb1290e82014-11-18 16:17:48 -0500120
Jon Hall6aec96b2015-01-19 14:49:31 -0800121 main.step( "Creating ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800122 packageResult = main.ONOSbench.onosPackage()
Jon Hallb1290e82014-11-18 16:17:48 -0500123
Jon Hall6aec96b2015-01-19 14:49:31 -0800124 main.step( "Installing ONOS package" )
Jon Hall65844a32015-03-09 19:09:37 -0700125 onosInstallResult = main.TRUE
126 for node in nodes:
127 tmpResult = main.ONOSbench.onosInstall( options="-f",
128 node=node.ip_address )
129 onosInstallResult = onosInstallResult and tmpResult
Jon Hallb1290e82014-11-18 16:17:48 -0500130
Jon Hall6aec96b2015-01-19 14:49:31 -0800131 main.step( "Checking if ONOS is up yet" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800132 for i in range( 2 ):
Jon Hall65844a32015-03-09 19:09:37 -0700133 onosIsupResult = main.TRUE
134 for node in nodes:
135 started = main.ONOSbench.isup( node.ip_address )
136 if not started:
137 main.log.report( node.name + " didn't start!" )
138 main.ONOSbench.onosStop( node.ip_address )
139 main.ONOSbench.onosStart( node.ip_address )
140 onosIsupResult = onosIsupResult and started
Jon Hall8f89dda2015-01-22 16:03:33 -0800141 if onosIsupResult == main.TRUE:
Jon Hallffb386d2014-11-21 13:43:38 -0800142 break
Jon Hallb1290e82014-11-18 16:17:48 -0500143
Jon Hall65844a32015-03-09 19:09:37 -0700144 main.log.step( "Starting ONOS CLI sessions" )
145 cliResults = main.TRUE
146 threads = []
147 for i in range( numControllers ):
148 t = main.Thread( target=CLIs[i].startOnosCli,
Jon Hall65844a32015-03-09 19:09:37 -0700149 name="startOnosCli-" + str( i ),
150 args=[nodes[i].ip_address] )
151 threads.append( t )
152 t.start()
Jon Hall65844a32015-03-09 19:09:37 -0700153
154 for t in threads:
155 t.join()
156 cliResults = cliResults and t.result
Jon Hallb1290e82014-11-18 16:17:48 -0500157
Jon Hall6aec96b2015-01-19 14:49:31 -0800158 main.step( "Start Packet Capture MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800159 main.Mininet2.startTcpdump(
Jon Hall6aec96b2015-01-19 14:49:31 -0800160 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
161 + "-MN.pcap",
162 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
163 port=main.params[ 'MNtcpdump' ][ 'port' ] )
Jon Hallb1290e82014-11-18 16:17:48 -0500164
Jon Halla9d26da2015-03-30 16:45:32 -0700165 appCheck = main.TRUE
166 threads = []
167 for i in range( numControllers ):
168 t = main.Thread( target=CLIs[i].appToIDCheck,
169 name="appToIDCheck-" + str( i ),
170 args=[] )
171 threads.append( t )
172 t.start()
173
174 for t in threads:
175 t.join()
176 appCheck = appCheck and t.result
177 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
178 onpass="App Ids seem to be correct",
179 onfail="Something is wrong with app Ids" )
180 if appCheck != main.TRUE:
181 main.log.warn( CLIs[0].apps() )
182 main.log.warn( CLIs[0].appIDs() )
183
Jon Hall8f89dda2015-01-22 16:03:33 -0800184 case1Result = ( cleanInstallResult and packageResult and
185 cellResult and verifyResult and onosInstallResult
Jon Halla9d26da2015-03-30 16:45:32 -0700186 and onosIsupResult and cliResults and appCheck)
Jon Hallb1290e82014-11-18 16:17:48 -0500187
Jon Hall8f89dda2015-01-22 16:03:33 -0800188 utilities.assert_equals( expect=main.TRUE, actual=case1Result,
Jon Hall58c76b72015-02-23 11:09:24 -0800189 onpass="Test startup successful",
190 onfail="Test startup NOT successful" )
Jon Hallb1290e82014-11-18 16:17:48 -0500191
Jon Hall8f89dda2015-01-22 16:03:33 -0800192 if case1Result == main.FALSE:
Jon Hallffb386d2014-11-21 13:43:38 -0800193 main.cleanup()
194 main.exit()
Jon Hallb1290e82014-11-18 16:17:48 -0500195
Jon Hall6aec96b2015-01-19 14:49:31 -0800196 def CASE2( self, main ):
197 """
Jon Hallb1290e82014-11-18 16:17:48 -0500198 Assign mastership to controllers
Jon Hall6aec96b2015-01-19 14:49:31 -0800199 """
Jon Hallb1290e82014-11-18 16:17:48 -0500200 import re
Jon Hall5cfd23c2015-03-19 11:40:57 -0700201 assert numControllers, "numControllers not defined"
202 assert main, "main not defined"
203 assert utilities.assert_equals, "utilities.assert_equals not defined"
204 assert CLIs, "CLIs not defined"
205 assert nodes, "nodes not defined"
206 assert ONOS1Port, "ONOS1Port not defined"
207 assert ONOS2Port, "ONOS2Port not defined"
208 assert ONOS3Port, "ONOS3Port not defined"
209 assert ONOS4Port, "ONOS4Port not defined"
210 assert ONOS5Port, "ONOS5Port not defined"
211 assert ONOS6Port, "ONOS6Port not defined"
212 assert ONOS7Port, "ONOS7Port not defined"
Jon Hallb1290e82014-11-18 16:17:48 -0500213
Jon Hall6aec96b2015-01-19 14:49:31 -0800214 main.log.report( "Assigning switches to controllers" )
215 main.case( "Assigning Controllers" )
216 main.step( "Assign switches to controllers" )
Jon Hallb1290e82014-11-18 16:17:48 -0500217
Jon Hall65844a32015-03-09 19:09:37 -0700218 # TODO: rewrite this function to take lists of ips and ports?
219 # or list of tuples?
Jon Hall6aec96b2015-01-19 14:49:31 -0800220 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800221 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -0800222 sw=str( i ),
Jon Hall8f89dda2015-01-22 16:03:33 -0800223 count=numControllers,
Jon Hall65844a32015-03-09 19:09:37 -0700224 ip1=nodes[ 0 ].ip_address, port1=ONOS1Port,
225 ip2=nodes[ 1 ].ip_address, port2=ONOS2Port,
226 ip3=nodes[ 2 ].ip_address, port3=ONOS3Port,
227 ip4=nodes[ 3 ].ip_address, port4=ONOS4Port,
228 ip5=nodes[ 4 ].ip_address, port5=ONOS5Port,
229 ip6=nodes[ 5 ].ip_address, port6=ONOS6Port,
230 ip7=nodes[ 6 ].ip_address, port7=ONOS7Port )
Jon Hallb1290e82014-11-18 16:17:48 -0500231
Jon Hall8f89dda2015-01-22 16:03:33 -0800232 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800233 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800234 response = main.Mininet1.getSwController( "s" + str( i ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800235 try:
Jon Hall6aec96b2015-01-19 14:49:31 -0800236 main.log.info( str( response ) )
Jon Hallfebb1c72015-03-05 13:30:09 -0800237 except Exception:
Jon Hall6aec96b2015-01-19 14:49:31 -0800238 main.log.info( repr( response ) )
Jon Hall65844a32015-03-09 19:09:37 -0700239 for node in nodes:
240 if re.search( "tcp:" + node.ip_address, response ):
241 mastershipCheck = mastershipCheck and main.TRUE
242 else:
Jon Halla9d26da2015-03-30 16:45:32 -0700243 main.log.error( "Error, node " + node.ip_address + " is " +
244 "not in the list of controllers s" +
245 str( i ) + " is connecting to." )
Jon Hall65844a32015-03-09 19:09:37 -0700246 mastershipCheck = main.FALSE
Jon Hall8f89dda2015-01-22 16:03:33 -0800247 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800248 main.log.report( "Switch mastership assigned correctly" )
249 utilities.assert_equals(
250 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800251 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800252 onpass="Switch mastership assigned correctly",
253 onfail="Switches not assigned correctly to controllers" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800254 # Manually assign mastership to the controller we want
Jon Hall8f89dda2015-01-22 16:03:33 -0800255 roleCall = main.TRUE
256 roleCheck = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -0800257 try:
Jon Halla9d26da2015-03-30 16:45:32 -0700258 for i in range( 1, 29 ): # switches 1 through 28
259 # set up correct variables:
260 if i == 1:
261 ip = nodes[ 0 ].ip_address # ONOS1
262 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
263 elif i == 2:
264 ip = nodes[ 1 ].ip_address # ONOS2
265 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
266 elif i == 3:
267 ip = nodes[ 1 ].ip_address # ONOS2
268 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
269 elif i == 4:
270 ip = nodes[ 3 ].ip_address # ONOS4
271 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
272 elif i == 5:
273 ip = nodes[ 2 ].ip_address # ONOS3
274 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
275 elif i == 6:
276 ip = nodes[ 2 ].ip_address # ONOS3
277 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
278 elif i == 7:
279 ip = nodes[ 5 ].ip_address # ONOS6
280 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
281 elif i >= 8 and i <= 17:
282 ip = nodes[ 4 ].ip_address # ONOS5
283 dpid = '3' + str( i ).zfill( 3 )
284 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
285 elif i >= 18 and i <= 27:
286 ip = nodes[ 6 ].ip_address # ONOS7
287 dpid = '6' + str( i ).zfill( 3 )
288 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
289 elif i == 28:
290 ip = nodes[ 0 ].ip_address # ONOS1
291 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
292 else:
293 main.log.error( "You didn't write an else statement for " +
294 "switch s" + str( i ) )
295 # Assign switch
296 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
297 # TODO: make this controller dynamic
298 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId,
299 ip )
Jon Hall58c76b72015-02-23 11:09:24 -0800300 # Check assignment
Jon Hall678f4512015-03-31 09:48:31 -0700301 master = main.ONOScli1.getRole( deviceId ).get( 'master' )
Jon Halla9d26da2015-03-30 16:45:32 -0700302 if ip in master:
Jon Hall58c76b72015-02-23 11:09:24 -0800303 roleCheck = roleCheck and main.TRUE
304 else:
305 roleCheck = roleCheck and main.FALSE
Jon Halla9d26da2015-03-30 16:45:32 -0700306 main.log.error( "Error, controller " + ip + " is not" +
307 " master " + "of device " +
308 str( deviceId ) + ". Master is " +
309 repr( master ) + "." )
Jon Hall58c76b72015-02-23 11:09:24 -0800310 except ( AttributeError, AssertionError ):
311 main.log.exception( "Something is wrong with ONOS device view" )
312 main.log.info( main.ONOScli1.devices() )
Jon Hall6aec96b2015-01-19 14:49:31 -0800313 utilities.assert_equals(
314 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800315 actual=roleCall,
Jon Hall6aec96b2015-01-19 14:49:31 -0800316 onpass="Re-assigned switch mastership to designated controller",
Jon Hall8f89dda2015-01-22 16:03:33 -0800317 onfail="Something wrong with deviceRole calls" )
Jon Hall94fd0472014-12-08 11:52:42 -0800318
Jon Hall6aec96b2015-01-19 14:49:31 -0800319 utilities.assert_equals(
320 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800321 actual=roleCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800322 onpass="Switches were successfully reassigned to designated " +
323 "controller",
324 onfail="Switches were not successfully reassigned" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800325 mastershipCheck = mastershipCheck and roleCall and roleCheck
326 utilities.assert_equals( expect=main.TRUE, actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800327 onpass="Switch mastership correctly assigned",
328 onfail="Error in (re)assigning switch" +
Jon Hall8f89dda2015-01-22 16:03:33 -0800329 " mastership" )
Jon Hallb1290e82014-11-18 16:17:48 -0500330
Jon Hall6aec96b2015-01-19 14:49:31 -0800331 def CASE3( self, main ):
Jon Hallb1290e82014-11-18 16:17:48 -0500332 """
333 Assign intents
Jon Hallb1290e82014-11-18 16:17:48 -0500334 """
335 import time
Jon Hall58c76b72015-02-23 11:09:24 -0800336 import json
Jon Hall5cfd23c2015-03-19 11:40:57 -0700337 assert numControllers, "numControllers not defined"
338 assert main, "main not defined"
339 assert utilities.assert_equals, "utilities.assert_equals not defined"
340 assert CLIs, "CLIs not defined"
341 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -0800342 main.log.report( "Adding host intents" )
343 main.case( "Adding host Intents" )
Jon Hallb1290e82014-11-18 16:17:48 -0500344
Jon Hall8f89dda2015-01-22 16:03:33 -0800345 main.step( "Discovering Hosts( Via pingall for now )" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800346 # FIXME: Once we have a host discovery mechanism, use that instead
Jon Hallb1290e82014-11-18 16:17:48 -0500347
Jon Hall6aec96b2015-01-19 14:49:31 -0800348 # install onos-app-fwd
349 main.log.info( "Install reactive forwarding app" )
Jon Halla9d26da2015-03-30 16:45:32 -0700350 appResults = CLIs[0].activateApp( "org.onosproject.fwd" )
351
352 # FIXME: add this to asserts
353 appCheck = main.TRUE
Jon Hall65844a32015-03-09 19:09:37 -0700354 threads = []
355 for i in range( numControllers ):
Jon Halla9d26da2015-03-30 16:45:32 -0700356 t = main.Thread( target=CLIs[i].appToIDCheck,
357 name="appToIDCheck-" + str( i ),
358 args=[] )
Jon Hall65844a32015-03-09 19:09:37 -0700359 threads.append( t )
360 t.start()
Jon Hall65844a32015-03-09 19:09:37 -0700361
362 for t in threads:
363 t.join()
Jon Halla9d26da2015-03-30 16:45:32 -0700364 appCheck = appCheck and t.result
365 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
366 onpass="App Ids seem to be correct",
367 onfail="Something is wrong with app Ids" )
368 if appCheck != main.TRUE:
369 main.log.warn( CLIs[0].apps() )
370 main.log.warn( CLIs[0].appIDs() )
Jon Hall94fd0472014-12-08 11:52:42 -0800371
Jon Hall6aec96b2015-01-19 14:49:31 -0800372 # REACTIVE FWD test
Jon Hall8f89dda2015-01-22 16:03:33 -0800373 pingResult = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -0700374 for i in range(2): # Retry if pingall fails first time
375 time1 = time.time()
376 pingResult = main.Mininet1.pingall()
377 utilities.assert_equals(
378 expect=main.TRUE,
379 actual=pingResult,
380 onpass="Reactive Pingall test passed",
381 onfail="Reactive Pingall failed, one or more ping pairs failed" )
382 time2 = time.time()
383 main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
Jon Hallb1290e82014-11-18 16:17:48 -0500384
Jon Hall6aec96b2015-01-19 14:49:31 -0800385 # uninstall onos-app-fwd
386 main.log.info( "Uninstall reactive forwarding app" )
Jon Halla9d26da2015-03-30 16:45:32 -0700387 appResults = appResults and CLIs[0].deactivateApp( "org.onosproject.fwd" )
Jon Hall65844a32015-03-09 19:09:37 -0700388 threads = []
389 for i in range( numControllers ):
Jon Halla9d26da2015-03-30 16:45:32 -0700390 t = main.Thread( target=CLIs[i].appToIDCheck,
391 name="appToIDCheck-" + str( i ),
392 args=[] )
Jon Hall65844a32015-03-09 19:09:37 -0700393 threads.append( t )
394 t.start()
Jon Hall65844a32015-03-09 19:09:37 -0700395
396 for t in threads:
397 t.join()
Jon Halla9d26da2015-03-30 16:45:32 -0700398 appCheck = appCheck and t.result
399 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
400 onpass="App Ids seem to be correct",
401 onfail="Something is wrong with app Ids" )
402 if appCheck != main.TRUE:
403 main.log.warn( CLIs[0].apps() )
404 main.log.warn( CLIs[0].appIDs() )
Jon Hall65844a32015-03-09 19:09:37 -0700405
Jon Hall6aec96b2015-01-19 14:49:31 -0800406 # timeout for fwd flows
Jon Hall5cfd23c2015-03-19 11:40:57 -0700407 time.sleep( 11 )
Jon Hallb1290e82014-11-18 16:17:48 -0500408
Jon Hall5cfd23c2015-03-19 11:40:57 -0700409 main.step( "Add host intents" )
Jon Hall58c76b72015-02-23 11:09:24 -0800410 intentIds = []
Jon Hall6aec96b2015-01-19 14:49:31 -0800411 # TODO: move the host numbers to params
Jon Hall58c76b72015-02-23 11:09:24 -0800412 # Maybe look at all the paths we ping?
Jon Hall8f89dda2015-01-22 16:03:33 -0800413 intentAddResult = True
Jon Hall58c76b72015-02-23 11:09:24 -0800414 hostResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800415 for i in range( 8, 18 ):
416 main.log.info( "Adding host intent between h" + str( i ) +
417 " and h" + str( i + 10 ) )
418 host1 = "00:00:00:00:00:" + \
419 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
420 host2 = "00:00:00:00:00:" + \
421 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800422 # NOTE: getHost can return None
Jon Hall5cfd23c2015-03-19 11:40:57 -0700423 host1Dict = main.ONOScli1.getHost( host1 )
424 host2Dict = main.ONOScli1.getHost( host2 )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800425 host1Id = None
426 host2Id = None
427 if host1Dict and host2Dict:
428 host1Id = host1Dict.get( 'id', None )
429 host2Id = host2Dict.get( 'id', None )
Jon Hall8f89dda2015-01-22 16:03:33 -0800430 if host1Id and host2Id:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700431 nodeNum = ( i % 7 )
432 tmpId = CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
Jon Hall63604932015-02-26 17:09:50 -0800433 if tmpId:
434 main.log.info( "Added intent with id: " + tmpId )
435 intentIds.append( tmpId )
436 else:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700437 main.log.error( "addHostIntent returned: " +
438 repr( tmpId ) )
Jon Hall669173b2014-12-17 11:36:30 -0800439 else:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700440 main.log.error( "Error, getHost() failed for h" + str( i ) +
441 " and/or h" + str( i + 10 ) )
442 hosts = CLIs[ 0 ].hosts()
443 main.log.warn( "Hosts output: " )
444 try:
445 main.log.warn( json.dumps( json.loads( hosts ),
446 sort_keys=True,
447 indent=4,
448 separators=( ',', ': ' ) ) )
449 except ( ValueError, TypeError ):
450 main.log.warn( repr( hosts ) )
Jon Hall58c76b72015-02-23 11:09:24 -0800451 hostResult = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -0700452 # FIXME: DEBUG
453 intentStart = time.time()
Jon Hall58c76b72015-02-23 11:09:24 -0800454 onosIds = main.ONOScli1.getAllIntentsId()
455 main.log.info( "Submitted intents: " + str( intentIds ) )
456 main.log.info( "Intents in ONOS: " + str( onosIds ) )
457 for intent in intentIds:
458 if intent in onosIds:
Jon Hall5cfd23c2015-03-19 11:40:57 -0700459 pass # intent submitted is in onos
Jon Hall58c76b72015-02-23 11:09:24 -0800460 else:
461 intentAddResult = False
Jon Hall5cfd23c2015-03-19 11:40:57 -0700462 # FIXME: DEBUG
463 if intentAddResult:
464 intentStop = time.time()
465 else:
466 intentStop = None
Jon Hall1b8f54a2015-02-04 13:24:20 -0800467 # Print the intent states
Jon Hall58c76b72015-02-23 11:09:24 -0800468 intents = main.ONOScli1.intents()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800469 intentStates = []
Jon Hall65844a32015-03-09 19:09:37 -0700470 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800471 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
472 count = 0
Jon Hall5cfd23c2015-03-19 11:40:57 -0700473 try:
474 for intent in json.loads( intents ):
475 state = intent.get( 'state', None )
476 if "INSTALLED" not in state:
477 installedCheck = False
478 intentId = intent.get( 'id', None )
479 intentStates.append( ( intentId, state ) )
480 except ( ValueError, TypeError ):
481 main.log.exception( "Error parsing intents" )
Jon Hall58c76b72015-02-23 11:09:24 -0800482 # add submitted intents not in the store
483 tmplist = [ i for i, s in intentStates ]
484 missingIntents = False
485 for i in intentIds:
486 if i not in tmplist:
487 intentStates.append( ( i, " - " ) )
488 missingIntents = True
489 intentStates.sort()
490 for i, s in intentStates:
491 count += 1
492 main.log.info( "%-6s%-15s%-15s" %
493 ( str( count ), str( i ), str( s ) ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700494 leaders = main.ONOScli1.leaders()
495 try:
496 if leaders:
497 parsedLeaders = json.loads( leaders )
498 main.log.warn( json.dumps( parsedLeaders,
499 sort_keys=True,
500 indent=4,
501 separators=( ',', ': ' ) ) )
502 # check for all intent partitions
Jon Hall5cfd23c2015-03-19 11:40:57 -0700503 topics = []
504 for i in range( 14 ):
505 topics.append( "intent-partition-" + str( i ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700506 main.log.debug( topics )
507 ONOStopics = [ j['topic'] for j in parsedLeaders ]
508 for topic in topics:
509 if topic not in ONOStopics:
510 main.log.error( "Error: " + topic +
511 " not in leaders" )
512 else:
513 main.log.error( "leaders() returned None" )
514 except ( ValueError, TypeError ):
515 main.log.exception( "Error parsing leaders" )
516 main.log.error( repr( leaders ) )
517 partitions = main.ONOScli1.partitions()
518 try:
519 if partitions :
520 parsedPartitions = json.loads( partitions )
521 main.log.warn( json.dumps( parsedPartitions,
522 sort_keys=True,
523 indent=4,
524 separators=( ',', ': ' ) ) )
525 # TODO check for a leader in all paritions
526 # TODO check for consistency among nodes
527 else:
528 main.log.error( "partitions() returned None" )
529 except ( ValueError, TypeError ):
530 main.log.exception( "Error parsing partitions" )
531 main.log.error( repr( partitions ) )
Jon Hall63604932015-02-26 17:09:50 -0800532 pendingMap = main.ONOScli1.pendingMap()
Jon Hall5cfd23c2015-03-19 11:40:57 -0700533 try:
534 if pendingMap :
535 parsedPending = json.loads( pendingMap )
536 main.log.warn( json.dumps( parsedPending,
537 sort_keys=True,
538 indent=4,
539 separators=( ',', ': ' ) ) )
540 # TODO check something here?
541 else:
542 main.log.error( "pendingMap() returned None" )
543 except ( ValueError, TypeError ):
544 main.log.exception( "Error parsing pending map" )
545 main.log.error( repr( pendingMap ) )
546
Jon Hall58c76b72015-02-23 11:09:24 -0800547 intentAddResult = bool( pingResult and hostResult and intentAddResult
Jon Hall63604932015-02-26 17:09:50 -0800548 and not missingIntents and installedCheck )
Jon Hall6aec96b2015-01-19 14:49:31 -0800549 utilities.assert_equals(
550 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800551 actual=intentAddResult,
552 onpass="Pushed host intents to ONOS",
553 onfail="Error in pushing host intents to ONOS" )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700554 for i in range(100):
555 onosIds = main.ONOScli1.getAllIntentsId()
556 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
557 main.log.info( "Intents in ONOS: " + str( sorted( onosIds ) ) )
558 if sorted(onosIds) == sorted(intentIds):
559 break
560 else:
561 time.sleep(1)
562 # FIXME: DEBUG
563 if not intentStop:
564 intentStop = time.time()
565 gossipTime = intentStop - intentStart
566 main.log.info( "It took about " + str( gossipTime ) +
567 " seconds for all intents to appear on ONOS1" )
Jon Halla9d26da2015-03-30 16:45:32 -0700568 # FIXME: make this time configurable/calculate based off of number of
569 # nodes and gossip rounds
Jon Hall5cfd23c2015-03-19 11:40:57 -0700570 utilities.assert_greater_equals(
571 expect=30, actual=gossipTime,
572 onpass="ECM anti-entropy for intents worked within " +
573 "expected time",
574 onfail="Intent ECM anti-entropy took too long" )
Jon Halla9d26da2015-03-30 16:45:32 -0700575 if gossipTime <= 30:
576 intentAddResult = True
Jon Hall58c76b72015-02-23 11:09:24 -0800577
Jon Hall63604932015-02-26 17:09:50 -0800578 if not intentAddResult or "key" in pendingMap:
Jon Hall58c76b72015-02-23 11:09:24 -0800579 import time
Jon Hall63604932015-02-26 17:09:50 -0800580 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800581 main.log.info( "Sleeping 60 seconds to see if intents are found" )
582 time.sleep( 60 )
583 onosIds = main.ONOScli1.getAllIntentsId()
584 main.log.info( "Submitted intents: " + str( intentIds ) )
585 main.log.info( "Intents in ONOS: " + str( onosIds ) )
586 # Print the intent states
587 intents = main.ONOScli1.intents()
588 intentStates = []
589 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
590 count = 0
Jon Hall5cfd23c2015-03-19 11:40:57 -0700591 try:
592 for intent in json.loads( intents ):
593 # Iter through intents of a node
594 state = intent.get( 'state', None )
595 if "INSTALLED" not in state:
596 installedCheck = False
597 intentId = intent.get( 'id', None )
598 intentStates.append( ( intentId, state ) )
599 except ( ValueError, TypeError ):
600 main.log.exception( "Error parsing intents" )
Jon Hall58c76b72015-02-23 11:09:24 -0800601 # add submitted intents not in the store
602 tmplist = [ i for i, s in intentStates ]
603 for i in intentIds:
604 if i not in tmplist:
605 intentStates.append( ( i, " - " ) )
606 intentStates.sort()
607 for i, s in intentStates:
608 count += 1
609 main.log.info( "%-6s%-15s%-15s" %
610 ( str( count ), str( i ), str( s ) ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700611 leaders = main.ONOScli1.leaders()
612 try:
613 if leaders:
614 parsedLeaders = json.loads( leaders )
615 main.log.warn( json.dumps( parsedLeaders,
616 sort_keys=True,
617 indent=4,
618 separators=( ',', ': ' ) ) )
619 # check for all intent partitions
620 # check for election
621 topics = []
622 for i in range( 14 ):
623 topics.append( "intent-partition-" + str( i ) )
624 # FIXME: this should only be after we start the app
625 topics.append( "org.onosproject.election" )
626 main.log.debug( topics )
627 ONOStopics = [ j['topic'] for j in parsedLeaders ]
628 for topic in topics:
629 if topic not in ONOStopics:
630 main.log.error( "Error: " + topic +
631 " not in leaders" )
632 else:
633 main.log.error( "leaders() returned None" )
634 except ( ValueError, TypeError ):
635 main.log.exception( "Error parsing leaders" )
636 main.log.error( repr( leaders ) )
637 partitions = main.ONOScli1.partitions()
638 try:
639 if partitions :
640 parsedPartitions = json.loads( partitions )
641 main.log.warn( json.dumps( parsedPartitions,
642 sort_keys=True,
643 indent=4,
644 separators=( ',', ': ' ) ) )
645 # TODO check for a leader in all paritions
646 # TODO check for consistency among nodes
647 else:
648 main.log.error( "partitions() returned None" )
649 except ( ValueError, TypeError ):
650 main.log.exception( "Error parsing partitions" )
651 main.log.error( repr( partitions ) )
652 pendingMap = main.ONOScli1.pendingMap()
653 try:
654 if pendingMap :
655 parsedPending = json.loads( pendingMap )
656 main.log.warn( json.dumps( parsedPending,
657 sort_keys=True,
658 indent=4,
659 separators=( ',', ': ' ) ) )
660 # TODO check something here?
661 else:
662 main.log.error( "pendingMap() returned None" )
663 except ( ValueError, TypeError ):
664 main.log.exception( "Error parsing pending map" )
665 main.log.error( repr( pendingMap ) )
Jon Hallb1290e82014-11-18 16:17:48 -0500666
Jon Hall6aec96b2015-01-19 14:49:31 -0800667 def CASE4( self, main ):
Jon Hallb1290e82014-11-18 16:17:48 -0500668 """
669 Ping across added host intents
670 """
Jon Hall58c76b72015-02-23 11:09:24 -0800671 import json
Jon Hall65844a32015-03-09 19:09:37 -0700672 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -0700673 assert numControllers, "numControllers not defined"
674 assert main, "main not defined"
675 assert utilities.assert_equals, "utilities.assert_equals not defined"
676 assert CLIs, "CLIs not defined"
677 assert nodes, "nodes not defined"
Jon Hall368769f2014-11-19 15:43:35 -0800678 description = " Ping across added host intents"
Jon Hall6aec96b2015-01-19 14:49:31 -0800679 main.log.report( description )
680 main.case( description )
Jon Hall8f89dda2015-01-22 16:03:33 -0800681 PingResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800682 for i in range( 8, 18 ):
Jon Hall58c76b72015-02-23 11:09:24 -0800683 ping = main.Mininet1.pingHost( src="h" + str( i ),
684 target="h" + str( i + 10 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800685 PingResult = PingResult and ping
Jon Hall6aec96b2015-01-19 14:49:31 -0800686 if ping == main.FALSE:
687 main.log.warn( "Ping failed between h" + str( i ) +
688 " and h" + str( i + 10 ) )
689 elif ping == main.TRUE:
690 main.log.info( "Ping test passed!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800691 # Don't set PingResult or you'd override failures
Jon Hall8f89dda2015-01-22 16:03:33 -0800692 if PingResult == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800693 main.log.report(
694 "Intents have not been installed correctly, pings failed." )
Jon Hall58c76b72015-02-23 11:09:24 -0800695 # TODO: pretty print
Jon Hall65844a32015-03-09 19:09:37 -0700696 main.log.warn( "ONOS1 intents: " )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700697 try:
698 tmpIntents = main.ONOScli1.intents()
699 main.log.warn( json.dumps( json.loads( tmpIntents ),
700 sort_keys=True,
701 indent=4,
702 separators=( ',', ': ' ) ) )
703 except ( ValueError, TypeError ):
704 main.log.warn( repr( tmpIntents ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800705 if PingResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800706 main.log.report(
707 "Intents have been installed correctly and verified by pings" )
708 utilities.assert_equals(
709 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800710 actual=PingResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800711 onpass="Intents have been installed correctly and pings work",
712 onfail="Intents have not been installed correctly, pings failed." )
Jon Hallb1290e82014-11-18 16:17:48 -0500713
Jon Hall63604932015-02-26 17:09:50 -0800714 installedCheck = True
Jon Hall58c76b72015-02-23 11:09:24 -0800715 if PingResult is not main.TRUE:
716 # Print the intent states
717 intents = main.ONOScli1.intents()
718 intentStates = []
719 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
720 count = 0
721 # Iter through intents of a node
Jon Hall5cfd23c2015-03-19 11:40:57 -0700722 try:
723 for intent in json.loads( intents ):
724 state = intent.get( 'state', None )
725 if "INSTALLED" not in state:
726 installedCheck = False
727 intentId = intent.get( 'id', None )
728 intentStates.append( ( intentId, state ) )
729 except ( ValueError, TypeError ):
730 main.log.exception( "Error parsing intents." )
Jon Hall58c76b72015-02-23 11:09:24 -0800731 intentStates.sort()
732 for i, s in intentStates:
733 count += 1
734 main.log.info( "%-6s%-15s%-15s" %
735 ( str( count ), str( i ), str( s ) ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700736 leaders = main.ONOScli1.leaders()
737 try:
738 if leaders:
739 parsedLeaders = json.loads( leaders )
740 main.log.warn( json.dumps( parsedLeaders,
741 sort_keys=True,
742 indent=4,
743 separators=( ',', ': ' ) ) )
744 # check for all intent partitions
745 # check for election
746 topics = []
747 for i in range( 14 ):
748 topics.append( "intent-partition-" + str( i ) )
749 # FIXME: this should only be after we start the app
750 topics.append( "org.onosproject.election" )
751 main.log.debug( topics )
752 ONOStopics = [ j['topic'] for j in parsedLeaders ]
753 for topic in topics:
754 if topic not in ONOStopics:
755 main.log.error( "Error: " + topic +
756 " not in leaders" )
757 else:
758 main.log.error( "leaders() returned None" )
759 except ( ValueError, TypeError ):
760 main.log.exception( "Error parsing leaders" )
761 main.log.error( repr( leaders ) )
762 partitions = main.ONOScli1.partitions()
763 try:
764 if partitions :
765 parsedPartitions = json.loads( partitions )
766 main.log.warn( json.dumps( parsedPartitions,
767 sort_keys=True,
768 indent=4,
769 separators=( ',', ': ' ) ) )
770 # TODO check for a leader in all paritions
771 # TODO check for consistency among nodes
772 else:
773 main.log.error( "partitions() returned None" )
774 except ( ValueError, TypeError ):
775 main.log.exception( "Error parsing partitions" )
776 main.log.error( repr( partitions ) )
777 pendingMap = main.ONOScli1.pendingMap()
778 try:
779 if pendingMap :
780 parsedPending = json.loads( pendingMap )
781 main.log.warn( json.dumps( parsedPending,
782 sort_keys=True,
783 indent=4,
784 separators=( ',', ': ' ) ) )
785 # TODO check something here?
786 else:
787 main.log.error( "pendingMap() returned None" )
788 except ( ValueError, TypeError ):
789 main.log.exception( "Error parsing pending map" )
790 main.log.error( repr( pendingMap ) )
791
Jon Hall63604932015-02-26 17:09:50 -0800792 if not installedCheck:
Jon Hall65844a32015-03-09 19:09:37 -0700793 main.log.info( "Waiting 60 seconds to see if the state of " +
794 "intents change" )
Jon Hall63604932015-02-26 17:09:50 -0800795 time.sleep( 60 )
796 # Print the intent states
797 intents = main.ONOScli1.intents()
798 intentStates = []
799 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
800 count = 0
801 # Iter through intents of a node
Jon Hall5cfd23c2015-03-19 11:40:57 -0700802 try:
803 for intent in json.loads( intents ):
804 state = intent.get( 'state', None )
805 if "INSTALLED" not in state:
806 installedCheck = False
807 intentId = intent.get( 'id', None )
808 intentStates.append( ( intentId, state ) )
809 except ( ValueError, TypeError ):
810 main.log.exception( "Error parsing intents." )
Jon Hall63604932015-02-26 17:09:50 -0800811 intentStates.sort()
812 for i, s in intentStates:
813 count += 1
814 main.log.info( "%-6s%-15s%-15s" %
815 ( str( count ), str( i ), str( s ) ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -0700816 leaders = main.ONOScli1.leaders()
817 try:
818 if leaders:
819 parsedLeaders = json.loads( leaders )
820 main.log.warn( json.dumps( parsedLeaders,
821 sort_keys=True,
822 indent=4,
823 separators=( ',', ': ' ) ) )
824 # check for all intent partitions
825 # check for election
826 topics = []
827 for i in range( 14 ):
828 topics.append( "intent-partition-" + str( i ) )
829 # FIXME: this should only be after we start the app
830 topics.append( "org.onosproject.election" )
831 main.log.debug( topics )
832 ONOStopics = [ j['topic'] for j in parsedLeaders ]
833 for topic in topics:
834 if topic not in ONOStopics:
835 main.log.error( "Error: " + topic +
836 " not in leaders" )
837 else:
838 main.log.error( "leaders() returned None" )
839 except ( ValueError, TypeError ):
840 main.log.exception( "Error parsing leaders" )
841 main.log.error( repr( leaders ) )
842 partitions = main.ONOScli1.partitions()
843 try:
844 if partitions :
845 parsedPartitions = json.loads( partitions )
846 main.log.warn( json.dumps( parsedPartitions,
847 sort_keys=True,
848 indent=4,
849 separators=( ',', ': ' ) ) )
850 # TODO check for a leader in all paritions
851 # TODO check for consistency among nodes
852 else:
853 main.log.error( "partitions() returned None" )
854 except ( ValueError, TypeError ):
855 main.log.exception( "Error parsing partitions" )
856 main.log.error( repr( partitions ) )
857 pendingMap = main.ONOScli1.pendingMap()
858 try:
859 if pendingMap :
860 parsedPending = json.loads( pendingMap )
861 main.log.warn( json.dumps( parsedPending,
862 sort_keys=True,
863 indent=4,
864 separators=( ',', ': ' ) ) )
865 # TODO check something here?
866 else:
867 main.log.error( "pendingMap() returned None" )
868 except ( ValueError, TypeError ):
869 main.log.exception( "Error parsing pending map" )
870 main.log.error( repr( pendingMap ) )
Jon Hallb1290e82014-11-18 16:17:48 -0500871
Jon Hall6aec96b2015-01-19 14:49:31 -0800872 def CASE5( self, main ):
873 """
Jon Hallb1290e82014-11-18 16:17:48 -0500874 Reading state of ONOS
Jon Hall6aec96b2015-01-19 14:49:31 -0800875 """
Jon Hallb1290e82014-11-18 16:17:48 -0500876 import json
Jon Hall5cfd23c2015-03-19 11:40:57 -0700877 import time
878 assert numControllers, "numControllers not defined"
879 assert main, "main not defined"
880 assert utilities.assert_equals, "utilities.assert_equals not defined"
881 assert CLIs, "CLIs not defined"
882 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -0800883 # assumes that sts is already in you PYTHONPATH
884 from sts.topology.teston_topology import TestONTopology
Jon Hallb1290e82014-11-18 16:17:48 -0500885
Jon Hall6aec96b2015-01-19 14:49:31 -0800886 main.log.report( "Setting up and gathering data for current state" )
887 main.case( "Setting up and gathering data for current state" )
888 # The general idea for this test case is to pull the state of
889 # ( intents,flows, topology,... ) from each ONOS node
Jon Hall65844a32015-03-09 19:09:37 -0700890 # We can then compare them with each other and also with past states
Jon Hallb1290e82014-11-18 16:17:48 -0500891
Jon Hall65844a32015-03-09 19:09:37 -0700892 main.step( "Check that each switch has a master" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800893 global mastershipState
Jon Hall5cfd23c2015-03-19 11:40:57 -0700894 mastershipState = '[]'
Jon Hall94fd0472014-12-08 11:52:42 -0800895
Jon Hall6aec96b2015-01-19 14:49:31 -0800896 # Assert that each device has a master
Jon Hall65844a32015-03-09 19:09:37 -0700897 rolesNotNull = main.TRUE
898 threads = []
899 for i in range( numControllers ):
900 t = main.Thread( target=CLIs[i].rolesNotNull,
Jon Hall65844a32015-03-09 19:09:37 -0700901 name="rolesNotNull-" + str( i ),
902 args=[] )
903 threads.append( t )
904 t.start()
Jon Hall65844a32015-03-09 19:09:37 -0700905
906 for t in threads:
907 t.join()
908 rolesNotNull = rolesNotNull and t.result
Jon Hall6aec96b2015-01-19 14:49:31 -0800909 utilities.assert_equals(
910 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800911 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -0800912 onpass="Each device has a master",
913 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -0800914
Jon Hall65844a32015-03-09 19:09:37 -0700915 main.step( "Get the Mastership of each switch from each controller" )
916 ONOSMastership = []
917 mastershipCheck = main.FALSE
918 consistentMastership = True
919 rolesResults = True
920 threads = []
921 for i in range( numControllers ):
922 t = main.Thread( target=CLIs[i].roles,
Jon Hall65844a32015-03-09 19:09:37 -0700923 name="roles-" + str( i ),
924 args=[] )
925 threads.append( t )
926 t.start()
Jon Hall65844a32015-03-09 19:09:37 -0700927
928 for t in threads:
929 t.join()
930 ONOSMastership.append( t.result )
931
932 for i in range( numControllers ):
933 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
934 main.log.report( "Error in getting ONOS" + str( i + 1 ) +
935 " roles" )
936 main.log.warn(
937 "ONOS" + str( i + 1 ) + " mastership response: " +
938 repr( ONOSMastership[i] ) )
939 rolesResults = False
940 utilities.assert_equals(
941 expect=True,
942 actual=rolesResults,
943 onpass="No error in reading roles output",
944 onfail="Error in reading roles from ONOS" )
945
946 main.step( "Check for consistency in roles from each controller" )
947 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall6aec96b2015-01-19 14:49:31 -0800948 main.log.report(
949 "Switch roles are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -0500950 else:
Jon Hall65844a32015-03-09 19:09:37 -0700951 consistentMastership = False
Jon Hall6aec96b2015-01-19 14:49:31 -0800952 utilities.assert_equals(
Jon Hall65844a32015-03-09 19:09:37 -0700953 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800954 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -0800955 onpass="Switch roles are consistent across all ONOS nodes",
956 onfail="ONOS nodes have different views of switch roles" )
Jon Hallb1290e82014-11-18 16:17:48 -0500957
Jon Hall65844a32015-03-09 19:09:37 -0700958 if rolesResults and not consistentMastership:
959 for i in range( numControllers ):
Jon Hall5cfd23c2015-03-19 11:40:57 -0700960 try:
961 main.log.warn(
962 "ONOS" + str( i + 1 ) + " roles: ",
963 json.dumps(
964 json.loads( ONOSMastership[ i ] ),
965 sort_keys=True,
966 indent=4,
967 separators=( ',', ': ' ) ) )
968 except ( ValueError, TypeError ):
969 main.log.warn( repr( ONOSMastership[ i ] ) )
970 elif rolesResults and consistentMastership:
Jon Hall65844a32015-03-09 19:09:37 -0700971 mastershipCheck = main.TRUE
972 mastershipState = ONOSMastership[ 0 ]
973
Jon Hall6aec96b2015-01-19 14:49:31 -0800974 main.step( "Get the intents from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800975 global intentState
976 intentState = []
Jon Hall65844a32015-03-09 19:09:37 -0700977 ONOSIntents = []
Jon Hall8f89dda2015-01-22 16:03:33 -0800978 intentCheck = main.FALSE
Jon Hall65844a32015-03-09 19:09:37 -0700979 consistentIntents = True
980 intentsResults = True
981 threads = []
982 for i in range( numControllers ):
983 t = main.Thread( target=CLIs[i].intents,
Jon Hall65844a32015-03-09 19:09:37 -0700984 name="intents-" + str( i ),
985 args=[],
986 kwargs={ 'jsonFormat': True } )
987 threads.append( t )
988 t.start()
Jon Hall65844a32015-03-09 19:09:37 -0700989
990 for t in threads:
991 t.join()
992 ONOSIntents.append( t.result )
993
994 for i in range( numControllers ):
995 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
996 main.log.report( "Error in getting ONOS" + str( i + 1 ) +
997 " intents" )
998 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
999 repr( ONOSIntents[ i ] ) )
1000 intentsResults = False
Jon Hall6aec96b2015-01-19 14:49:31 -08001001 utilities.assert_equals(
Jon Hall65844a32015-03-09 19:09:37 -07001002 expect=True,
1003 actual=intentsResults,
1004 onpass="No error in reading intents output",
1005 onfail="Error in reading intents from ONOS" )
1006
1007 main.step( "Check for consistency in Intents from each controller" )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001008 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall65844a32015-03-09 19:09:37 -07001009 main.log.report( "Intents are consistent across all ONOS " +
1010 "nodes" )
1011 else:
1012 consistentIntents = False
Jon Hall5cfd23c2015-03-19 11:40:57 -07001013 main.log.report( "Intents not consistent" )
Jon Hall65844a32015-03-09 19:09:37 -07001014 utilities.assert_equals(
1015 expect=True,
1016 actual=consistentIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001017 onpass="Intents are consistent across all ONOS nodes",
1018 onfail="ONOS nodes have different views of intents" )
Jon Hallb1290e82014-11-18 16:17:48 -05001019
Jon Hall65844a32015-03-09 19:09:37 -07001020 if intentsResults and not consistentIntents:
Jon Hall5cfd23c2015-03-19 11:40:57 -07001021 n = len(ONOSIntents)
1022 main.log.warn( "ONOS" + str( n ) + " intents: " )
1023 main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
1024 sort_keys=True,
1025 indent=4,
1026 separators=( ',', ': ' ) ) )
Jon Hall65844a32015-03-09 19:09:37 -07001027 for i in range( numControllers ):
Jon Hall5cfd23c2015-03-19 11:40:57 -07001028 if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
1029 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1030 main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
1031 sort_keys=True,
1032 indent=4,
1033 separators=( ',', ': ' ) ) )
1034 else:
1035 main.log.warn( nodes[ i ].name + " intents match ONOS" +
1036 str( n ) + " intents" )
Jon Hall65844a32015-03-09 19:09:37 -07001037 elif intentsResults and consistentIntents:
1038 intentCheck = main.TRUE
1039 intentState = ONOSIntents[ 0 ]
1040
Jon Hall6aec96b2015-01-19 14:49:31 -08001041 main.step( "Get the flows from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001042 global flowState
1043 flowState = []
Jon Hall65844a32015-03-09 19:09:37 -07001044 ONOSFlows = []
1045 ONOSFlowsJson = []
Jon Hall8f89dda2015-01-22 16:03:33 -08001046 flowCheck = main.FALSE
Jon Hall65844a32015-03-09 19:09:37 -07001047 consistentFlows = True
1048 flowsResults = True
1049 threads = []
1050 for i in range( numControllers ):
1051 t = main.Thread( target=CLIs[i].flows,
Jon Hall65844a32015-03-09 19:09:37 -07001052 name="flows-" + str( i ),
1053 args=[],
1054 kwargs={ 'jsonFormat': True } )
1055 threads.append( t )
1056 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001057
Jon Halla9d26da2015-03-30 16:45:32 -07001058 # NOTE: Flows command can take some time to run
Jon Hall5cfd23c2015-03-19 11:40:57 -07001059 time.sleep(30)
Jon Hall65844a32015-03-09 19:09:37 -07001060 for t in threads:
1061 t.join()
Jon Hall5cfd23c2015-03-19 11:40:57 -07001062 result = t.result
Jon Hall65844a32015-03-09 19:09:37 -07001063 ONOSFlows.append( result )
1064
1065 for i in range( numControllers ):
1066 num = str( i + 1 )
1067 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
1068 main.log.report( "Error in getting ONOS" + num + " flows" )
1069 main.log.warn( "ONOS" + num + " flows response: " +
1070 repr( ONOSFlows[ i ] ) )
1071 flowsResults = False
1072 ONOSFlowsJson.append( None )
Jon Hall58c76b72015-02-23 11:09:24 -08001073 else:
Jon Hall65844a32015-03-09 19:09:37 -07001074 try:
1075 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
1076 except ( ValueError, TypeError ):
1077 # FIXME: change this to log.error?
1078 main.log.exception( "Error in parsing ONOS" + num +
1079 " response as json." )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001080 main.log.error( repr( ONOSFlows[ i ] ) )
Jon Hall65844a32015-03-09 19:09:37 -07001081 ONOSFlowsJson.append( None )
1082 flowsResults = False
Jon Hall6aec96b2015-01-19 14:49:31 -08001083 utilities.assert_equals(
Jon Hall65844a32015-03-09 19:09:37 -07001084 expect=True,
1085 actual=flowsResults,
1086 onpass="No error in reading flows output",
1087 onfail="Error in reading flows from ONOS" )
1088
1089 main.step( "Check for consistency in Flows from each controller" )
1090 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
1091 if all( tmp ):
1092 main.log.report( "Flow count is consistent across all ONOS nodes" )
1093 else:
1094 consistentFlows = False
1095 utilities.assert_equals(
1096 expect=True,
1097 actual=consistentFlows,
Jon Hall6aec96b2015-01-19 14:49:31 -08001098 onpass="The flow count is consistent across all ONOS nodes",
1099 onfail="ONOS nodes have different flow counts" )
Jon Hallb1290e82014-11-18 16:17:48 -05001100
Jon Hall65844a32015-03-09 19:09:37 -07001101 if flowsResults and not consistentFlows:
1102 for i in range( numControllers ):
Jon Hall5cfd23c2015-03-19 11:40:57 -07001103 try:
1104 main.log.warn(
1105 "ONOS" + str( i + 1 ) + " flows: " +
1106 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
1107 indent=4, separators=( ',', ': ' ) ) )
1108 except ( ValueError, TypeError ):
1109 main.log.warn(
1110 "ONOS" + str( i + 1 ) + " flows: " +
1111 repr( ONOSFlows[ i ] ) )
Jon Hall65844a32015-03-09 19:09:37 -07001112 elif flowsResults and consistentFlows:
1113 flowCheck = main.TRUE
1114 flowState = ONOSFlows[ 0 ]
1115
Jon Hall6aec96b2015-01-19 14:49:31 -08001116 main.step( "Get the OF Table entries" )
Jon Hallb1290e82014-11-18 16:17:48 -05001117 global flows
Jon Hall6aec96b2015-01-19 14:49:31 -08001118 flows = []
1119 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001120 flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001121 if flowCheck == main.FALSE:
1122 for table in flows:
1123 main.log.warn( table )
Jon Hall6aec96b2015-01-19 14:49:31 -08001124 # TODO: Compare switch flow tables with ONOS flow tables
Jon Hallb1290e82014-11-18 16:17:48 -05001125
Jon Hall6aec96b2015-01-19 14:49:31 -08001126 main.step( "Start continuous pings" )
1127 main.Mininet2.pingLong(
1128 src=main.params[ 'PING' ][ 'source1' ],
1129 target=main.params[ 'PING' ][ 'target1' ],
1130 pingTime=500 )
1131 main.Mininet2.pingLong(
1132 src=main.params[ 'PING' ][ 'source2' ],
1133 target=main.params[ 'PING' ][ 'target2' ],
1134 pingTime=500 )
1135 main.Mininet2.pingLong(
1136 src=main.params[ 'PING' ][ 'source3' ],
1137 target=main.params[ 'PING' ][ 'target3' ],
1138 pingTime=500 )
1139 main.Mininet2.pingLong(
1140 src=main.params[ 'PING' ][ 'source4' ],
1141 target=main.params[ 'PING' ][ 'target4' ],
1142 pingTime=500 )
1143 main.Mininet2.pingLong(
1144 src=main.params[ 'PING' ][ 'source5' ],
1145 target=main.params[ 'PING' ][ 'target5' ],
1146 pingTime=500 )
1147 main.Mininet2.pingLong(
1148 src=main.params[ 'PING' ][ 'source6' ],
1149 target=main.params[ 'PING' ][ 'target6' ],
1150 pingTime=500 )
1151 main.Mininet2.pingLong(
1152 src=main.params[ 'PING' ][ 'source7' ],
1153 target=main.params[ 'PING' ][ 'target7' ],
1154 pingTime=500 )
1155 main.Mininet2.pingLong(
1156 src=main.params[ 'PING' ][ 'source8' ],
1157 target=main.params[ 'PING' ][ 'target8' ],
1158 pingTime=500 )
1159 main.Mininet2.pingLong(
1160 src=main.params[ 'PING' ][ 'source9' ],
1161 target=main.params[ 'PING' ][ 'target9' ],
1162 pingTime=500 )
1163 main.Mininet2.pingLong(
1164 src=main.params[ 'PING' ][ 'source10' ],
1165 target=main.params[ 'PING' ][ 'target10' ],
1166 pingTime=500 )
Jon Hallb1290e82014-11-18 16:17:48 -05001167
Jon Hall6aec96b2015-01-19 14:49:31 -08001168 main.step( "Create TestONTopology object" )
Jon Hallb1290e82014-11-18 16:17:48 -05001169 ctrls = []
Jon Hall65844a32015-03-09 19:09:37 -07001170 for node in nodes:
1171 temp = ( node, node.name, node.ip_address, 6633 )
Jon Hall65844a32015-03-09 19:09:37 -07001172 ctrls.append( temp )
Jon Hall65844a32015-03-09 19:09:37 -07001173 MNTopo = TestONTopology( main.Mininet1, ctrls )
Jon Hallb1290e82014-11-18 16:17:48 -05001174
Jon Hall6aec96b2015-01-19 14:49:31 -08001175 main.step( "Collecting topology information from ONOS" )
Jon Hallb1290e82014-11-18 16:17:48 -05001176 devices = []
Jon Hall65844a32015-03-09 19:09:37 -07001177 threads = []
1178 for i in range( numControllers ):
1179 t = main.Thread( target=CLIs[i].devices,
Jon Hall65844a32015-03-09 19:09:37 -07001180 name="devices-" + str( i ),
1181 args=[ ] )
1182 threads.append( t )
1183 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001184
1185 for t in threads:
1186 t.join()
1187 devices.append( t.result )
Jon Hallb1290e82014-11-18 16:17:48 -05001188 hosts = []
Jon Hall65844a32015-03-09 19:09:37 -07001189 threads = []
1190 for i in range( numControllers ):
1191 t = main.Thread( target=CLIs[i].hosts,
Jon Hall65844a32015-03-09 19:09:37 -07001192 name="hosts-" + str( i ),
1193 args=[ ] )
1194 threads.append( t )
1195 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001196
1197 for t in threads:
1198 t.join()
1199 try:
1200 hosts.append( json.loads( t.result ) )
1201 except ( ValueError, TypeError ):
1202 # FIXME: better handling of this, print which node
1203 # Maybe use thread name?
1204 main.log.exception( "Error parsing json output of hosts" )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001205 # FIXME: should this be an empty json object instead?
1206 hosts.append( None )
Jon Hall65844a32015-03-09 19:09:37 -07001207
Jon Hallb1290e82014-11-18 16:17:48 -05001208 ports = []
Jon Hall65844a32015-03-09 19:09:37 -07001209 threads = []
1210 for i in range( numControllers ):
1211 t = main.Thread( target=CLIs[i].ports,
Jon Hall65844a32015-03-09 19:09:37 -07001212 name="ports-" + str( i ),
1213 args=[ ] )
1214 threads.append( t )
1215 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001216
1217 for t in threads:
1218 t.join()
1219 ports.append( t.result )
Jon Hallb1290e82014-11-18 16:17:48 -05001220 links = []
Jon Hall65844a32015-03-09 19:09:37 -07001221 threads = []
1222 for i in range( numControllers ):
1223 t = main.Thread( target=CLIs[i].links,
Jon Hall65844a32015-03-09 19:09:37 -07001224 name="links-" + str( i ),
1225 args=[ ] )
1226 threads.append( t )
1227 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001228
1229 for t in threads:
1230 t.join()
1231 links.append( t.result )
Jon Hall94fd0472014-12-08 11:52:42 -08001232 clusters = []
Jon Hall65844a32015-03-09 19:09:37 -07001233 threads = []
1234 for i in range( numControllers ):
1235 t = main.Thread( target=CLIs[i].clusters,
Jon Hall65844a32015-03-09 19:09:37 -07001236 name="clusters-" + str( i ),
1237 args=[ ] )
1238 threads.append( t )
1239 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001240
1241 for t in threads:
1242 t.join()
1243 clusters.append( t.result )
Jon Hall529a37f2015-01-28 10:02:00 -08001244 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001245
Jon Hall6aec96b2015-01-19 14:49:31 -08001246 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -08001247 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -08001248 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -08001249 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001250 if "Error" not in hosts[ controller ]:
1251 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001252 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001253 else: # hosts not consistent
1254 main.log.report( "hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001255 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001256 " is inconsistent with ONOS1" )
1257 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001258 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001259
1260 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001261 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001262 controllerStr )
1263 consistentHostsResult = main.FALSE
1264 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001265 " hosts response: " +
1266 repr( hosts[ controller ] ) )
1267 utilities.assert_equals(
1268 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001269 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001270 onpass="Hosts view is consistent across all ONOS nodes",
1271 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001272
Jon Hall58c76b72015-02-23 11:09:24 -08001273 ipResult = main.TRUE
1274 for controller in range( 0, len( hosts ) ):
1275 controllerStr = str( controller + 1 )
1276 for host in hosts[ controller ]:
Jon Hall65844a32015-03-09 19:09:37 -07001277 if not host.get( 'ips', [ ] ):
1278 main.log.error( "DEBUG:Error with host ips on controller" +
1279 controllerStr + ": " + str( host ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001280 ipResult = main.FALSE
1281 utilities.assert_equals(
1282 expect=main.TRUE,
1283 actual=ipResult,
1284 onpass="The ips of the hosts aren't empty",
1285 onfail="The ip of at least one host is missing" )
1286
Jon Hall6aec96b2015-01-19 14:49:31 -08001287 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -08001288 consistentClustersResult = main.TRUE
Jon Hall5cfd23c2015-03-19 11:40:57 -07001289 for controller in range( len( clusters ) ):
Jon Hall65844a32015-03-09 19:09:37 -07001290 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001291 if "Error" not in clusters[ controller ]:
1292 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001293 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001294 else: # clusters not consistent
Jon Hall65844a32015-03-09 19:09:37 -07001295 main.log.report( "clusters from ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001296 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001297 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001298
1299 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001300 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001301 "from ONOS" + controllerStr )
1302 consistentClustersResult = main.FALSE
1303 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001304 " clusters response: " +
1305 repr( clusters[ controller ] ) )
1306 utilities.assert_equals(
1307 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001308 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001309 onpass="Clusters view is consistent across all ONOS nodes",
1310 onfail="ONOS nodes have different views of clusters" )
1311 # there should always only be one cluster
Jon Hall5cfd23c2015-03-19 11:40:57 -07001312 try:
1313 numClusters = len( json.loads( clusters[ 0 ] ) )
1314 except ( ValueError, TypeError ):
1315 main.log.exception( "Error parsing clusters[0]: " +
1316 repr( clusters[ 0 ] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001317 clusterResults = main.FALSE
1318 if numClusters == 1:
1319 clusterResults = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001320 utilities.assert_equals(
1321 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001322 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001323 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08001324 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001325
Jon Hall6aec96b2015-01-19 14:49:31 -08001326 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001327 devicesResults = main.TRUE
1328 portsResults = main.TRUE
1329 linksResults = main.TRUE
1330 for controller in range( numControllers ):
1331 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001332 if devices[ controller ] or "Error" not in devices[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001333 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001334 MNTopo,
Jon Hall65844a32015-03-09 19:09:37 -07001335 json.loads( devices[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001336 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001337 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001338 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001339 actual=currentDevicesResult,
1340 onpass="ONOS" + controllerStr +
1341 " Switches view is correct",
1342 onfail="ONOS" + controllerStr +
1343 " Switches view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001344
Jon Hall6aec96b2015-01-19 14:49:31 -08001345 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001346 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001347 MNTopo,
Jon Hall65844a32015-03-09 19:09:37 -07001348 json.loads( ports[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001349 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001350 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001351 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001352 actual=currentPortsResult,
1353 onpass="ONOS" + controllerStr +
1354 " ports view is correct",
1355 onfail="ONOS" + controllerStr +
1356 " ports view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001357
Jon Hall6aec96b2015-01-19 14:49:31 -08001358 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001359 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001360 MNTopo,
Jon Hall65844a32015-03-09 19:09:37 -07001361 json.loads( links[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001362 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001363 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001364 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001365 actual=currentLinksResult,
1366 onpass="ONOS" + controllerStr +
1367 " links view is correct",
1368 onfail="ONOS" + controllerStr +
1369 " links view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001370
Jon Hall8f89dda2015-01-22 16:03:33 -08001371 devicesResults = devicesResults and currentDevicesResult
1372 portsResults = portsResults and currentPortsResult
1373 linksResults = linksResults and currentLinksResult
Jon Hallb1290e82014-11-18 16:17:48 -05001374
Jon Hall65844a32015-03-09 19:09:37 -07001375 topoResult = ( devicesResults and portsResults and linksResults
1376 and consistentHostsResult and consistentClustersResult
1377 and clusterResults and ipResult )
Jon Hall8f89dda2015-01-22 16:03:33 -08001378 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08001379 onpass="Topology Check Test successful",
1380 onfail="Topology Check Test NOT successful" )
Jon Hallb1290e82014-11-18 16:17:48 -05001381
Jon Hall8f89dda2015-01-22 16:03:33 -08001382 finalAssert = main.TRUE
Jon Hall65844a32015-03-09 19:09:37 -07001383 finalAssert = ( finalAssert and topoResult and flowCheck
1384 and intentCheck and consistentMastership
Jon Halla9d26da2015-03-30 16:45:32 -07001385 and mastershipCheck and rolesNotNull )
Jon Hall8f89dda2015-01-22 16:03:33 -08001386 utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
Jon Hall58c76b72015-02-23 11:09:24 -08001387 onpass="State check successful",
1388 onfail="State check NOT successful" )
Jon Hallb1290e82014-11-18 16:17:48 -05001389
Jon Hall6aec96b2015-01-19 14:49:31 -08001390 def CASE6( self, main ):
1391 """
Jon Hallb1290e82014-11-18 16:17:48 -05001392 The Failure case. Since this is the Sanity test, we do nothing.
Jon Hall6aec96b2015-01-19 14:49:31 -08001393 """
Jon Hall368769f2014-11-19 15:43:35 -08001394 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07001395 assert numControllers, "numControllers not defined"
1396 assert main, "main not defined"
1397 assert utilities.assert_equals, "utilities.assert_equals not defined"
1398 assert CLIs, "CLIs not defined"
1399 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08001400 main.log.report( "Wait 60 seconds instead of inducing a failure" )
1401 time.sleep( 60 )
1402 utilities.assert_equals(
1403 expect=main.TRUE,
1404 actual=main.TRUE,
1405 onpass="Sleeping 60 seconds",
1406 onfail="Something is terribly wrong with my math" )
Jon Hallb1290e82014-11-18 16:17:48 -05001407
Jon Hall6aec96b2015-01-19 14:49:31 -08001408 def CASE7( self, main ):
1409 """
Jon Hall368769f2014-11-19 15:43:35 -08001410 Check state after ONOS failure
Jon Hall6aec96b2015-01-19 14:49:31 -08001411 """
Jon Hallb1290e82014-11-18 16:17:48 -05001412 import json
Jon Hall5cfd23c2015-03-19 11:40:57 -07001413 assert numControllers, "numControllers not defined"
1414 assert main, "main not defined"
1415 assert utilities.assert_equals, "utilities.assert_equals not defined"
1416 assert CLIs, "CLIs not defined"
1417 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08001418 main.case( "Running ONOS Constant State Tests" )
Jon Hallb1290e82014-11-18 16:17:48 -05001419
Jon Hall65844a32015-03-09 19:09:37 -07001420 main.step( "Check that each switch has a master" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001421 # Assert that each device has a master
Jon Hall65844a32015-03-09 19:09:37 -07001422 rolesNotNull = main.TRUE
1423 threads = []
1424 for i in range( numControllers ):
1425 t = main.Thread( target=CLIs[i].rolesNotNull,
Jon Hall65844a32015-03-09 19:09:37 -07001426 name="rolesNotNull-" + str( i ),
1427 args=[ ] )
1428 threads.append( t )
1429 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001430
1431 for t in threads:
1432 t.join()
1433 rolesNotNull = rolesNotNull and t.result
Jon Hall6aec96b2015-01-19 14:49:31 -08001434 utilities.assert_equals(
1435 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001436 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -08001437 onpass="Each device has a master",
1438 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -08001439
Jon Hall65844a32015-03-09 19:09:37 -07001440 ONOSMastership = []
1441 mastershipCheck = main.FALSE
1442 consistentMastership = True
1443 rolesResults = True
1444 threads = []
1445 for i in range( numControllers ):
1446 t = main.Thread( target=CLIs[i].roles,
Jon Hall65844a32015-03-09 19:09:37 -07001447 name="roles-" + str( i ),
1448 args=[] )
1449 threads.append( t )
1450 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001451
1452 for t in threads:
1453 t.join()
1454 ONOSMastership.append( t.result )
1455
1456 for i in range( numControllers ):
1457 if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
1458 main.log.report( "Error in getting ONOS" + str( i + 1 ) +
1459 " roles" )
1460 main.log.warn(
1461 "ONOS" + str( i + 1 ) + " mastership response: " +
1462 repr( ONOSMastership[i] ) )
1463 rolesResults = False
1464 utilities.assert_equals(
1465 expect=True,
1466 actual=rolesResults,
1467 onpass="No error in reading roles output",
1468 onfail="Error in reading roles from ONOS" )
1469
1470 main.step( "Check for consistency in roles from each controller" )
1471 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001472 main.log.report(
1473 "Switch roles are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -05001474 else:
Jon Hall65844a32015-03-09 19:09:37 -07001475 consistentMastership = False
Jon Hall6aec96b2015-01-19 14:49:31 -08001476 utilities.assert_equals(
Jon Hall65844a32015-03-09 19:09:37 -07001477 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -08001478 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -08001479 onpass="Switch roles are consistent across all ONOS nodes",
1480 onfail="ONOS nodes have different views of switch roles" )
Jon Hallb1290e82014-11-18 16:17:48 -05001481
Jon Hall65844a32015-03-09 19:09:37 -07001482 if rolesResults and not consistentMastership:
1483 for i in range( numControllers ):
1484 main.log.warn(
1485 "ONOS" + str( i + 1 ) + " roles: ",
1486 json.dumps(
1487 json.loads( ONOSMastership[ i ] ),
1488 sort_keys=True,
1489 indent=4,
1490 separators=( ',', ': ' ) ) )
1491 elif rolesResults and not consistentMastership:
1492 mastershipCheck = main.TRUE
1493
Jon Hallb1290e82014-11-18 16:17:48 -05001494 description2 = "Compare switch roles from before failure"
Jon Hall6aec96b2015-01-19 14:49:31 -08001495 main.step( description2 )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001496 try:
1497 currentJson = json.loads( ONOSMastership[0] )
1498 oldJson = json.loads( mastershipState )
1499 except ( ValueError, TypeError ):
1500 main.log.exception( "Something is wrong with parsing " +
1501 "ONOSMastership[0] or mastershipState" )
1502 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
1503 main.log.error( "mastershipState" + repr( mastershipState ) )
1504 main.cleanup()
1505 main.exit()
Jon Hall8f89dda2015-01-22 16:03:33 -08001506 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001507 for i in range( 1, 29 ):
1508 switchDPID = str(
Jon Hall65844a32015-03-09 19:09:37 -07001509 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001510 current = [ switch[ 'master' ] for switch in currentJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001511 if switchDPID in switch[ 'id' ] ]
Jon Hall8f89dda2015-01-22 16:03:33 -08001512 old = [ switch[ 'master' ] for switch in oldJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001513 if switchDPID in switch[ 'id' ] ]
Jon Hallb1290e82014-11-18 16:17:48 -05001514 if current == old:
Jon Hall8f89dda2015-01-22 16:03:33 -08001515 mastershipCheck = mastershipCheck and main.TRUE
Jon Hallb1290e82014-11-18 16:17:48 -05001516 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001517 main.log.warn( "Mastership of switch %s changed" % switchDPID )
Jon Hall8f89dda2015-01-22 16:03:33 -08001518 mastershipCheck = main.FALSE
1519 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001520 main.log.report( "Mastership of Switches was not changed" )
1521 utilities.assert_equals(
1522 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001523 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001524 onpass="Mastership of Switches was not changed",
1525 onfail="Mastership of some switches changed" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001526 mastershipCheck = mastershipCheck and consistentMastership
Jon Hallb1290e82014-11-18 16:17:48 -05001527
Jon Hall6aec96b2015-01-19 14:49:31 -08001528 main.step( "Get the intents and compare across all nodes" )
Jon Hall65844a32015-03-09 19:09:37 -07001529 ONOSIntents = []
Jon Hall8f89dda2015-01-22 16:03:33 -08001530 intentCheck = main.FALSE
Jon Hall65844a32015-03-09 19:09:37 -07001531 consistentIntents = True
1532 intentsResults = True
1533 threads = []
1534 for i in range( numControllers ):
1535 t = main.Thread( target=CLIs[i].intents,
Jon Hall65844a32015-03-09 19:09:37 -07001536 name="intents-" + str( i ),
1537 args=[],
1538 kwargs={ 'jsonFormat': True } )
1539 threads.append( t )
1540 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001541
1542 for t in threads:
1543 t.join()
1544 ONOSIntents.append( t.result )
1545
1546 for i in range( numControllers ):
1547 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
1548 main.log.report( "Error in getting ONOS" + str( i + 1 ) +
1549 " intents" )
1550 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
1551 repr( ONOSIntents[ i ] ) )
1552 intentsResults = False
Jon Hall6aec96b2015-01-19 14:49:31 -08001553 utilities.assert_equals(
Jon Hall65844a32015-03-09 19:09:37 -07001554 expect=True,
1555 actual=intentsResults,
1556 onpass="No error in reading intents output",
1557 onfail="Error in reading intents from ONOS" )
1558
1559 main.step( "Check for consistency in Intents from each controller" )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001560 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
Jon Hall65844a32015-03-09 19:09:37 -07001561 main.log.report( "Intents are consistent across all ONOS " +
1562 "nodes" )
1563 else:
1564 consistentIntents = False
1565 utilities.assert_equals(
1566 expect=True,
1567 actual=consistentIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001568 onpass="Intents are consistent across all ONOS nodes",
1569 onfail="ONOS nodes have different views of intents" )
Jon Hall1b8f54a2015-02-04 13:24:20 -08001570 intentStates = []
Jon Hall65844a32015-03-09 19:09:37 -07001571 for node in ONOSIntents: # Iter through ONOS nodes
Jon Hall1b8f54a2015-02-04 13:24:20 -08001572 nodeStates = []
Jon Hall58c76b72015-02-23 11:09:24 -08001573 # Iter through intents of a node
Jon Hall5cfd23c2015-03-19 11:40:57 -07001574 try:
1575 for intent in json.loads( node ):
1576 nodeStates.append( intent[ 'state' ] )
1577 except ( ValueError, TypeError ):
1578 main.log.exception( "Error in parsing intents" )
1579 main.log.error( repr( node ) )
Jon Hall1b8f54a2015-02-04 13:24:20 -08001580 intentStates.append( nodeStates )
1581 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1582 main.log.info( dict( out ) )
1583
Jon Hall65844a32015-03-09 19:09:37 -07001584 if intentsResults and not consistentIntents:
1585 for i in range( numControllers ):
Jon Hall5cfd23c2015-03-19 11:40:57 -07001586 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
1587 main.log.warn( json.dumps(
1588 json.loads( ONOSIntents[ i ] ),
1589 sort_keys=True,
1590 indent=4,
1591 separators=( ',', ': ' ) ) )
Jon Hall65844a32015-03-09 19:09:37 -07001592 elif intentsResults and consistentIntents:
1593 intentCheck = main.TRUE
Jon Hall65844a32015-03-09 19:09:37 -07001594
Jon Hall58c76b72015-02-23 11:09:24 -08001595 # NOTE: Store has no durability, so intents are lost across system
1596 # restarts
Jon Hall6aec96b2015-01-19 14:49:31 -08001597 main.step( "Compare current intents with intents before the failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001598 # NOTE: this requires case 5 to pass for intentState to be set.
Jon Hall94fd0472014-12-08 11:52:42 -08001599 # maybe we should stop the test if that fails?
Jon Hall1b8f54a2015-02-04 13:24:20 -08001600 sameIntents = main.TRUE
Jon Hall65844a32015-03-09 19:09:37 -07001601 if intentState and intentState == ONOSIntents[ 0 ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001602 sameIntents = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001603 main.log.report( "Intents are consistent with before failure" )
1604 # TODO: possibly the states have changed? we may need to figure out
Jon Hall65844a32015-03-09 19:09:37 -07001605 # what the acceptable states are
Jon Hallb1290e82014-11-18 16:17:48 -05001606 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001607 try:
Jon Hall65844a32015-03-09 19:09:37 -07001608 main.log.warn( "ONOS intents: " )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001609 main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1610 sort_keys=True, indent=4,
1611 separators=( ',', ': ' ) ) )
1612 except ( ValueError, TypeError ):
Jon Hall65844a32015-03-09 19:09:37 -07001613 main.log.exception( "Exception printing intents" )
Jon Hall5cfd23c2015-03-19 11:40:57 -07001614 main.log.warn( repr( ONOSIntents[0] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001615 sameIntents = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001616 utilities.assert_equals(
1617 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001618 actual=sameIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001619 onpass="Intents are consistent with before failure",
1620 onfail="The Intents changed during failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001621 intentCheck = intentCheck and sameIntents
Jon Hallb1290e82014-11-18 16:17:48 -05001622
Jon Hall6aec96b2015-01-19 14:49:31 -08001623 main.step( "Get the OF Table entries and compare to before " +
1624 "component failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001625 FlowTables = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001626 flows2 = []
1627 for i in range( 28 ):
1628 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001629 tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
1630 flows2.append( tmpFlows )
1631 tempResult = main.Mininet2.flowComp(
Jon Hall6aec96b2015-01-19 14:49:31 -08001632 flow1=flows[ i ],
Jon Hall8f89dda2015-01-22 16:03:33 -08001633 flow2=tmpFlows )
1634 FlowTables = FlowTables and tempResult
1635 if FlowTables == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001636 main.log.info( "Differences in flow table for switch: s" +
1637 str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001638 if FlowTables == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001639 main.log.report( "No changes were found in the flow tables" )
1640 utilities.assert_equals(
1641 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001642 actual=FlowTables,
Jon Hall6aec96b2015-01-19 14:49:31 -08001643 onpass="No changes were found in the flow tables",
1644 onfail="Changes were found in the flow tables" )
Jon Hallb1290e82014-11-18 16:17:48 -05001645
Jon Hall6aec96b2015-01-19 14:49:31 -08001646 main.step( "Check the continuous pings to ensure that no packets " +
1647 "were dropped during component failure" )
Jon Hall65844a32015-03-09 19:09:37 -07001648 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
1649 main.params[ 'TESTONIP' ] )
Jon Hall8f89dda2015-01-22 16:03:33 -08001650 LossInPings = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001651 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1652 for i in range( 8, 18 ):
1653 main.log.info(
1654 "Checking for a loss in pings along flow from s" +
1655 str( i ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001656 LossInPings = main.Mininet2.checkForLoss(
Jon Hall6aec96b2015-01-19 14:49:31 -08001657 "/tmp/ping.h" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001658 str( i ) ) or LossInPings
1659 if LossInPings == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001660 main.log.info( "Loss in ping detected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001661 elif LossInPings == main.ERROR:
Jon Hall6aec96b2015-01-19 14:49:31 -08001662 main.log.info( "There are multiple mininet process running" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001663 elif LossInPings == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001664 main.log.info( "No Loss in the pings" )
1665 main.log.report( "No loss of dataplane connectivity" )
1666 utilities.assert_equals(
1667 expect=main.FALSE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001668 actual=LossInPings,
Jon Hall6aec96b2015-01-19 14:49:31 -08001669 onpass="No Loss of connectivity",
1670 onfail="Loss of dataplane connectivity detected" )
Jon Hallb1290e82014-11-18 16:17:48 -05001671
Jon Hall6aec96b2015-01-19 14:49:31 -08001672 # Test of LeadershipElection
1673 # NOTE: this only works for the sanity test. In case of failures,
Jon Hall58c76b72015-02-23 11:09:24 -08001674 # leader will likely change
Jon Hall65844a32015-03-09 19:09:37 -07001675 leader = nodes[ 0 ].ip_address
Jon Hall8f89dda2015-01-22 16:03:33 -08001676 leaderResult = main.TRUE
Jon Hall5cfd23c2015-03-19 11:40:57 -07001677 for cli in CLIs:
1678 leaderN = cli.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001679 # verify leader is ONOS1
Jon Hall669173b2014-12-17 11:36:30 -08001680 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08001681 # all is well
1682 # NOTE: In failure scenario, this could be a new node, maybe
1683 # check != ONOS1
Jon Hall669173b2014-12-17 11:36:30 -08001684 pass
1685 elif leaderN == main.FALSE:
Jon Hall65844a32015-03-09 19:09:37 -07001686 # error in response
Jon Hall6aec96b2015-01-19 14:49:31 -08001687 main.log.report( "Something is wrong with " +
Jon Hall58c76b72015-02-23 11:09:24 -08001688 "electionTestLeader function, check the" +
1689 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001690 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001691 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08001692 leaderResult = main.FALSE
Jon Hall5cfd23c2015-03-19 11:40:57 -07001693 main.log.report( cli.name + " sees " + str( leaderN ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001694 " as the leader of the election app. " +
1695 "Leader should be " + str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001696 if leaderResult:
1697 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001698 "view of leader across listeners and a new " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001699 "leader was re-elected if applicable )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001700 utilities.assert_equals(
1701 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001702 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001703 onpass="Leadership election passed",
1704 onfail="Something went wrong with Leadership election" )
Jon Hallb1290e82014-11-18 16:17:48 -05001705
Jon Hall58c76b72015-02-23 11:09:24 -08001706 result = ( mastershipCheck and intentCheck and FlowTables and
1707 ( not LossInPings ) and rolesNotNull and leaderResult )
Jon Hall6aec96b2015-01-19 14:49:31 -08001708 result = int( result )
Jon Hallb1290e82014-11-18 16:17:48 -05001709 if result == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001710 main.log.report( "Constant State Tests Passed" )
1711 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall58c76b72015-02-23 11:09:24 -08001712 onpass="Constant State Tests Passed",
1713 onfail="Constant state tests failed" )
Jon Hallb1290e82014-11-18 16:17:48 -05001714
Jon Hall6aec96b2015-01-19 14:49:31 -08001715 def CASE8( self, main ):
1716 """
Jon Hallb1290e82014-11-18 16:17:48 -05001717 Compare topo
Jon Hall6aec96b2015-01-19 14:49:31 -08001718 """
Jon Hallb1290e82014-11-18 16:17:48 -05001719 import sys
Jon Hall6aec96b2015-01-19 14:49:31 -08001720 # FIXME add this path to params
1721 sys.path.append( "/home/admin/sts" )
1722 # assumes that sts is already in you PYTHONPATH
1723 from sts.topology.teston_topology import TestONTopology
Jon Hallb1290e82014-11-18 16:17:48 -05001724 import json
Jon Hall73cf9cc2014-11-20 22:28:38 -08001725 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07001726 assert numControllers, "numControllers not defined"
1727 assert main, "main not defined"
1728 assert utilities.assert_equals, "utilities.assert_equals not defined"
1729 assert CLIs, "CLIs not defined"
1730 assert nodes, "nodes not defined"
Jon Hallb1290e82014-11-18 16:17:48 -05001731
Jon Hall6aec96b2015-01-19 14:49:31 -08001732 description = "Compare ONOS Topology view to Mininet topology"
1733 main.case( description )
1734 main.log.report( description )
1735 main.step( "Create TestONTopology object" )
Jon Hallb1290e82014-11-18 16:17:48 -05001736 ctrls = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07001737 for node in nodes:
1738 temp = ( node, node.name, node.ip_address, 6633 )
1739 ctrls.append( temp )
Jon Hall65844a32015-03-09 19:09:37 -07001740 MNTopo = TestONTopology( main.Mininet1, ctrls )
Jon Hallb1290e82014-11-18 16:17:48 -05001741
Jon Hall6aec96b2015-01-19 14:49:31 -08001742 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001743 devicesResults = main.TRUE
1744 portsResults = main.TRUE
1745 linksResults = main.TRUE
Jon Hall58c76b72015-02-23 11:09:24 -08001746 hostsResults = main.TRUE
Jon Hall8f89dda2015-01-22 16:03:33 -08001747 topoResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001748 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -08001749 count = 0
Jon Hall6aec96b2015-01-19 14:49:31 -08001750 main.step( "Collecting topology information from ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001751 startTime = time.time()
Jon Hall21270ac2015-02-16 17:59:55 -08001752 # Give time for Gossip to work
Jon Hall8f89dda2015-01-22 16:03:33 -08001753 while topoResult == main.FALSE and elapsed < 60:
Jon Hall65844a32015-03-09 19:09:37 -07001754 count += 1
Jon Hall94fd0472014-12-08 11:52:42 -08001755 if count > 1:
Jon Hall65844a32015-03-09 19:09:37 -07001756 # TODO: Deprecate STS usage
Jon Hall58c76b72015-02-23 11:09:24 -08001757 MNTopo = TestONTopology( main.Mininet1, ctrls )
Jon Hall8f89dda2015-01-22 16:03:33 -08001758 cliStart = time.time()
Jon Hall94fd0472014-12-08 11:52:42 -08001759 devices = []
Jon Hall65844a32015-03-09 19:09:37 -07001760 threads = []
1761 for i in range( numControllers ):
1762 t = main.Thread( target=CLIs[i].devices,
Jon Hall65844a32015-03-09 19:09:37 -07001763 name="devices-" + str( i ),
1764 args=[ ] )
1765 threads.append( t )
1766 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001767
1768 for t in threads:
1769 t.join()
1770 devices.append( t.result )
Jon Hall94fd0472014-12-08 11:52:42 -08001771 hosts = []
Jon Hall58c76b72015-02-23 11:09:24 -08001772 ipResult = main.TRUE
Jon Hall65844a32015-03-09 19:09:37 -07001773 threads = []
1774 for i in range( numControllers ):
1775 t = main.Thread( target=CLIs[i].hosts,
Jon Hall65844a32015-03-09 19:09:37 -07001776 name="hosts-" + str( i ),
1777 args=[ ] )
1778 threads.append( t )
1779 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001780
1781 for t in threads:
1782 t.join()
Jon Hall5cfd23c2015-03-19 11:40:57 -07001783 try:
1784 hosts.append( json.loads( t.result ) )
1785 except ( ValueError, TypeError ):
1786 main.log.exception( "Error parsing hosts results" )
1787 main.log.error( repr( t.result ) )
Jon Hall529a37f2015-01-28 10:02:00 -08001788 for controller in range( 0, len( hosts ) ):
1789 controllerStr = str( controller + 1 )
1790 for host in hosts[ controller ]:
Jon Hall58c76b72015-02-23 11:09:24 -08001791 if host is None or host.get( 'ips', [] ) == []:
Jon Hall529a37f2015-01-28 10:02:00 -08001792 main.log.error(
1793 "DEBUG:Error with host ips on controller" +
1794 controllerStr + ": " + str( host ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001795 ipResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001796 ports = []
Jon Hall65844a32015-03-09 19:09:37 -07001797 threads = []
1798 for i in range( numControllers ):
1799 t = main.Thread( target=CLIs[i].ports,
Jon Hall65844a32015-03-09 19:09:37 -07001800 name="ports-" + str( i ),
1801 args=[ ] )
1802 threads.append( t )
1803 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001804
1805 for t in threads:
1806 t.join()
1807 ports.append( t.result )
Jon Hall94fd0472014-12-08 11:52:42 -08001808 links = []
Jon Hall65844a32015-03-09 19:09:37 -07001809 threads = []
1810 for i in range( numControllers ):
1811 t = main.Thread( target=CLIs[i].links,
Jon Hall65844a32015-03-09 19:09:37 -07001812 name="links-" + str( i ),
1813 args=[ ] )
1814 threads.append( t )
1815 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001816
1817 for t in threads:
1818 t.join()
1819 links.append( t.result )
Jon Hall94fd0472014-12-08 11:52:42 -08001820 clusters = []
Jon Hall65844a32015-03-09 19:09:37 -07001821 threads = []
1822 for i in range( numControllers ):
1823 t = main.Thread( target=CLIs[i].clusters,
Jon Hall65844a32015-03-09 19:09:37 -07001824 name="clusters-" + str( i ),
1825 args=[ ] )
1826 threads.append( t )
1827 t.start()
Jon Hall65844a32015-03-09 19:09:37 -07001828
1829 for t in threads:
1830 t.join()
1831 clusters.append( t.result )
Jon Hallb1290e82014-11-18 16:17:48 -05001832
Jon Hall8f89dda2015-01-22 16:03:33 -08001833 elapsed = time.time() - startTime
1834 cliTime = time.time() - cliStart
1835 print "CLI time: " + str( cliTime )
Jon Hallb1290e82014-11-18 16:17:48 -05001836
Jon Hall21270ac2015-02-16 17:59:55 -08001837 for controller in range( numControllers ):
1838 controllerStr = str( controller + 1 )
1839 if devices[ controller ] or "Error" not in devices[
1840 controller ]:
1841 currentDevicesResult = main.Mininet1.compareSwitches(
1842 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001843 json.loads( devices[ controller ] ) )
Jon Hall21270ac2015-02-16 17:59:55 -08001844 else:
1845 currentDevicesResult = main.FALSE
1846 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001847 actual=currentDevicesResult,
1848 onpass="ONOS" + controllerStr +
1849 " Switches view is correct",
1850 onfail="ONOS" + controllerStr +
1851 " Switches view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001852
Jon Hall21270ac2015-02-16 17:59:55 -08001853 if ports[ controller ] or "Error" not in ports[ controller ]:
1854 currentPortsResult = main.Mininet1.comparePorts(
1855 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001856 json.loads( ports[ controller ] ) )
Jon Hall21270ac2015-02-16 17:59:55 -08001857 else:
1858 currentPortsResult = main.FALSE
1859 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001860 actual=currentPortsResult,
1861 onpass="ONOS" + controllerStr +
1862 " ports view is correct",
1863 onfail="ONOS" + controllerStr +
1864 " ports view is incorrect" )
Jon Hall94fd0472014-12-08 11:52:42 -08001865
Jon Hall21270ac2015-02-16 17:59:55 -08001866 if links[ controller ] or "Error" not in links[ controller ]:
1867 currentLinksResult = main.Mininet1.compareLinks(
1868 MNTopo,
Jon Hall58c76b72015-02-23 11:09:24 -08001869 json.loads( links[ controller ] ) )
Jon Hall21270ac2015-02-16 17:59:55 -08001870 else:
1871 currentLinksResult = main.FALSE
1872 utilities.assert_equals( expect=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08001873 actual=currentLinksResult,
1874 onpass="ONOS" + controllerStr +
1875 " links view is correct",
1876 onfail="ONOS" + controllerStr +
1877 " links view is incorrect" )
1878
1879 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1880 currentHostsResult = main.Mininet1.compareHosts(
1881 MNTopo, hosts[ controller ] )
1882 else:
1883 currentHostsResult = main.FALSE
1884 utilities.assert_equals( expect=main.TRUE,
1885 actual=currentHostsResult,
1886 onpass="ONOS" + controllerStr +
1887 " hosts exist in Mininet",
1888 onfail="ONOS" + controllerStr +
1889 " hosts don't match Mininet" )
1890
1891 devicesResults = devicesResults and currentDevicesResult
1892 portsResults = portsResults and currentPortsResult
1893 linksResults = linksResults and currentLinksResult
1894 hostsResults = hostsResults and currentHostsResult
Jon Hall94fd0472014-12-08 11:52:42 -08001895
Jon Hall21270ac2015-02-16 17:59:55 -08001896 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001897
Jon Hall21270ac2015-02-16 17:59:55 -08001898 # hosts
1899 consistentHostsResult = main.TRUE
1900 for controller in range( len( hosts ) ):
1901 controllerStr = str( controller + 1 )
1902 if "Error" not in hosts[ controller ]:
1903 if hosts[ controller ] == hosts[ 0 ]:
1904 continue
1905 else: # hosts not consistent
1906 main.log.report( "hosts from ONOS" + controllerStr +
1907 " is inconsistent with ONOS1" )
1908 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001909 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001910
Jon Hall21270ac2015-02-16 17:59:55 -08001911 else:
1912 main.log.report( "Error in getting ONOS hosts from ONOS" +
1913 controllerStr )
1914 consistentHostsResult = main.FALSE
1915 main.log.warn( "ONOS" + controllerStr +
1916 " hosts response: " +
1917 repr( hosts[ controller ] ) )
1918 utilities.assert_equals(
1919 expect=main.TRUE,
1920 actual=consistentHostsResult,
1921 onpass="Hosts view is consistent across all ONOS nodes",
1922 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001923
Jon Hall21270ac2015-02-16 17:59:55 -08001924 # Strongly connected clusters of devices
1925 consistentClustersResult = main.TRUE
1926 for controller in range( len( clusters ) ):
1927 controllerStr = str( controller + 1 )
1928 if "Error" not in clusters[ controller ]:
1929 if clusters[ controller ] == clusters[ 0 ]:
1930 continue
1931 else: # clusters not consistent
1932 main.log.report( "clusters from ONOS" +
1933 controllerStr +
1934 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001935 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001936
Jon Hall21270ac2015-02-16 17:59:55 -08001937 else:
1938 main.log.report( "Error in getting dataplane clusters " +
1939 "from ONOS" + controllerStr )
1940 consistentClustersResult = main.FALSE
1941 main.log.warn( "ONOS" + controllerStr +
1942 " clusters response: " +
1943 repr( clusters[ controller ] ) )
1944 utilities.assert_equals(
1945 expect=main.TRUE,
1946 actual=consistentClustersResult,
1947 onpass="Clusters view is consistent across all ONOS nodes",
1948 onfail="ONOS nodes have different views of clusters" )
1949 # there should always only be one cluster
Jon Hall5cfd23c2015-03-19 11:40:57 -07001950 try:
1951 numClusters = len( json.loads( clusters[ 0 ] ) )
1952 except ( ValueError, TypeError ):
1953 main.log.exception( "Error parsing clusters[0]: " +
1954 repr( clusters[0] ) )
Jon Hall58c76b72015-02-23 11:09:24 -08001955 clusterResults = main.FALSE
1956 if numClusters == 1:
1957 clusterResults = main.TRUE
Jon Hall21270ac2015-02-16 17:59:55 -08001958 utilities.assert_equals(
1959 expect=1,
1960 actual=numClusters,
1961 onpass="ONOS shows 1 SCC",
Jon Hall58c76b72015-02-23 11:09:24 -08001962 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001963
Jon Hall21270ac2015-02-16 17:59:55 -08001964 topoResult = ( devicesResults and portsResults and linksResults
Jon Hall58c76b72015-02-23 11:09:24 -08001965 and hostsResults and consistentHostsResult
1966 and consistentClustersResult and clusterResults
1967 and ipResult )
Jon Hall94fd0472014-12-08 11:52:42 -08001968
Jon Hall21270ac2015-02-16 17:59:55 -08001969 topoResult = topoResult and int( count <= 2 )
1970 note = "note it takes about " + str( int( cliTime ) ) + \
1971 " seconds for the test to make all the cli calls to fetch " +\
1972 "the topology from each ONOS instance"
1973 main.log.info(
1974 "Very crass estimate for topology discovery/convergence( " +
1975 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1976 str( count ) + " tries" )
1977 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
Jon Hall58c76b72015-02-23 11:09:24 -08001978 onpass="Topology Check Test successful",
1979 onfail="Topology Check Test NOT successful" )
Jon Hall21270ac2015-02-16 17:59:55 -08001980 if topoResult == main.TRUE:
1981 main.log.report( "ONOS topology view matches Mininet topology" )
Jon Hallb1290e82014-11-18 16:17:48 -05001982
Jon Halla9d26da2015-03-30 16:45:32 -07001983 # FIXME: move this to an ONOS state case
Jon Hall5cfd23c2015-03-19 11:40:57 -07001984 main.step( "Checking ONOS nodes" )
1985 nodesOutput = []
1986 threads = []
1987 for i in range( numControllers ):
1988 t = main.Thread( target=CLIs[i].nodes,
1989 name="nodes-" + str( i ),
1990 args=[ ] )
1991 threads.append( t )
1992 t.start()
1993
1994 for t in threads:
1995 t.join()
1996 nodesOutput.append( t.result )
1997 ips = [ node.ip_address for node in nodes ]
1998 for i in nodesOutput:
1999 try:
2000 current = json.loads( i )
2001 for node in current:
2002 if node['ip'] in ips: # node in nodes() output is in cell
2003 if node['state'] == 'ACTIVE':
Jon Halla9d26da2015-03-30 16:45:32 -07002004 pass # as it should be
Jon Hall5cfd23c2015-03-19 11:40:57 -07002005 else:
2006 main.log.error( "Error in ONOS node availability" )
2007 main.log.error(
2008 json.dumps( current,
2009 sort_keys=True,
2010 indent=4,
2011 separators=( ',', ': ' ) ) )
2012 break
2013 except ( ValueError, TypeError ):
2014 main.log.error( "Error parsing nodes output" )
2015 main.log.warn( repr( i ) )
2016
Jon Hall6aec96b2015-01-19 14:49:31 -08002017 def CASE9( self, main ):
2018 """
Jon Hallb1290e82014-11-18 16:17:48 -05002019 Link s3-s28 down
Jon Hall6aec96b2015-01-19 14:49:31 -08002020 """
2021 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002022 assert numControllers, "numControllers not defined"
2023 assert main, "main not defined"
2024 assert utilities.assert_equals, "utilities.assert_equals not defined"
2025 assert CLIs, "CLIs not defined"
2026 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08002027 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05002028
Jon Hall8f89dda2015-01-22 16:03:33 -08002029 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hallb1290e82014-11-18 16:17:48 -05002030
Jon Hall6aec96b2015-01-19 14:49:31 -08002031 description = "Turn off a link to ensure that Link Discovery " +\
Jon Hall58c76b72015-02-23 11:09:24 -08002032 "is working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002033 main.log.report( description )
2034 main.case( description )
Jon Hallb1290e82014-11-18 16:17:48 -05002035
Jon Hall6aec96b2015-01-19 14:49:31 -08002036 main.step( "Kill Link between s3 and s28" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002037 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
Jon Hall58c76b72015-02-23 11:09:24 -08002038 main.log.info( "Waiting " + str( linkSleep ) +
2039 " seconds for link down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002040 time.sleep( linkSleep )
2041 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
Jon Hall65844a32015-03-09 19:09:37 -07002042 onpass="Link down successful",
Jon Hall58c76b72015-02-23 11:09:24 -08002043 onfail="Failed to bring link down" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002044 # TODO do some sort of check here
Jon Hallb1290e82014-11-18 16:17:48 -05002045
Jon Hall6aec96b2015-01-19 14:49:31 -08002046 def CASE10( self, main ):
2047 """
Jon Hallb1290e82014-11-18 16:17:48 -05002048 Link s3-s28 up
Jon Hall6aec96b2015-01-19 14:49:31 -08002049 """
2050 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002051 assert numControllers, "numControllers not defined"
2052 assert main, "main not defined"
2053 assert utilities.assert_equals, "utilities.assert_equals not defined"
2054 assert CLIs, "CLIs not defined"
2055 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08002056 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05002057
Jon Hall8f89dda2015-01-22 16:03:33 -08002058 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hallb1290e82014-11-18 16:17:48 -05002059
Jon Hall6aec96b2015-01-19 14:49:31 -08002060 description = "Restore a link to ensure that Link Discovery is " + \
Jon Hall63604932015-02-26 17:09:50 -08002061 "working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002062 main.log.report( description )
2063 main.case( description )
Jon Hallb1290e82014-11-18 16:17:48 -05002064
Jon Hall6aec96b2015-01-19 14:49:31 -08002065 main.step( "Bring link between s3 and s28 back up" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002066 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
Jon Hall58c76b72015-02-23 11:09:24 -08002067 main.log.info( "Waiting " + str( linkSleep ) +
2068 " seconds for link up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002069 time.sleep( linkSleep )
2070 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
Jon Hall65844a32015-03-09 19:09:37 -07002071 onpass="Link up successful",
Jon Hall58c76b72015-02-23 11:09:24 -08002072 onfail="Failed to bring link up" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002073 # TODO do some sort of check here
Jon Hallb1290e82014-11-18 16:17:48 -05002074
Jon Hall6aec96b2015-01-19 14:49:31 -08002075 def CASE11( self, main ):
2076 """
Jon Hallb1290e82014-11-18 16:17:48 -05002077 Switch Down
Jon Hall6aec96b2015-01-19 14:49:31 -08002078 """
2079 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05002080 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002081 assert numControllers, "numControllers not defined"
2082 assert main, "main not defined"
2083 assert utilities.assert_equals, "utilities.assert_equals not defined"
2084 assert CLIs, "CLIs not defined"
2085 assert nodes, "nodes not defined"
Jon Hallb1290e82014-11-18 16:17:48 -05002086
Jon Hall8f89dda2015-01-22 16:03:33 -08002087 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hallb1290e82014-11-18 16:17:48 -05002088
2089 description = "Killing a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002090 main.log.report( description )
2091 main.case( description )
2092 switch = main.params[ 'kill' ][ 'switch' ]
2093 switchDPID = main.params[ 'kill' ][ 'dpid' ]
Jon Hallb1290e82014-11-18 16:17:48 -05002094
Jon Hall6aec96b2015-01-19 14:49:31 -08002095 # TODO: Make this switch parameterizable
2096 main.step( "Kill " + switch )
2097 main.log.report( "Deleting " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08002098 main.Mininet1.delSwitch( switch )
2099 main.log.info( "Waiting " + str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002100 " seconds for switch down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002101 time.sleep( switchSleep )
2102 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002103 # Peek at the deleted switch
2104 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08002105 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002106 if device and device[ 'available' ] is False:
Jon Hall94fd0472014-12-08 11:52:42 -08002107 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002108 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall65844a32015-03-09 19:09:37 -07002109 onpass="Kill switch successful",
Jon Hall58c76b72015-02-23 11:09:24 -08002110 onfail="Failed to kill switch?" )
Jon Hallb1290e82014-11-18 16:17:48 -05002111
Jon Hall6aec96b2015-01-19 14:49:31 -08002112 def CASE12( self, main ):
2113 """
Jon Hallb1290e82014-11-18 16:17:48 -05002114 Switch Up
Jon Hall6aec96b2015-01-19 14:49:31 -08002115 """
2116 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05002117 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002118 assert numControllers, "numControllers not defined"
2119 assert main, "main not defined"
2120 assert utilities.assert_equals, "utilities.assert_equals not defined"
2121 assert CLIs, "CLIs not defined"
2122 assert nodes, "nodes not defined"
2123 assert ONOS1Port, "ONOS1Port not defined"
2124 assert ONOS2Port, "ONOS2Port not defined"
2125 assert ONOS3Port, "ONOS3Port not defined"
2126 assert ONOS4Port, "ONOS4Port not defined"
2127 assert ONOS5Port, "ONOS5Port not defined"
2128 assert ONOS6Port, "ONOS6Port not defined"
2129 assert ONOS7Port, "ONOS7Port not defined"
Jon Hall669173b2014-12-17 11:36:30 -08002130
Jon Hall8f89dda2015-01-22 16:03:33 -08002131 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall6aec96b2015-01-19 14:49:31 -08002132 switch = main.params[ 'kill' ][ 'switch' ]
2133 switchDPID = main.params[ 'kill' ][ 'dpid' ]
2134 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallb1290e82014-11-18 16:17:48 -05002135 description = "Adding a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08002136 main.log.report( description )
2137 main.case( description )
Jon Hallb1290e82014-11-18 16:17:48 -05002138
Jon Hall6aec96b2015-01-19 14:49:31 -08002139 main.step( "Add back " + switch )
2140 main.log.report( "Adding back " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08002141 main.Mininet1.addSwitch( switch, dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002142 for peer in links:
Jon Hall8f89dda2015-01-22 16:03:33 -08002143 main.Mininet1.addLink( switch, peer )
Jon Hall58c76b72015-02-23 11:09:24 -08002144 main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
2145 count=numControllers,
Jon Hall65844a32015-03-09 19:09:37 -07002146 ip1=nodes[ 0 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002147 port1=ONOS1Port,
Jon Hall65844a32015-03-09 19:09:37 -07002148 ip2=nodes[ 1 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002149 port2=ONOS2Port,
Jon Hall65844a32015-03-09 19:09:37 -07002150 ip3=nodes[ 2 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002151 port3=ONOS3Port,
Jon Hall65844a32015-03-09 19:09:37 -07002152 ip4=nodes[ 3 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002153 port4=ONOS4Port,
Jon Hall65844a32015-03-09 19:09:37 -07002154 ip5=nodes[ 4 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002155 port5=ONOS5Port,
Jon Hall65844a32015-03-09 19:09:37 -07002156 ip6=nodes[ 5 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002157 port6=ONOS6Port,
Jon Hall65844a32015-03-09 19:09:37 -07002158 ip7=nodes[ 6 ].ip_address,
Jon Hall58c76b72015-02-23 11:09:24 -08002159 port7=ONOS7Port )
2160 main.log.info( "Waiting " + str( switchSleep ) +
2161 " seconds for switch up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002162 time.sleep( switchSleep )
2163 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08002164 # Peek at the deleted switch
2165 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08002166 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002167 if device and device[ 'available' ]:
Jon Hall94fd0472014-12-08 11:52:42 -08002168 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002169 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall65844a32015-03-09 19:09:37 -07002170 onpass="add switch successful",
Jon Hall58c76b72015-02-23 11:09:24 -08002171 onfail="Failed to add switch?" )
Jon Hallb1290e82014-11-18 16:17:48 -05002172
Jon Hall6aec96b2015-01-19 14:49:31 -08002173 def CASE13( self, main ):
2174 """
Jon Hallb1290e82014-11-18 16:17:48 -05002175 Clean up
Jon Hall6aec96b2015-01-19 14:49:31 -08002176 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08002177 import os
2178 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002179 assert numControllers, "numControllers not defined"
2180 assert main, "main not defined"
2181 assert utilities.assert_equals, "utilities.assert_equals not defined"
2182 assert CLIs, "CLIs not defined"
2183 assert nodes, "nodes not defined"
Jon Hall6aec96b2015-01-19 14:49:31 -08002184
2185 # printing colors to terminal
Jon Hall65844a32015-03-09 19:09:37 -07002186 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
2187 'blue': '\033[94m', 'green': '\033[92m',
2188 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
Jon Hall368769f2014-11-19 15:43:35 -08002189 description = "Test Cleanup"
Jon Hall6aec96b2015-01-19 14:49:31 -08002190 main.log.report( description )
2191 main.case( description )
2192 main.step( "Killing tcpdumps" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002193 main.Mininet2.stopTcpdump()
Jon Hallb1290e82014-11-18 16:17:48 -05002194
Jon Hall6aec96b2015-01-19 14:49:31 -08002195 main.step( "Copying MN pcap and ONOS log files to test station" )
Jon Hallb1290e82014-11-18 16:17:48 -05002196 testname = main.TEST
Jon Hall8f89dda2015-01-22 16:03:33 -08002197 teststationUser = main.params[ 'TESTONUSER' ]
2198 teststationIP = main.params[ 'TESTONIP' ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002199 # NOTE: MN Pcap file is being saved to ~/packet_captures
Jon Hall73cf9cc2014-11-20 22:28:38 -08002200 # scp this file as MN and TestON aren't necessarily the same vm
Jon Hall6aec96b2015-01-19 14:49:31 -08002201 # FIXME: scp
2202 # mn files
2203 # TODO: Load these from params
2204 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002205 logFolder = "/opt/onos/log/"
2206 logFiles = [ "karaf.log", "karaf.log.1" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002207 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002208 dstDir = "~/packet_captures/"
2209 for f in logFiles:
Jon Hall65844a32015-03-09 19:09:37 -07002210 for node in nodes:
2211 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2212 ":" + logFolder + f + " " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002213 teststationUser + "@" +
2214 teststationIP + ":" +
2215 dstDir + str( testname ) +
Jon Hall65844a32015-03-09 19:09:37 -07002216 "-" + node.name + "-" + f )
Jon Hall58c76b72015-02-23 11:09:24 -08002217 main.ONOSbench.handle.expect( "\$" )
2218
Jon Hall6aec96b2015-01-19 14:49:31 -08002219 # std*.log's
2220 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002221 logFolder = "/opt/onos/var/"
2222 logFiles = [ "stderr.log", "stdout.log" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08002223 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08002224 dstDir = "~/packet_captures/"
2225 for f in logFiles:
Jon Hall65844a32015-03-09 19:09:37 -07002226 for node in nodes:
2227 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
2228 ":" + logFolder + f + " " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002229 teststationUser + "@" +
2230 teststationIP + ":" +
2231 dstDir + str( testname ) +
Jon Hall65844a32015-03-09 19:09:37 -07002232 "-" + node.name + "-" + f )
Jon Hall58c76b72015-02-23 11:09:24 -08002233 main.ONOSbench.handle.expect( "\$" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002234 # sleep so scp can finish
2235 time.sleep( 10 )
Jon Hall65844a32015-03-09 19:09:37 -07002236
2237 main.step( "Stopping Mininet" )
Jon Hall58c76b72015-02-23 11:09:24 -08002238 main.Mininet1.stopNet()
Jon Hall65844a32015-03-09 19:09:37 -07002239
2240 main.step( "Checking ONOS Logs for errors" )
2241 for node in nodes:
2242 print colors[ 'purple' ] + "Checking logs for errors on " + \
2243 node.name + ":" + colors[ 'end' ]
2244 print main.ONOSbench.checkLogs( node.ip_address )
2245
Jon Hall6aec96b2015-01-19 14:49:31 -08002246 main.step( "Packing and rotating pcap archives" )
2247 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002248
Jon Hall6aec96b2015-01-19 14:49:31 -08002249 # TODO: actually check something here
2250 utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
Jon Hall58c76b72015-02-23 11:09:24 -08002251 onpass="Test cleanup successful",
2252 onfail="Test cleanup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08002253
Jon Hall6aec96b2015-01-19 14:49:31 -08002254 def CASE14( self, main ):
2255 """
Jon Hall94fd0472014-12-08 11:52:42 -08002256 start election app on all onos nodes
Jon Hall6aec96b2015-01-19 14:49:31 -08002257 """
Jon Halla9d26da2015-03-30 16:45:32 -07002258 import time
Jon Hall5cfd23c2015-03-19 11:40:57 -07002259 assert numControllers, "numControllers not defined"
2260 assert main, "main not defined"
2261 assert utilities.assert_equals, "utilities.assert_equals not defined"
2262 assert CLIs, "CLIs not defined"
2263 assert nodes, "nodes not defined"
2264
Jon Hall8f89dda2015-01-22 16:03:33 -08002265 leaderResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08002266 main.log.info( "Install leadership election app" )
Jon Halla9d26da2015-03-30 16:45:32 -07002267 main.ONOScli1.activateApp( "org.onosproject.election" )
2268 leaders = []
2269 for cli in CLIs:
2270 leader = cli.electionTestLeader()
2271 if leader is None or leader == main.FALSE:
2272 main.log.report( cli.name + ": Leader for the election app " +
2273 "should be an ONOS node, instead got '" +
2274 str( leader ) + "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002275 leaderResult = main.FALSE
Jon Halla9d26da2015-03-30 16:45:32 -07002276 leaders.append( leader )
2277 if len( set( leaders ) ) != 1:
2278 leaderResult = main.FALSE
2279 main.log.error( "Results of electionTestLeader is order of CLIs:" +
2280 str( leaders ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002281 if leaderResult:
Jon Hall6aec96b2015-01-19 14:49:31 -08002282 main.log.report( "Leadership election tests passed( consistent " +
2283 "view of leader across listeners and a leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002284 "was elected )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002285 utilities.assert_equals(
2286 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002287 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002288 onpass="Leadership election passed",
2289 onfail="Something went wrong with Leadership election" )
Jon Hall94fd0472014-12-08 11:52:42 -08002290
Jon Hall6aec96b2015-01-19 14:49:31 -08002291 def CASE15( self, main ):
2292 """
Jon Hall669173b2014-12-17 11:36:30 -08002293 Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -08002294 """
Jon Hall5cfd23c2015-03-19 11:40:57 -07002295 assert numControllers, "numControllers not defined"
2296 assert main, "main not defined"
2297 assert utilities.assert_equals, "utilities.assert_equals not defined"
2298 assert CLIs, "CLIs not defined"
2299 assert nodes, "nodes not defined"
2300
Jon Hall8f89dda2015-01-22 16:03:33 -08002301 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002302 description = "Check that Leadership Election is still functional"
Jon Hall6aec96b2015-01-19 14:49:31 -08002303 main.log.report( description )
2304 main.case( description )
2305 main.step( "Find current leader and withdraw" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002306 leader = main.ONOScli1.electionTestLeader()
Jon Halla9d26da2015-03-30 16:45:32 -07002307 # do some sanity checking on leader before using it
Jon Hall8f89dda2015-01-22 16:03:33 -08002308 withdrawResult = main.FALSE
Jon Hall65844a32015-03-09 19:09:37 -07002309 if leader is None or leader == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002310 main.log.report(
2311 "Leader for the election app should be an ONOS node," +
Jon Hall58c76b72015-02-23 11:09:24 -08002312 "instead got '" + str( leader ) + "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002313 leaderResult = main.FALSE
Jon Hall63604932015-02-26 17:09:50 -08002314 oldLeader = None
Jon Hall65844a32015-03-09 19:09:37 -07002315 for i in range( len( CLIs ) ):
2316 if leader == nodes[ i ].ip_address:
2317 oldLeader = CLIs[ i ]
2318 break
Jon Halla9d26da2015-03-30 16:45:32 -07002319 else: # FOR/ELSE statement
Jon Hall65844a32015-03-09 19:09:37 -07002320 main.log.error( "Leader election, could not find current leader" )
Jon Hall63604932015-02-26 17:09:50 -08002321 if oldLeader:
2322 withdrawResult = oldLeader.electionTestWithdraw()
Jon Hall6aec96b2015-01-19 14:49:31 -08002323 utilities.assert_equals(
2324 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002325 actual=withdrawResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002326 onpass="App was withdrawn from election",
2327 onfail="App was not withdrawn from election" )
Jon Hall669173b2014-12-17 11:36:30 -08002328
Jon Hall6aec96b2015-01-19 14:49:31 -08002329 main.step( "Make sure new leader is elected" )
Jon Halla9d26da2015-03-30 16:45:32 -07002330 # FIXME: use threads
Jon Hall8f89dda2015-01-22 16:03:33 -08002331 leaderList = []
Jon Hall5cfd23c2015-03-19 11:40:57 -07002332 for cli in CLIs:
2333 leaderN = cli.electionTestLeader()
Jon Hall65844a32015-03-09 19:09:37 -07002334 leaderList.append( leaderN )
Jon Hall669173b2014-12-17 11:36:30 -08002335 if leaderN == leader:
Jon Hall5cfd23c2015-03-19 11:40:57 -07002336 main.log.report( cli.name + " still sees " + str( leader ) +
Jon Hall65844a32015-03-09 19:09:37 -07002337 " as leader after they withdrew" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002338 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002339 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002340 # error in response
2341 # TODO: add check for "Command not found:" in the driver, this
Jon Hall65844a32015-03-09 19:09:37 -07002342 # means the app isn't loaded
Jon Hall6aec96b2015-01-19 14:49:31 -08002343 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002344 "electionTestLeader function, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002345 "check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002346 leaderResult = main.FALSE
Jon Halla9d26da2015-03-30 16:45:32 -07002347 elif leaderN is None:
2348 # node may not have recieved the event yet
2349 leaderN = cli.electionTestLeader()
2350 leaderList.pop()
2351 leaderList.append( leaderN )
Jon Hall8f89dda2015-01-22 16:03:33 -08002352 consistentLeader = main.FALSE
2353 if len( set( leaderList ) ) == 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08002354 main.log.info( "Each Election-app sees '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08002355 str( leaderList[ 0 ] ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002356 "' as the leader" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002357 consistentLeader = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002358 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002359 main.log.report(
2360 "Inconsistent responses for leader of Election-app:" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002361 for n in range( len( leaderList ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002362 main.log.report( "ONOS" + str( n + 1 ) + " response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002363 str( leaderList[ n ] ) )
Jon Hall5cfd23c2015-03-19 11:40:57 -07002364 leaderResult = leaderResult and consistentLeader
Jon Hall8f89dda2015-01-22 16:03:33 -08002365 if leaderResult:
2366 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002367 "view of leader across listeners and a new " +
2368 "leader was elected when the old leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002369 "resigned )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002370 utilities.assert_equals(
2371 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002372 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002373 onpass="Leadership election passed",
2374 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002375
Jon Hall58c76b72015-02-23 11:09:24 -08002376 main.step( "Run for election on old leader( just so everyone " +
2377 "is in the hat )" )
Jon Hall63604932015-02-26 17:09:50 -08002378 if oldLeader:
2379 runResult = oldLeader.electionTestRun()
2380 else:
2381 runResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002382 utilities.assert_equals(
2383 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002384 actual=runResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002385 onpass="App re-ran for election",
2386 onfail="App failed to run for election" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002387 if consistentLeader == main.TRUE:
2388 afterRun = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002389 # verify leader didn't just change
Jon Hall8f89dda2015-01-22 16:03:33 -08002390 if afterRun == leaderList[ 0 ]:
2391 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002392 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08002393 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002394 # TODO: assert on run and withdraw results?
Jon Hall669173b2014-12-17 11:36:30 -08002395
Jon Hall6aec96b2015-01-19 14:49:31 -08002396 utilities.assert_equals(
2397 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002398 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002399 onpass="Leadership election passed",
2400 onfail="Something went wrong with Leadership election after " +
2401 "the old leader re-ran for election" )