blob: 4b9b287eab307dfd1f1396de2e6f9fa69cfb6fbd [file] [log] [blame]
Jon Hall6aec96b2015-01-19 14:49:31 -08001"""
Jon Hall73cf9cc2014-11-20 22:28:38 -08002Description: This test is to determine if a single
3 instance ONOS 'cluster' can handle a restart
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign mastership to controllers
8CASE3: Assign intents
9CASE4: Ping across added host intents
10CASE5: Reading state of ONOS
11CASE6: The Failure case. Since this is the Sanity test, we do nothing.
12CASE7: Check state after control plane failure
13CASE8: Compare topo
14CASE9: Link s3-s28 down
15CASE10: Link s3-s28 up
16CASE11: Switch down
17CASE12: Switch up
18CASE13: Clean up
Jon Hall669173b2014-12-17 11:36:30 -080019CASE14: start election app on all onos nodes
20CASE15: Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -080021"""
Jon Hall8f89dda2015-01-22 16:03:33 -080022
23
Jon Hall48cf3ce2015-01-12 15:43:18 -080024class HATestSingleInstanceRestart:
Jon Hall73cf9cc2014-11-20 22:28:38 -080025
Jon Hall6aec96b2015-01-19 14:49:31 -080026 def __init__( self ):
Jon Hall73cf9cc2014-11-20 22:28:38 -080027 self.default = ''
28
Jon Hall6aec96b2015-01-19 14:49:31 -080029 def CASE1( self, main ):
30 """
Jon Hall73cf9cc2014-11-20 22:28:38 -080031 CASE1 is to compile ONOS and push it to the test machines
32
33 Startup sequence:
34 git pull
35 mvn clean install
36 onos-package
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-install -f
41 onos-wait-for-start
Jon Hall6aec96b2015-01-19 14:49:31 -080042 """
43 main.log.report( "ONOS Single node cluster restart " +
44 "HA test - initialization" )
45 main.case( "Setting up test environment" )
46 # TODO: save all the timers and output them for plotting
Jon Hall73cf9cc2014-11-20 22:28:38 -080047
48 # load some vairables from the params file
Jon Hall8f89dda2015-01-22 16:03:33 -080049 PULLCODE = False
Jon Hall6aec96b2015-01-19 14:49:31 -080050 if main.params[ 'Git' ] == 'True':
Jon Hall8f89dda2015-01-22 16:03:33 -080051 PULLCODE = True
Jon Hall529a37f2015-01-28 10:02:00 -080052 gitBranch = main.params[ 'branch' ]
Jon Hall8f89dda2015-01-22 16:03:33 -080053 cellName = main.params[ 'ENV' ][ 'cellName' ]
Jon Hall6aec96b2015-01-19 14:49:31 -080054
55 # set global variables
Jon Hall8f89dda2015-01-22 16:03:33 -080056 global ONOS1Ip
57 global ONOS1Port
58 global ONOS2Ip
59 global ONOS2Port
60 global ONOS3Ip
61 global ONOS3Port
62 global ONOS4Ip
63 global ONOS4Port
64 global ONOS5Ip
65 global ONOS5Port
66 global ONOS6Ip
67 global ONOS6Port
68 global ONOS7Ip
69 global ONOS7Port
70 global numControllers
Jon Hall73cf9cc2014-11-20 22:28:38 -080071
Jon Hall8f89dda2015-01-22 16:03:33 -080072 ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
73 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
74 ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
75 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
76 ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
77 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
78 ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
79 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
80 ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
81 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
82 ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
83 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
84 ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
85 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
86 numControllers = int( main.params[ 'num_controllers' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -080087
Jon Hall6aec96b2015-01-19 14:49:31 -080088 main.step( "Applying cell variable to environment" )
Jon Hall8f89dda2015-01-22 16:03:33 -080089 cellResult = main.ONOSbench.setCell( cellName )
90 verifyResult = main.ONOSbench.verifyCell()
Jon Hall73cf9cc2014-11-20 22:28:38 -080091
Jon Hall6aec96b2015-01-19 14:49:31 -080092 # FIXME:this is short term fix
93 main.log.report( "Removing raft logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -080094 main.ONOSbench.onosRemoveRaftLogs()
Jon Hall6aec96b2015-01-19 14:49:31 -080095 main.log.report( "Uninstalling ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -080096 main.ONOSbench.onosUninstall( ONOS1Ip )
97 main.ONOSbench.onosUninstall( ONOS2Ip )
98 main.ONOSbench.onosUninstall( ONOS3Ip )
99 main.ONOSbench.onosUninstall( ONOS4Ip )
100 main.ONOSbench.onosUninstall( ONOS5Ip )
101 main.ONOSbench.onosUninstall( ONOS6Ip )
102 main.ONOSbench.onosUninstall( ONOS7Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800103
Jon Hall8f89dda2015-01-22 16:03:33 -0800104 cleanInstallResult = main.TRUE
105 gitPullResult = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800106
Jon Hall97f31752015-02-04 12:01:04 -0800107 main.step( "Starting Mininet" )
108 main.Mininet1.startNet( )
109
Jon Hall6aec96b2015-01-19 14:49:31 -0800110 main.step( "Compiling the latest version of ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800111 if PULLCODE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800112 # TODO Configure branch in params
113 main.step( "Git checkout and pull master" )
Jon Hall529a37f2015-01-28 10:02:00 -0800114 main.ONOSbench.gitCheckout( gitBranch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800115 gitPullResult = main.ONOSbench.gitPull()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800116
Jon Hall6aec96b2015-01-19 14:49:31 -0800117 main.step( "Using mvn clean & install" )
Jon Hall529a37f2015-01-28 10:02:00 -0800118 cleanInstallResult = main.ONOSbench.cleanInstall()
119 else:
120 main.log.warn( "Did not pull new code so skipping mvn " +
121 "clean install" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800122 main.ONOSbench.getVersion( report=True )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800123
Jon Hall8f89dda2015-01-22 16:03:33 -0800124 cellResult = main.ONOSbench.setCell( "SingleHA" )
125 verifyResult = main.ONOSbench.verifyCell()
Jon Hall6aec96b2015-01-19 14:49:31 -0800126 main.step( "Creating ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800127 packageResult = main.ONOSbench.onosPackage()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800128
Jon Hall6aec96b2015-01-19 14:49:31 -0800129 main.step( "Installing ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800130 onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
131 node=ONOS1Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800132
Jon Hall6aec96b2015-01-19 14:49:31 -0800133 main.step( "Checking if ONOS is up yet" )
134 # TODO check bundle:list?
135 for i in range( 2 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800136 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
137 if onos1Isup:
Jon Hall94fd0472014-12-08 11:52:42 -0800138 break
Jon Hall8f89dda2015-01-22 16:03:33 -0800139 if not onos1Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800140 main.log.report( "ONOS1 didn't start!" )
Jon Hall94fd0472014-12-08 11:52:42 -0800141
Jon Hall8f89dda2015-01-22 16:03:33 -0800142 cliResult = main.ONOScli1.startOnosCli( ONOS1Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800143
Jon Hall6aec96b2015-01-19 14:49:31 -0800144 main.step( "Start Packet Capture MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800145 main.Mininet2.startTcpdump(
Jon Hall6aec96b2015-01-19 14:49:31 -0800146 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
147 + "-MN.pcap",
148 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
149 port=main.params[ 'MNtcpdump' ][ 'port' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800150
Jon Hall8f89dda2015-01-22 16:03:33 -0800151 case1Result = ( cleanInstallResult and packageResult and
152 cellResult and verifyResult and onos1InstallResult
153 and onos1Isup and cliResult )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800154
Jon Hall8f89dda2015-01-22 16:03:33 -0800155 utilities.assert_equals( expect=main.TRUE, actual=case1Result,
156 onpass="Test startup successful",
157 onfail="Test startup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800158
Jon Hall8f89dda2015-01-22 16:03:33 -0800159 if case1Result == main.FALSE:
Jon Hall73cf9cc2014-11-20 22:28:38 -0800160 main.cleanup()
161 main.exit()
162
Jon Hall6aec96b2015-01-19 14:49:31 -0800163 def CASE2( self, main ):
164 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800165 Assign mastership to controllers
Jon Hall6aec96b2015-01-19 14:49:31 -0800166 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800167 import re
168
Jon Hall6aec96b2015-01-19 14:49:31 -0800169 main.log.report( "Assigning switches to controllers" )
170 main.case( "Assigning Controllers" )
171 main.step( "Assign switches to controllers" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800172
Jon Hall6aec96b2015-01-19 14:49:31 -0800173 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800174 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -0800175 sw=str( i ),
Jon Hall8f89dda2015-01-22 16:03:33 -0800176 ip1=ONOS1Ip, port1=ONOS1Port )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800177
Jon Hall8f89dda2015-01-22 16:03:33 -0800178 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800179 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800180 response = main.Mininet1.getSwController( "s" + str( i ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800181 try:
Jon Hall6aec96b2015-01-19 14:49:31 -0800182 main.log.info( str( response ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800183 except:
Jon Hall6aec96b2015-01-19 14:49:31 -0800184 main.log.info( repr( response ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800185 if re.search( "tcp:" + ONOS1Ip, response ):
186 mastershipCheck = mastershipCheck and main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800187 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800188 mastershipCheck = main.FALSE
189 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800190 main.log.report( "Switch mastership assigned correctly" )
191 utilities.assert_equals(
192 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800193 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800194 onpass="Switch mastership assigned correctly",
195 onfail="Switches not assigned correctly to controllers" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800196
Jon Hall6aec96b2015-01-19 14:49:31 -0800197 def CASE3( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800198 """
199 Assign intents
Jon Hall73cf9cc2014-11-20 22:28:38 -0800200 """
Jon Hall6aec96b2015-01-19 14:49:31 -0800201 # FIXME: we must reinstall intents until we have a persistant
202 # datastore!
Jon Hall73cf9cc2014-11-20 22:28:38 -0800203 import time
Jon Hall6aec96b2015-01-19 14:49:31 -0800204 main.log.report( "Adding host intents" )
205 main.case( "Adding host Intents" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800206
Jon Hall8f89dda2015-01-22 16:03:33 -0800207 main.step( "Discovering Hosts( Via pingall for now )" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800208 # FIXME: Once we have a host discovery mechanism, use that instead
Jon Hall73cf9cc2014-11-20 22:28:38 -0800209
Jon Hall6aec96b2015-01-19 14:49:31 -0800210 # install onos-app-fwd
211 main.log.info( "Install reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800212 main.ONOScli1.featureInstall( "onos-app-fwd" )
Jon Hall94fd0472014-12-08 11:52:42 -0800213
Jon Hall6aec96b2015-01-19 14:49:31 -0800214 # REACTIVE FWD test
Jon Hall8f89dda2015-01-22 16:03:33 -0800215 pingResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800216 time1 = time.time()
Jon Hall8f89dda2015-01-22 16:03:33 -0800217 pingResult = main.Mininet1.pingall()
Jon Hall529a37f2015-01-28 10:02:00 -0800218 utilities.assert_equals(
219 expect=main.TRUE,
220 actual=pingResult,
221 onpass="Reactive Pingall test passed",
222 onfail="Reactive Pingall failed, one or more ping pairs failed" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800223 time2 = time.time()
Jon Hall6aec96b2015-01-19 14:49:31 -0800224 main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800225
Jon Hall6aec96b2015-01-19 14:49:31 -0800226 # uninstall onos-app-fwd
227 main.log.info( "Uninstall reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800228 main.ONOScli1.featureUninstall( "onos-app-fwd" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800229 # timeout for fwd flows
230 time.sleep( 10 )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800231
Jon Hall6aec96b2015-01-19 14:49:31 -0800232 main.step( "Add host intents" )
233 # TODO: move the host numbers to params
Jon Hall8f89dda2015-01-22 16:03:33 -0800234 intentAddResult = True
Jon Hall6aec96b2015-01-19 14:49:31 -0800235 for i in range( 8, 18 ):
236 main.log.info( "Adding host intent between h" + str( i ) +
237 " and h" + str( i + 10 ) )
238 host1 = "00:00:00:00:00:" + \
239 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
240 host2 = "00:00:00:00:00:" + \
241 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800242 # NOTE: getHost can return None
243 host1Dict = main.ONOScli1.getHost( host1 )
244 host2Dict = main.ONOScli1.getHost( host2 )
245 host1Id = None
246 host2Id = None
247 if host1Dict and host2Dict:
248 host1Id = host1Dict.get( 'id', None )
249 host2Id = host2Dict.get( 'id', None )
Jon Hall8f89dda2015-01-22 16:03:33 -0800250 if host1Id and host2Id:
251 tmpResult = main.ONOScli1.addHostIntent(
252 host1Id,
253 host2Id )
Jon Hall669173b2014-12-17 11:36:30 -0800254 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800255 main.log.error( "Error, getHost() failed" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800256 main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
257 sort_keys=True,
258 indent=4,
259 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800260 tmpResult = main.FALSE
261 intentAddResult = bool( pingResult and intentAddResult
262 and tmpResult )
Jon Hall529a37f2015-01-28 10:02:00 -0800263 # TODO Check that intents were added?
Jon Hall1b8f54a2015-02-04 13:24:20 -0800264 # Print the intent states
265 intents = main.ONOScli1.intents( )
266 intentStates = []
267 for intent in json.loads( intents ): # Iter through intents of a node
268 intentStates.append( intent.get( 'state', None ) )
269 out = [ (i, intentStates.count( i ) ) for i in set( intentStates ) ]
270 main.log.info( dict( out ) )
271
Jon Hall6aec96b2015-01-19 14:49:31 -0800272 utilities.assert_equals(
273 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800274 actual=intentAddResult,
Jon Hall529a37f2015-01-28 10:02:00 -0800275 onpass="Pushed host intents to ONOS",
276 onfail="Error in pushing host intents to ONOS" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800277 # TODO Check if intents all exist in datastore
Jon Hall73cf9cc2014-11-20 22:28:38 -0800278
Jon Hall6aec96b2015-01-19 14:49:31 -0800279 def CASE4( self, main ):
Jon Hall73cf9cc2014-11-20 22:28:38 -0800280 """
281 Ping across added host intents
282 """
283 description = " Ping across added host intents"
Jon Hall6aec96b2015-01-19 14:49:31 -0800284 main.log.report( description )
285 main.case( description )
Jon Hall8f89dda2015-01-22 16:03:33 -0800286 PingResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800287 for i in range( 8, 18 ):
288 ping = main.Mininet1.pingHost(
289 src="h" + str( i ), target="h" + str( i + 10 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800290 PingResult = PingResult and ping
Jon Hall6aec96b2015-01-19 14:49:31 -0800291 if ping == main.FALSE:
292 main.log.warn( "Ping failed between h" + str( i ) +
293 " and h" + str( i + 10 ) )
294 elif ping == main.TRUE:
295 main.log.info( "Ping test passed!" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800296 PingResult = main.TRUE
297 if PingResult == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800298 main.log.report(
299 "Intents have not been installed correctly, pings failed." )
Jon Hall529a37f2015-01-28 10:02:00 -0800300 #TODO: pretty print
301 main.log.warn( "ONSO1 intents: " )
302 main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
303 sort_keys=True,
304 indent=4,
305 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800306 if PingResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800307 main.log.report(
308 "Intents have been installed correctly and verified by pings" )
309 utilities.assert_equals(
310 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800311 actual=PingResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800312 onpass="Intents have been installed correctly and pings work",
313 onfail="Intents have not been installed correctly, pings failed." )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800314
Jon Hall6aec96b2015-01-19 14:49:31 -0800315 def CASE5( self, main ):
316 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800317 Reading state of ONOS
Jon Hall6aec96b2015-01-19 14:49:31 -0800318 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800319 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800320 # assumes that sts is already in you PYTHONPATH
321 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -0800322
Jon Hall6aec96b2015-01-19 14:49:31 -0800323 main.log.report( "Setting up and gathering data for current state" )
324 main.case( "Setting up and gathering data for current state" )
325 # The general idea for this test case is to pull the state of
326 # ( intents,flows, topology,... ) from each ONOS node
327 # We can then compare them with eachother and also with past states
Jon Hall73cf9cc2014-11-20 22:28:38 -0800328
Jon Hall6aec96b2015-01-19 14:49:31 -0800329 main.step( "Get the Mastership of each switch from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800330 global mastershipState
331 mastershipState = []
Jon Hall94fd0472014-12-08 11:52:42 -0800332
Jon Hall6aec96b2015-01-19 14:49:31 -0800333 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -0800334 rolesNotNull = main.ONOScli1.rolesNotNull()
Jon Hall6aec96b2015-01-19 14:49:31 -0800335 utilities.assert_equals(
336 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800337 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -0800338 onpass="Each device has a master",
339 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -0800340
Jon Hall8f89dda2015-01-22 16:03:33 -0800341 ONOS1Mastership = main.ONOScli1.roles()
Jon Hall6aec96b2015-01-19 14:49:31 -0800342 # TODO: Make this a meaningful check
Jon Hall8f89dda2015-01-22 16:03:33 -0800343 if "Error" in ONOS1Mastership or not ONOS1Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -0800344 main.log.report( "Error in getting ONOS roles" )
345 main.log.warn(
346 "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800347 repr( ONOS1Mastership ) )
348 consistentMastership = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800349 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800350 mastershipState = ONOS1Mastership
351 consistentMastership = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800352
Jon Hall6aec96b2015-01-19 14:49:31 -0800353 main.step( "Get the intents from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800354 global intentState
355 intentState = []
356 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
357 intentCheck = main.FALSE
358 if "Error" in ONOS1Intents or not ONOS1Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -0800359 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800360 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800361 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800362 intentCheck = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800363
Jon Hall6aec96b2015-01-19 14:49:31 -0800364 main.step( "Get the flows from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800365 global flowState
366 flowState = []
367 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
368 flowCheck = main.FALSE
369 if "Error" in ONOS1Flows or not ONOS1Flows:
Jon Hall6aec96b2015-01-19 14:49:31 -0800370 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800371 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800372 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800373 # TODO: Do a better check, maybe compare flows on switches?
Jon Hall8f89dda2015-01-22 16:03:33 -0800374 flowState = ONOS1Flows
375 flowCheck = main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800376
Jon Hall6aec96b2015-01-19 14:49:31 -0800377 main.step( "Get the OF Table entries" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800378 global flows
Jon Hall6aec96b2015-01-19 14:49:31 -0800379 flows = []
380 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800381 flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800382
Jon Hall6aec96b2015-01-19 14:49:31 -0800383 # TODO: Compare switch flow tables with ONOS flow tables
Jon Hall73cf9cc2014-11-20 22:28:38 -0800384
Jon Hall6aec96b2015-01-19 14:49:31 -0800385 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800386 ctrls = []
387 count = 1
388 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -0800389 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
390 temp = temp + ( "ONOS" + str( count ), )
391 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
392 temp = temp + \
393 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
394 ctrls.append( temp )
395 MNTopo = TestONTopology(
396 main.Mininet1,
397 ctrls ) # can also add Intent API info for intent operations
Jon Hall73cf9cc2014-11-20 22:28:38 -0800398
Jon Hall6aec96b2015-01-19 14:49:31 -0800399 main.step( "Collecting topology information from ONOS" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800400 devices = []
401 devices.append( main.ONOScli1.devices() )
Jon Hall6aec96b2015-01-19 14:49:31 -0800402 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800403 hosts = []
404 hosts.append( main.ONOScli1.hosts() )
Jon Hall6aec96b2015-01-19 14:49:31 -0800405 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800406 ports = []
407 ports.append( main.ONOScli1.ports() )
408 links = []
409 links.append( main.ONOScli1.links() )
410
Jon Hall6aec96b2015-01-19 14:49:31 -0800411 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800412 devicesResults = main.TRUE
413 portsResults = main.TRUE
414 linksResults = main.TRUE
415 for controller in range( numControllers ):
416 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -0800417 if devices[ controller ] or "Error" not in devices[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -0800418 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -0800419 MNTopo,
420 json.loads(
421 devices[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800422 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800423 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800424 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800425 actual=currentDevicesResult,
426 onpass="ONOS" + controllerStr +
427 " Switches view is correct",
428 onfail="ONOS" + controllerStr +
429 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800430
Jon Hall6aec96b2015-01-19 14:49:31 -0800431 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -0800432 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -0800433 MNTopo,
434 json.loads(
435 ports[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800436 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800437 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800438 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800439 actual=currentPortsResult,
440 onpass="ONOS" + controllerStr +
441 " ports view is correct",
442 onfail="ONOS" + controllerStr +
443 " ports view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800444
Jon Hall6aec96b2015-01-19 14:49:31 -0800445 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -0800446 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -0800447 MNTopo,
448 json.loads(
449 links[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800450 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800451 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800452 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800453 actual=currentLinksResult,
454 onpass="ONOS" + controllerStr +
455 " links view is correct",
456 onfail="ONOS" + controllerStr +
457 " links view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800458
Jon Hall8f89dda2015-01-22 16:03:33 -0800459 devicesResults = devicesResults and currentDevicesResult
460 portsResults = portsResults and currentPortsResult
461 linksResults = linksResults and currentLinksResult
Jon Hall73cf9cc2014-11-20 22:28:38 -0800462
Jon Hall8f89dda2015-01-22 16:03:33 -0800463 topoResult = devicesResults and portsResults and linksResults
464 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
465 onpass="Topology Check Test successful",
466 onfail="Topology Check Test NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800467
Jon Hall8f89dda2015-01-22 16:03:33 -0800468 finalAssert = main.TRUE
469 finalAssert = finalAssert and topoResult and flowCheck \
470 and intentCheck and consistentMastership and rolesNotNull
471 utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
472 onpass="State check successful",
473 onfail="State check NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800474
Jon Hall6aec96b2015-01-19 14:49:31 -0800475 def CASE6( self, main ):
476 """
Jon Hallffb386d2014-11-21 13:43:38 -0800477 The Failure case.
Jon Hall6aec96b2015-01-19 14:49:31 -0800478 """
Jon Hallffb386d2014-11-21 13:43:38 -0800479 import time
Jon Hall73cf9cc2014-11-20 22:28:38 -0800480
Jon Hall6aec96b2015-01-19 14:49:31 -0800481 main.log.report( "Restart ONOS node" )
482 main.log.case( "Restart ONOS node" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800483 main.ONOSbench.onosKill( ONOS1Ip )
Jon Hallffb386d2014-11-21 13:43:38 -0800484 start = time.time()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800485
Jon Hall6aec96b2015-01-19 14:49:31 -0800486 main.step( "Checking if ONOS is up yet" )
Jon Hallffb386d2014-11-21 13:43:38 -0800487 count = 0
Jon Hall94fd0472014-12-08 11:52:42 -0800488 while count < 10:
Jon Hall8f89dda2015-01-22 16:03:33 -0800489 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
490 if onos1Isup == main.TRUE:
Jon Hallffb386d2014-11-21 13:43:38 -0800491 elapsed = time.time() - start
492 break
493 else:
494 count = count + 1
Jon Hall73cf9cc2014-11-20 22:28:38 -0800495
Jon Hall8f89dda2015-01-22 16:03:33 -0800496 cliResult = main.ONOScli1.startOnosCli( ONOS1Ip )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800497
Jon Hall8f89dda2015-01-22 16:03:33 -0800498 caseResults = main.TRUE and onos1Isup and cliResult
499 utilities.assert_equals( expect=main.TRUE, actual=caseResults,
500 onpass="ONOS restart successful",
501 onfail="ONOS restart NOT successful" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800502 main.log.info(
503 "ESTIMATE: ONOS took %s seconds to restart" %
504 str( elapsed ) )
505 time.sleep( 5 )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800506
Jon Hall6aec96b2015-01-19 14:49:31 -0800507 def CASE7( self, main ):
508 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800509 Check state after ONOS failure
Jon Hall6aec96b2015-01-19 14:49:31 -0800510 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800511 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800512 main.case( "Running ONOS Constant State Tests" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800513
Jon Hall6aec96b2015-01-19 14:49:31 -0800514 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -0800515 rolesNotNull = main.ONOScli1.rolesNotNull()
Jon Hall6aec96b2015-01-19 14:49:31 -0800516 utilities.assert_equals(
517 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800518 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -0800519 onpass="Each device has a master",
520 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -0800521
Jon Hall6aec96b2015-01-19 14:49:31 -0800522 main.step( "Check if switch roles are consistent across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800523 ONOS1Mastership = main.ONOScli1.roles()
Jon Hall6aec96b2015-01-19 14:49:31 -0800524 # FIXME: Refactor this whole case for single instance
Jon Hall8f89dda2015-01-22 16:03:33 -0800525 if "Error" in ONOS1Mastership or not ONOS1Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -0800526 main.log.report( "Error in getting ONOS mastership" )
527 main.log.warn(
528 "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800529 repr( ONOS1Mastership ) )
530 consistentMastership = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800531 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800532 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800533 main.log.report(
534 "Switch roles are consistent across all ONOS nodes" )
535 utilities.assert_equals(
536 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800537 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -0800538 onpass="Switch roles are consistent across all ONOS nodes",
539 onfail="ONOS nodes have different views of switch roles" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800540
541 description2 = "Compare switch roles from before failure"
Jon Hall6aec96b2015-01-19 14:49:31 -0800542 main.step( description2 )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800543
Jon Hall8f89dda2015-01-22 16:03:33 -0800544 currentJson = json.loads( ONOS1Mastership )
545 oldJson = json.loads( mastershipState )
546 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800547 for i in range( 1, 29 ):
548 switchDPID = str(
549 main.Mininet1.getSwitchDPID(
550 switch="s" +
551 str( i ) ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800552
Jon Hall8f89dda2015-01-22 16:03:33 -0800553 current = [ switch[ 'master' ] for switch in currentJson
Jon Hall6aec96b2015-01-19 14:49:31 -0800554 if switchDPID in switch[ 'id' ] ]
Jon Hall8f89dda2015-01-22 16:03:33 -0800555 old = [ switch[ 'master' ] for switch in oldJson
Jon Hall6aec96b2015-01-19 14:49:31 -0800556 if switchDPID in switch[ 'id' ] ]
Jon Hall73cf9cc2014-11-20 22:28:38 -0800557 if current == old:
Jon Hall8f89dda2015-01-22 16:03:33 -0800558 mastershipCheck = mastershipCheck and main.TRUE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800559 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800560 main.log.warn( "Mastership of switch %s changed" % switchDPID )
Jon Hall8f89dda2015-01-22 16:03:33 -0800561 mastershipCheck = main.FALSE
562 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800563 main.log.report( "Mastership of Switches was not changed" )
564 utilities.assert_equals(
565 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800566 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800567 onpass="Mastership of Switches was not changed",
568 onfail="Mastership of some switches changed" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800569 mastershipCheck = mastershipCheck and consistentMastership
Jon Hall73cf9cc2014-11-20 22:28:38 -0800570
Jon Hall6aec96b2015-01-19 14:49:31 -0800571 main.step( "Get the intents and compare across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800572 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
573 intentCheck = main.FALSE
574 if "Error" in ONOS1Intents or not ONOS1Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -0800575 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800576 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800577 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800578 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800579 main.log.report( "Intents are consistent across all ONOS nodes" )
580 utilities.assert_equals(
581 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800582 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800583 onpass="Intents are consistent across all ONOS nodes",
584 onfail="ONOS nodes have different views of intents" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800585 # Print the intent states
586 intents = []
587 intents.append( ONOS1Intents )
588 intentStates = []
589 for node in intents: # Iter through ONOS nodes
590 nodeStates = []
591 for intent in json.loads( node ): # Iter through intents of a node
592 nodeStates.append( intent[ 'state' ] )
593 intentStates.append( nodeStates )
594 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
595 main.log.info( dict( out ) )
596
Jon Hall73cf9cc2014-11-20 22:28:38 -0800597
Jon Hall6aec96b2015-01-19 14:49:31 -0800598 # NOTE: Hazelcast has no durability, so intents are lost across system
599 # restarts
600 """
601 main.step( "Compare current intents with intents before the failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800602 # NOTE: this requires case 5 to pass for intentState to be set.
Jon Hall6aec96b2015-01-19 14:49:31 -0800603 # maybe we should stop the test if that fails?
Jon Hall1b8f54a2015-02-04 13:24:20 -0800604 sameIntents = main.TRUE
605 if intentState and intentState == ONOS1Intents:
Jon Hall8f89dda2015-01-22 16:03:33 -0800606 sameIntents = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800607 main.log.report( "Intents are consistent with before failure" )
608 # TODO: possibly the states have changed? we may need to figure out
609 # what the aceptable states are
Jon Hall73cf9cc2014-11-20 22:28:38 -0800610 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800611 try:
612 main.log.warn( "ONOS1 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -0800613 print json.dumps( json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800614 sort_keys=True, indent=4,
615 separators=( ',', ': ' ) )
616 except:
617 pass
Jon Hall8f89dda2015-01-22 16:03:33 -0800618 sameIntents = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800619 utilities.assert_equals(
620 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800621 actual=sameIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -0800622 onpass="Intents are consistent with before failure",
623 onfail="The Intents changed during failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800624 intentCheck = intentCheck and sameIntents
Jon Hall6aec96b2015-01-19 14:49:31 -0800625 """
626 main.step( "Get the OF Table entries and compare to before " +
627 "component failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800628 FlowTables = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800629 flows2 = []
630 for i in range( 28 ):
631 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800632 tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
633 flows2.append( tmpFlows )
634 tempResult = main.Mininet2.flowComp(
Jon Hall6aec96b2015-01-19 14:49:31 -0800635 flow1=flows[ i ],
Jon Hall8f89dda2015-01-22 16:03:33 -0800636 flow2=tmpFlows )
637 FlowTables = FlowTables and tempResult
638 if FlowTables == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800639 main.log.info( "Differences in flow table for switch: s" +
640 str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800641 if FlowTables == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800642 main.log.report( "No changes were found in the flow tables" )
643 utilities.assert_equals(
644 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800645 actual=FlowTables,
Jon Hall6aec96b2015-01-19 14:49:31 -0800646 onpass="No changes were found in the flow tables",
647 onfail="Changes were found in the flow tables" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800648
Jon Hall6aec96b2015-01-19 14:49:31 -0800649 # Test of LeadershipElection
Jon Hall669173b2014-12-17 11:36:30 -0800650
Jon Hall8f89dda2015-01-22 16:03:33 -0800651 leader = ONOS1Ip
652 leaderResult = main.TRUE
653 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -0800654 # loop through ONOScli handlers
655 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800656 leaderN = node.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -0800657 # verify leader is ONOS1
658 # NOTE even though we restarted ONOS, it is the only one so onos 1
659 # must be leader
Jon Hall669173b2014-12-17 11:36:30 -0800660 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -0800661 # all is well
Jon Hall669173b2014-12-17 11:36:30 -0800662 pass
663 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800664 # error in response
665 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800666 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -0800667 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800668 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -0800669 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -0800670 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800671 main.log.report( "ONOS" + str( controller ) +
672 " sees " + str( leaderN ) +
673 " as the leader of the election app." +
674 " Leader should be " + str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800675 if leaderResult:
676 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -0800677 "view of leader across listeners and a new " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800678 "leader was re-elected if applicable )" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800679 utilities.assert_equals(
680 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800681 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800682 onpass="Leadership election passed",
683 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -0800684
Jon Hall8f89dda2015-01-22 16:03:33 -0800685 result = ( mastershipCheck and intentCheck and FlowTables and
686 rolesNotNull and leaderResult )
Jon Hall6aec96b2015-01-19 14:49:31 -0800687 result = int( result )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800688 if result == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800689 main.log.report( "Constant State Tests Passed" )
690 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall8f89dda2015-01-22 16:03:33 -0800691 onpass="Constant State Tests Passed",
692 onfail="Constant state tests failed" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800693
Jon Hall6aec96b2015-01-19 14:49:31 -0800694 def CASE8( self, main ):
695 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800696 Compare topo
Jon Hall6aec96b2015-01-19 14:49:31 -0800697 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800698 import sys
Jon Hall6aec96b2015-01-19 14:49:31 -0800699 # FIXME add this path to params
700 sys.path.append( "/home/admin/sts" )
701 # assumes that sts is already in you PYTHONPATH
702 from sts.topology.teston_topology import TestONTopology
Jon Hall73cf9cc2014-11-20 22:28:38 -0800703 import json
704 import time
705
Jon Hall6aec96b2015-01-19 14:49:31 -0800706 description = "Compare ONOS Topology view to Mininet topology"
707 main.case( description )
708 main.log.report( description )
709 main.step( "Create TestONTopology object" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800710 ctrls = []
711 count = 1
712 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -0800713 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
714 temp = temp + ( "ONOS" + str( count ), )
715 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
716 temp = temp + \
717 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
718 ctrls.append( temp )
719 MNTopo = TestONTopology(
720 main.Mininet1,
721 ctrls ) # can also add Intent API info for intent operations
Jon Hall73cf9cc2014-11-20 22:28:38 -0800722
Jon Hall6aec96b2015-01-19 14:49:31 -0800723 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800724 devicesResults = main.TRUE
725 portsResults = main.TRUE
726 linksResults = main.TRUE
727 topoResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -0800728 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -0800729 count = 0
Jon Hall6aec96b2015-01-19 14:49:31 -0800730 main.step( "Collecting topology information from ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800731 startTime = time.time()
732 while topoResult == main.FALSE and elapsed < 60:
Jon Hallffb386d2014-11-21 13:43:38 -0800733 count = count + 1
Jon Hall94fd0472014-12-08 11:52:42 -0800734 if count > 1:
Jon Hall6aec96b2015-01-19 14:49:31 -0800735 MNTopo = TestONTopology(
736 main.Mininet1,
737 ctrls )
Jon Hall8f89dda2015-01-22 16:03:33 -0800738 cliStart = time.time()
Jon Hall94fd0472014-12-08 11:52:42 -0800739 devices = []
740 devices.append( main.ONOScli1.devices() )
Jon Hall6aec96b2015-01-19 14:49:31 -0800741 """
Jon Hall94fd0472014-12-08 11:52:42 -0800742 hosts = []
743 hosts.append( main.ONOScli1.hosts() )
Jon Hall6aec96b2015-01-19 14:49:31 -0800744 """
Jon Hall94fd0472014-12-08 11:52:42 -0800745 ports = []
746 ports.append( main.ONOScli1.ports() )
747 links = []
748 links.append( main.ONOScli1.links() )
Jon Hall8f89dda2015-01-22 16:03:33 -0800749 elapsed = time.time() - startTime
750 cliTime = time.time() - cliStart
751 print "CLI time: " + str( cliTime )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800752
Jon Hall8f89dda2015-01-22 16:03:33 -0800753 for controller in range( numControllers ):
754 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -0800755 if devices[ controller ] or "Error" not in devices[
756 controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -0800757 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -0800758 MNTopo,
759 json.loads(
760 devices[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -0800761 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800762 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800763 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800764 actual=currentDevicesResult,
765 onpass="ONOS" + controllerStr +
766 " Switches view is correct",
767 onfail="ONOS" + controllerStr +
768 " Switches view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800769
Jon Hall6aec96b2015-01-19 14:49:31 -0800770 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -0800771 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -0800772 MNTopo,
773 json.loads(
774 ports[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -0800775 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800776 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800777 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800778 actual=currentPortsResult,
779 onpass="ONOS" + controllerStr +
780 " ports view is correct",
781 onfail="ONOS" + controllerStr +
782 " ports view is incorrect" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800783
Jon Hall6aec96b2015-01-19 14:49:31 -0800784 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -0800785 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -0800786 MNTopo,
787 json.loads(
788 links[ controller ] ) )
Jon Hall94fd0472014-12-08 11:52:42 -0800789 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800790 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800791 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800792 actual=currentLinksResult,
793 onpass="ONOS" + controllerStr +
794 " links view is correct",
795 onfail="ONOS" + controllerStr +
796 " links view is incorrect" )
797 devicesResults = devicesResults and currentDevicesResult
798 portsResults = portsResults and currentPortsResult
799 linksResults = linksResults and currentLinksResult
800 topoResult = devicesResults and portsResults and linksResults
Jon Hall94fd0472014-12-08 11:52:42 -0800801
Jon Hall8f89dda2015-01-22 16:03:33 -0800802 topoResult = topoResult and int( count <= 2 )
803 note = "note it takes about " + str( int( cliTime ) ) + \
804 " seconds for the test to make all the cli calls to fetch " +\
805 "the topology from each ONOS instance"
Jon Hall1b8f54a2015-02-04 13:24:20 -0800806 main.log.info(
Jon Hall8f89dda2015-01-22 16:03:33 -0800807 "Very crass estimate for topology discovery/convergence( " +
808 str( note ) + " ): " + str( elapsed ) + " seconds, " +
Jon Hall6aec96b2015-01-19 14:49:31 -0800809 str( count ) + " tries" )
Jon Hall94fd0472014-12-08 11:52:42 -0800810 if elapsed > 60:
Jon Hall6aec96b2015-01-19 14:49:31 -0800811 main.log.report( "Giving up on topology convergence" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800812 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
813 onpass="Topology Check Test successful",
814 onfail="Topology Check Test NOT successful" )
815 if topoResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800816 main.log.report( "ONOS topology view matches Mininet topology" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800817
Jon Hall6aec96b2015-01-19 14:49:31 -0800818 def CASE9( self, main ):
819 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800820 Link s3-s28 down
Jon Hall6aec96b2015-01-19 14:49:31 -0800821 """
822 import time
823 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -0800824
Jon Hall8f89dda2015-01-22 16:03:33 -0800825 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800826
Jon Hall6aec96b2015-01-19 14:49:31 -0800827 description = "Turn off a link to ensure that Link Discovery " +\
Jon Hall8f89dda2015-01-22 16:03:33 -0800828 "is working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -0800829 main.log.report( description )
830 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800831
Jon Hall6aec96b2015-01-19 14:49:31 -0800832 main.step( "Kill Link between s3 and s28" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800833 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800834 main.log.info(
835 "Waiting " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800836 str( linkSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -0800837 " seconds for link down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800838 time.sleep( linkSleep )
839 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
840 onpass="Link down succesful",
841 onfail="Failed to bring link down" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800842 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -0800843
Jon Hall6aec96b2015-01-19 14:49:31 -0800844 def CASE10( self, main ):
845 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800846 Link s3-s28 up
Jon Hall6aec96b2015-01-19 14:49:31 -0800847 """
848 import time
849 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -0800850
Jon Hall8f89dda2015-01-22 16:03:33 -0800851 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800852
Jon Hall6aec96b2015-01-19 14:49:31 -0800853 description = "Restore a link to ensure that Link Discovery is " + \
Jon Hall8f89dda2015-01-22 16:03:33 -0800854 "working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -0800855 main.log.report( description )
856 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800857
Jon Hall6aec96b2015-01-19 14:49:31 -0800858 main.step( "Bring link between s3 and s28 back up" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800859 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800860 main.log.info(
861 "Waiting " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800862 str( linkSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -0800863 " seconds for link up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800864 time.sleep( linkSleep )
865 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
866 onpass="Link up succesful",
867 onfail="Failed to bring link up" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800868 # TODO do some sort of check here
Jon Hall73cf9cc2014-11-20 22:28:38 -0800869
Jon Hall6aec96b2015-01-19 14:49:31 -0800870 def CASE11( self, main ):
871 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800872 Switch Down
Jon Hall6aec96b2015-01-19 14:49:31 -0800873 """
874 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -0800875 import time
876
Jon Hall8f89dda2015-01-22 16:03:33 -0800877 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800878
879 description = "Killing a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -0800880 main.log.report( description )
881 main.case( description )
882 switch = main.params[ 'kill' ][ 'switch' ]
883 switchDPID = main.params[ 'kill' ][ 'dpid' ]
Jon Hall73cf9cc2014-11-20 22:28:38 -0800884
Jon Hall6aec96b2015-01-19 14:49:31 -0800885 # TODO: Make this switch parameterizable
886 main.step( "Kill " + switch )
887 main.log.report( "Deleting " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800888 main.Mininet1.delSwitch( switch )
889 main.log.info( "Waiting " + str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -0800890 " seconds for switch down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800891 time.sleep( switchSleep )
892 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -0800893 # Peek at the deleted switch
894 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -0800895 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800896 if device and device[ 'available' ] is False:
Jon Hall94fd0472014-12-08 11:52:42 -0800897 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800898 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall8f89dda2015-01-22 16:03:33 -0800899 onpass="Kill switch succesful",
900 onfail="Failed to kill switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800901
Jon Hall6aec96b2015-01-19 14:49:31 -0800902 def CASE12( self, main ):
903 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800904 Switch Up
Jon Hall6aec96b2015-01-19 14:49:31 -0800905 """
906 # NOTE: You should probably run a topology check after this
Jon Hall73cf9cc2014-11-20 22:28:38 -0800907 import time
Jon Hall669173b2014-12-17 11:36:30 -0800908
Jon Hall8f89dda2015-01-22 16:03:33 -0800909 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall6aec96b2015-01-19 14:49:31 -0800910 switch = main.params[ 'kill' ][ 'switch' ]
911 switchDPID = main.params[ 'kill' ][ 'dpid' ]
912 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800913 description = "Adding a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -0800914 main.log.report( description )
915 main.case( description )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800916
Jon Hall6aec96b2015-01-19 14:49:31 -0800917 main.step( "Add back " + switch )
918 main.log.report( "Adding back " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800919 main.Mininet1.addSwitch( switch, dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -0800920 # TODO: New dpid or same? Ask Thomas?
921 for peer in links:
Jon Hall8f89dda2015-01-22 16:03:33 -0800922 main.Mininet1.addLink( switch, peer )
923 main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
924 ip1=ONOS1Ip, port1=ONOS1Port )
Jon Hall6aec96b2015-01-19 14:49:31 -0800925 main.log.info(
926 "Waiting " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800927 str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -0800928 " seconds for switch up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800929 time.sleep( switchSleep )
930 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -0800931 # Peek at the deleted switch
932 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -0800933 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800934 if device and device[ 'available' ]:
Jon Hall94fd0472014-12-08 11:52:42 -0800935 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800936 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall8f89dda2015-01-22 16:03:33 -0800937 onpass="add switch succesful",
938 onfail="Failed to add switch?" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800939
Jon Hall6aec96b2015-01-19 14:49:31 -0800940 def CASE13( self, main ):
941 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800942 Clean up
Jon Hall6aec96b2015-01-19 14:49:31 -0800943 """
Jon Hall73cf9cc2014-11-20 22:28:38 -0800944 import os
945 import time
Jon Hall6aec96b2015-01-19 14:49:31 -0800946 # printing colors to terminal
Jon Hall669173b2014-12-17 11:36:30 -0800947 colors = {}
Jon Hall6aec96b2015-01-19 14:49:31 -0800948 colors[ 'cyan' ] = '\033[96m'
949 colors[ 'purple' ] = '\033[95m'
950 colors[ 'blue' ] = '\033[94m'
951 colors[ 'green' ] = '\033[92m'
952 colors[ 'yellow' ] = '\033[93m'
953 colors[ 'red' ] = '\033[91m'
954 colors[ 'end' ] = '\033[0m'
Jon Hall73cf9cc2014-11-20 22:28:38 -0800955 description = "Test Cleanup"
Jon Hall6aec96b2015-01-19 14:49:31 -0800956 main.log.report( description )
957 main.case( description )
958 main.step( "Killing tcpdumps" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800959 main.Mininet2.stopTcpdump()
Jon Hall73cf9cc2014-11-20 22:28:38 -0800960
Jon Hall6aec96b2015-01-19 14:49:31 -0800961 main.step( "Checking ONOS Logs for errors" )
962 print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
Jon Hall8f89dda2015-01-22 16:03:33 -0800963 colors[ 'end' ]
964 print main.ONOSbench.checkLogs( ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800965 main.step( "Copying MN pcap and ONOS log files to test station" )
Jon Hall73cf9cc2014-11-20 22:28:38 -0800966 testname = main.TEST
Jon Hall8f89dda2015-01-22 16:03:33 -0800967 teststationUser = main.params[ 'TESTONUSER' ]
968 teststationIP = main.params[ 'TESTONIP' ]
Jon Hall6aec96b2015-01-19 14:49:31 -0800969 # NOTE: MN Pcap file is being saved to ~/packet_captures
Jon Hall73cf9cc2014-11-20 22:28:38 -0800970 # scp this file as MN and TestON aren't necessarily the same vm
Jon Hall6aec96b2015-01-19 14:49:31 -0800971 # FIXME: scp
972 # mn files
973 # TODO: Load these from params
974 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -0800975 logFolder = "/opt/onos/log/"
976 logFiles = [ "karaf.log", "karaf.log.1" ]
Jon Hall6aec96b2015-01-19 14:49:31 -0800977 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -0800978 dstDir = "~/packet_captures/"
979 for f in logFiles:
980 main.ONOSbench.handle.sendline( "scp sdn@" + ONOS1Ip + ":" +
981 logFolder + f + " " +
982 teststationUser + "@" +
983 teststationIP + ":" + dstDir +
Jon Hall6aec96b2015-01-19 14:49:31 -0800984 str( testname ) + "-ONOS1-" + f )
985 main.ONOSbench.handle.expect( "\$" )
Jon Hall94fd0472014-12-08 11:52:42 -0800986 print main.ONOSbench.handle.before
Jon Hall73cf9cc2014-11-20 22:28:38 -0800987
Jon Hall6aec96b2015-01-19 14:49:31 -0800988 # std*.log's
989 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -0800990 logFolder = "/opt/onos/var/"
991 logFiles = [ "stderr.log", "stdout.log" ]
Jon Hall6aec96b2015-01-19 14:49:31 -0800992 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -0800993 dstDir = "~/packet_captures/"
994 for f in logFiles:
995 main.ONOSbench.handle.sendline( "scp sdn@" + ONOS1Ip + ":" +
996 logFolder + f + " " +
997 teststationUser + "@" +
998 teststationIP + ":" + dstDir +
Jon Hall6aec96b2015-01-19 14:49:31 -0800999 str( testname ) + "-ONOS1-" + f )
1000 # sleep so scp can finish
1001 time.sleep( 10 )
1002 main.step( "Packing and rotating pcap archives" )
1003 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001004
Jon Hall6aec96b2015-01-19 14:49:31 -08001005 # TODO: actually check something here
1006 utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001007 onpass="Test cleanup successful",
1008 onfail="Test cleanup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001009
Jon Hall6aec96b2015-01-19 14:49:31 -08001010 def CASE14( self, main ):
1011 """
Jon Hall669173b2014-12-17 11:36:30 -08001012 start election app on all onos nodes
Jon Hall6aec96b2015-01-19 14:49:31 -08001013 """
Jon Hall8f89dda2015-01-22 16:03:33 -08001014 leaderResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001015 # install app on onos 1
1016 main.log.info( "Install leadership election app" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001017 main.ONOScli1.featureInstall( "onos-app-election" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001018 # wait for election
1019 # check for leader
Jon Hall8f89dda2015-01-22 16:03:33 -08001020 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001021 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08001022 if leader == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08001023 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08001024 pass
Jon Hall6aec96b2015-01-19 14:49:31 -08001025 elif leader is None:
1026 # No leader elected
1027 main.log.report( "No leader was elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001028 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001029 elif leader == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001030 # error in response
1031 # TODO: add check for "Command not found:" in the driver, this
1032 # means the app isn't loaded
Jon Hall8f89dda2015-01-22 16:03:33 -08001033 main.log.report( "Something is wrong with electionTestLeader" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001034 " function, check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001035 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001036 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001037 # error in response
1038 main.log.report(
Jon Hall8f89dda2015-01-22 16:03:33 -08001039 "Unexpected response from electionTestLeader function:'" +
1040 str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001041 "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001042 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001043
Jon Hall6aec96b2015-01-19 14:49:31 -08001044 # install on other nodes and check for leader.
1045 # Should be onos1 and each app should show the same leader
Jon Hall8f89dda2015-01-22 16:03:33 -08001046 for controller in range( 2, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001047 # loop through ONOScli handlers
1048 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001049 node.featureInstall( "onos-app-election" )
1050 leaderN = node.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001051 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08001052 if leaderN == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08001053 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08001054 pass
1055 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001056 # error in response
1057 # TODO: add check for "Command not found:" in the driver, this
1058 # means the app isn't loaded
1059 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001060 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001061 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001062 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001063 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08001064 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001065 main.log.report( "ONOS" + str( controller ) + " sees " +
1066 str( leaderN ) +
1067 " as the leader of the election app. Leader" +
1068 " should be " +
1069 str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001070 if leaderResult:
Jon Hall6aec96b2015-01-19 14:49:31 -08001071 main.log.report( "Leadership election tests passed( consistent " +
1072 "view of leader across listeners and a leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001073 "was elected )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001074 utilities.assert_equals(
1075 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001076 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001077 onpass="Leadership election passed",
1078 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08001079
Jon Hall6aec96b2015-01-19 14:49:31 -08001080 def CASE15( self, main ):
1081 """
Jon Hall669173b2014-12-17 11:36:30 -08001082 Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -08001083 """
Jon Hall8f89dda2015-01-22 16:03:33 -08001084 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08001085 description = "Check that Leadership Election is still functional"
Jon Hall6aec96b2015-01-19 14:49:31 -08001086 main.log.report( description )
1087 main.case( description )
1088 main.step( "Find current leader and withdraw" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001089 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001090 # TODO: do some sanity checking on leader before using it
Jon Hall8f89dda2015-01-22 16:03:33 -08001091 withdrawResult = main.FALSE
1092 if leader == ONOS1Ip:
1093 oldLeader = getattr( main, "ONOScli1" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001094 elif leader is None or leader == main.FALSE:
1095 main.log.report(
1096 "Leader for the election app should be an ONOS node," +
1097 "instead got '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001098 str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001099 "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001100 leaderResult = main.FALSE
1101 withdrawResult = oldLeader.electionTestWithdraw()
Jon Hall6aec96b2015-01-19 14:49:31 -08001102 utilities.assert_equals(
1103 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001104 actual=withdrawResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001105 onpass="App was withdrawn from election",
1106 onfail="App was not withdrawn from election" )
Jon Hall669173b2014-12-17 11:36:30 -08001107
Jon Hall6aec96b2015-01-19 14:49:31 -08001108 main.step( "Make sure new leader is elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001109 leaderList = []
1110 leaderN = main.ONOScli1.electionTestLeader()
Jon Hall669173b2014-12-17 11:36:30 -08001111 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08001112 main.log.report( "ONOS still sees " + str( leaderN ) +
1113 " as leader after they withdrew" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001114 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001115 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001116 # error in response
1117 # TODO: add check for "Command not found:" in the driver, this
1118 # means the app isn't loaded
Jon Hall8f89dda2015-01-22 16:03:33 -08001119 main.log.report( "Something is wrong with electionTestLeader " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001120 "function, check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001121 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001122 elif leaderN is None:
1123 main.log.info(
1124 "There is no leader after the app withdrew from election" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001125 if leaderResult:
1126 main.log.report( "Leadership election tests passed( There is no " +
1127 "leader after the old leader resigned )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001128 utilities.assert_equals(
1129 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001130 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001131 onpass="Leadership election passed",
1132 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08001133
Jon Hall6aec96b2015-01-19 14:49:31 -08001134 main.step(
Jon Hall8f89dda2015-01-22 16:03:33 -08001135 "Run for election on old leader( just so everyone is in the hat )" )
1136 runResult = oldLeader.electionTestRun()
Jon Hall6aec96b2015-01-19 14:49:31 -08001137 utilities.assert_equals(
1138 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001139 actual=runResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001140 onpass="App re-ran for election",
1141 onfail="App failed to run for election" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001142 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001143 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08001144 if leader == ONOS1Ip:
1145 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08001146 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001147 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001148 # TODO: assert on run and withdraw results?
Jon Hall669173b2014-12-17 11:36:30 -08001149
Jon Hall6aec96b2015-01-19 14:49:31 -08001150 utilities.assert_equals(
1151 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001152 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001153 onpass="Leadership election passed",
1154 onfail="ONOS1's election app was not leader after it re-ran " +
1155 "for election" )